@hawkeyexl/manni 4.3.0 → 4.4.0

package/dist/chunk-NWZ2SLYL.js

@@ -0,0 +1,2957 @@
+#!/usr/bin/env node
+import {
+  isTurnBudgetSkip,
+  outputTail,
+  parseDuration,
+  realExec
+} from "./chunk-LJ2RDH3F.js";
+import {
+  EXECUTION_GRANTS,
+  NOT_GRANTED_REASON,
+  configuredGrants,
+  grantsFor,
+  isExecutionGrant,
+  unknownGrantsMessage
+} from "./chunk-YNHZHSSM.js";
+import {
+  DocevalsError
+} from "./chunk-SE4JN5XN.js";
+import {
+  declaredAt
+} from "./chunk-H2DLH2JI.js";
+import {
+  metaProvenanceEntries
+} from "./chunk-K3D6FPV6.js";
+import {
+  familyMarks,
+  loadExternalMetadata,
+  memberOf,
+  mergeWithMarks,
+  provenanceEntries
+} from "./chunk-RLR4TKUZ.js";
+import {
+  __default,
+  classifyRef,
+  extractFrontmatter
+} from "./chunk-JVTNMVYU.js";
+import {
+  PROVIDERS_KEY,
+  assertKnownProvider,
+  findConfigFileSync,
+  ownsKey,
+  parseCollections,
+  parseProviders,
+  readConfigFileSync,
+  selectCollections,
+  warn
+} from "./chunk-Q35YIDBV.js";
+import {
+  errorMessage
+} from "./chunk-G3T47CFV.js";
+
+// src/docevals/core/config.ts
+import { resolve, dirname } from "node:path";
+import { parse as parseYaml } from "yaml";
+import { Ajv2020 } from "ajv/dist/2020.js";
+
+// src/docevals/core/config-schema.json
+var config_schema_default = {
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  $id: "manni:config:docevals",
+  title: "manni docevals configuration",
+  description: "The docevals: section of manni.config.yaml. Every manni tool reads its own top-level key; keys that belong to sibling tools are ignored here. Section keys are camelCase, as every manni tool's are. The entries under evals, criteria and suites keep the kebab-case spelling of the frontmatter vocabulary, because they are the same entries a page carries.",
+  type: "object",
+  properties: {
+    docevals: {
+      $ref: "#/$defs/docevalsConfig"
+    }
+  },
+  additionalProperties: true,
+  $defs: {
+    docevalsConfig: {
+      type: "object",
+      properties: {
+        baseline: {
+          type: [
+            "string",
+            "null"
+          ],
+          description: "Path to the findings baseline, relative to this config file. A baseline records today's findings so a run fails only on new ones (ADR 01017). Null (the default) means no baseline. `--baseline`/`--no-baseline` override this; a bare `--write-baseline` records into *this* path, so a repo that points it somewhere custom cannot accidentally record into a file nothing reads."
+        },
+        defaults: {
+          type: "object",
+          properties: {
+            suite: {
+              type: [
+                "string",
+                "null"
+              ]
+            },
+            failFast: {
+              type: "boolean"
+            },
+            concurrency: {
+              type: "integer",
+              minimum: 1
+            }
+          },
+          additionalProperties: false
+        },
+        provider: {
+          type: "string",
+          minLength: 1,
+          description: "Which provider judges, generates, fills and promotes: auto | anthropic | openai | claude-cli | llama-cpp. Unset, the family's top-level providers.provider decides, then auto, detected exactly as manni meta fill detects: an Anthropic key, then an OpenAI key, then the Claude CLI, then a local model. --provider wins over it, and an ai eval's own provider wins over it for that eval. Connection settings live in the family's top-level providers: map."
+        },
+        model: {
+          type: "string",
+          minLength: 1,
+          description: "The model, within the provider named beside it. A model is carried only to the provider its own level names: a provider from --provider, an eval or providers.provider that differs from this one takes its own default instead. With no provider here, it goes to whichever provider is in force, which must not be auto, because a model name does not say which provider owns it. Unset (the default) takes the provider's default model from the inference library; manni pins none. --model wins over it."
+        },
+        judge: {
+          type: "object",
+          properties: {
+            ensembleRuns: {
+              type: "integer",
+              minimum: 1
+            },
+            concurrency: {
+              type: "integer",
+              minimum: 1
+            },
+            temperature: {
+              type: "number",
+              minimum: 0
+            },
+            zones: {
+              type: "object",
+              properties: {
+                autoPass: {
+                  type: "number",
+                  minimum: 0,
+                  maximum: 1
+                },
+                autoFail: {
+                  type: "number",
+                  minimum: 0,
+                  maximum: 1
+                }
+              },
+              additionalProperties: false
+            },
+            falsePositiveAlert: {
+              type: "number",
+              minimum: 0,
+              maximum: 1
+            },
+            cacheDir: {
+              type: "string"
+            },
+            maxTurns: {
+              $ref: "#/$defs/maxTurns"
+            },
+            chunkChars: {
+              type: "integer",
+              minimum: 1,
+              default: 12e3,
+              description: "Characters of page content per judge call. A page longer than this is graded by gathering evidence from each part and judging the collected evidence once."
+            }
+          },
+          additionalProperties: false
+        },
+        scripts: {
+          type: "object",
+          properties: {
+            dir: {
+              type: "string"
+            },
+            configDir: {
+              type: "string"
+            },
+            timeoutMs: {
+              type: "integer",
+              minimum: 1
+            }
+          },
+          additionalProperties: false
+        },
+        execution: {
+          type: "object",
+          additionalProperties: false,
+          properties: {
+            allow: {
+              type: "array",
+              items: {
+                enum: [
+                  "frontmatter-commands"
+                ]
+              },
+              default: ["frontmatter-commands"],
+              description: "Which content-authored code this corpus may execute. Absent, every grant holds; list fewer to narrow it, or write [] to run none. frontmatter-commands lets `command` evals declared in page frontmatter run. That is code a page author controls, so a run over an untrusted pull request passes --no-execution."
+            }
+          },
+          description: "Operator grant for executing content-authored code."
+        },
+        fill: {
+          type: "object",
+          properties: {
+            confidenceThreshold: {
+              type: "number",
+              minimum: 0,
+              maximum: 1
+            },
+            maxEvalsPerPage: {
+              type: "integer",
+              minimum: 1
+            },
+            temperature: {
+              type: "number",
+              minimum: 0
+            },
+            cacheDir: {
+              type: "string"
+            },
+            maxTurns: {
+              $ref: "#/$defs/maxTurns"
+            },
+            chunkChars: {
+              type: "integer",
+              minimum: 1,
+              default: 12e3,
+              description: "Characters of page content per fill call. A longer page is proposed against in parts and the proposals merged by id, keeping the highest confidence."
+            }
+          },
+          additionalProperties: false
+        },
+        evals: {
+          type: "object",
+          propertyNames: {
+            pattern: "^[a-z0-9][a-z0-9-]*$"
+          },
+          additionalProperties: {
+            $ref: "#/$defs/evalDef"
+          },
+          description: "The central library of named evals. The map key is the eval's id \u2014 the same id a page's `use:` reference names."
+        },
+        criteria: {
+          type: "object",
+          propertyNames: {
+            pattern: "^[a-z0-9][a-z0-9-]*$"
+          },
+          additionalProperties: {
+            $ref: "#/$defs/criterionDef"
+          },
+          description: "Named groups of evals scored as one unit. A criterion contributes a single weighted outcome to its suite and its members contribute none, so writing three checks as a group does not let them outvote three standalone evals."
+        },
+        suites: {
+          type: "object",
+          propertyNames: {
+            pattern: "^[a-z0-9][a-z0-9-]*$"
+          },
+          additionalProperties: {
+            $ref: "#/$defs/suiteDef"
+          }
+        }
+      },
+      additionalProperties: false
+    },
+    maxTurns: {
+      type: [
+        "integer",
+        "null"
+      ],
+      minimum: 1,
+      description: "Stop after this many turns. Counts work, not pages. Under `judge` a turn is one ensemble run, so a judged eval spends `judge.ensembleRuns` of them; a run may make a second provider call when the first response fails schema validation, so this is a floor on calls rather than an exact count. Under `fill` a turn is exactly one call. A cached result spends none. Null (the default) is unbounded. Replaces the `max-cost-usd` ceiling, which could not be honored exactly under concurrency \u2014 see ADR 01019."
+    },
+    evalDef: {
+      type: "object",
+      properties: {
+        assertion: {
+          type: "string",
+          minLength: 1
+        },
+        type: {
+          enum: [
+            "capability",
+            "regression"
+          ]
+        },
+        grader: {
+          type: "string",
+          pattern: "^(ai|command|human|tool:[a-z0-9][a-z0-9-]*)$"
+        },
+        provider: {
+          type: "string",
+          minLength: 1
+        },
+        model: {
+          type: "string",
+          minLength: 1,
+          description: "Model judging this ai eval, within provider. A CLI --model still wins over it. Meaningful only on ai evals."
+        },
+        runs: {
+          type: "integer",
+          minimum: 1,
+          maximum: 50,
+          description: "Ensemble runs for this eval, overriding judge.ensembleRuns. A CLI --runs still wins over it. Capped because it multiplies judge cost directly. Meaningful only on ai evals."
+        },
+        evidence: {
+          type: "string",
+          minLength: 1
+        },
+        target: {
+          oneOf: [
+            {
+              enum: [
+                "body",
+                "raw",
+                "frontmatter"
+              ]
+            },
+            {
+              type: "object",
+              additionalProperties: false,
+              required: [
+                "source",
+                "path"
+              ],
+              properties: {
+                source: {
+                  const: "file"
+                },
+                path: {
+                  type: "string",
+                  minLength: 1
+                }
+              }
+            }
+          ],
+          default: "body",
+          description: "Which bytes the grader receives: the page body with frontmatter stripped (body), the file verbatim (raw), the parsed frontmatter alone (frontmatter), or a companion file beside the page. Distinct from evidence, which hints where to look within what is graded."
+        },
+        examples: {
+          type: "object",
+          properties: {
+            pass: {
+              $ref: "#/$defs/anchorList"
+            },
+            fail: {
+              $ref: "#/$defs/anchorList"
+            }
+          },
+          additionalProperties: false
+        },
+        command: {
+          type: "array",
+          items: {
+            type: "string",
+            minLength: 1
+          },
+          minItems: 1
+        },
+        "success-exit-codes": {
+          type: "array",
+          minItems: 1,
+          items: {
+            type: "integer"
+          }
+        },
+        "timeout-ms": {
+          type: "integer",
+          minimum: 1
+        },
+        "generated-assertion-hash": {
+          type: "string"
+        },
+        options: {
+          type: "object"
+        },
+        severity: {
+          enum: [
+            "notice",
+            "warning",
+            "error"
+          ]
+        },
+        weight: {
+          type: "number",
+          exclusiveMinimum: 0,
+          default: 1,
+          description: "Relative contribution to its suite's pass rate. Does not change this eval's own pass/fail: the eval still passes or fails on its own terms, and weight only decides how much that outcome moves the suite total against target-pass-rate. Zero is excluded deliberately - a weightless eval is a silent disable, and skip already means that, loudly."
+        }
+      },
+      additionalProperties: false,
+      dependentRequired: {
+        "generated-assertion-hash": [
+          "command"
+        ]
+      },
+      allOf: [
+        {
+          if: {
+            properties: {
+              grader: {
+                const: "ai"
+              }
+            },
+            required: [
+              "grader"
+            ]
+          },
+          then: {
+            required: [
+              "assertion"
+            ]
+          }
+        },
+        {
+          if: {
+            not: {
+              required: [
+                "grader"
+              ]
+            }
+          },
+          then: {
+            required: [
+              "assertion"
+            ]
+          }
+        },
+        {
+          if: {
+            properties: {
+              grader: {
+                const: "command"
+              }
+            },
+            required: [
+              "grader"
+            ]
+          },
+          then: {
+            anyOf: [
+              {
+                required: [
+                  "assertion"
+                ]
+              },
+              {
+                required: [
+                  "command"
+                ]
+              }
+            ]
+          }
+        },
+        {
+          if: {
+            properties: {
+              grader: {
+                const: "human"
+              }
+            },
+            required: [
+              "grader"
+            ]
+          },
+          then: {
+            required: [
+              "assertion"
+            ]
+          }
+        },
+        {
+          if: {
+            required: [
+              "command"
+            ]
+          },
+          then: {
+            properties: {
+              grader: {
+                const: "command"
+              }
+            },
+            required: [
+              "grader"
+            ]
+          }
+        },
+        {
+          if: {
+            anyOf: [
+              {
+                required: [
+                  "success-exit-codes"
+                ]
+              },
+              {
+                required: [
+                  "timeout-ms"
+                ]
+              }
+            ]
+          },
+          then: {
+            properties: {
+              grader: {
+                const: "command"
+              }
+            },
+            required: [
+              "grader"
+            ]
+          }
+        }
+      ]
+    },
+    anchorList: {
+      if: {
+        type: "array"
+      },
+      then: {
+        type: "array",
+        minItems: 1,
+        uniqueItems: true,
+        items: {
+          type: "string",
+          minLength: 1
+        }
+      },
+      else: {
+        type: "string",
+        minLength: 1
+      }
+    },
+    suiteDef: {
+      type: "object",
+      properties: {
+        "target-pass-rate": {
+          type: "number",
+          minimum: 0,
+          maximum: 1
+        },
+        evals: {
+          type: "array",
+          items: {
+            type: "string"
+          }
+        },
+        criteria: {
+          type: "array",
+          items: {
+            type: "string",
+            pattern: "^[a-z0-9][a-z0-9-]*$"
+          },
+          description: "Criteria scored in this suite, alongside `evals`."
+        }
+      },
+      additionalProperties: false
+    },
+    criterionDef: {
+      type: "object",
+      additionalProperties: false,
+      required: [
+        "evals"
+      ],
+      properties: {
+        evals: {
+          type: "array",
+          minItems: 1,
+          items: {
+            type: "string",
+            pattern: "^[a-z0-9][a-z0-9-]*$"
+          },
+          description: "Ids of the evals this criterion is made of."
+        },
+        combine: {
+          enum: [
+            "all",
+            "any"
+          ],
+          default: "all",
+          description: "all: every member must pass. any: one passing member is enough."
+        },
+        weight: {
+          type: "number",
+          exclusiveMinimum: 0,
+          default: 1,
+          description: "The criterion's contribution to its suite's pass rate."
+        }
+      }
+    }
+  }
+};
+
+// src/docevals/core/split.ts
+var DEFAULT_CHUNK_CHARS = 12e3;
+function splitBody(body, chunkChars) {
+  if (body.length <= chunkChars) return [body];
+  const chunks = [];
+  let at = 0;
+  while (at < body.length) {
+    const end = Math.min(at + chunkChars, body.length);
+    let cut = end;
+    if (end < body.length) {
+      const nl = body.lastIndexOf("\n", end - 1);
+      if (nl > at) cut = nl + 1;
+    }
+    chunks.push(body.slice(at, cut));
+    at = cut;
+  }
+  return chunks;
+}
+function looksLikeOverflow(message) {
+  return /context|too long|too large|token limit|maximum.*tokens|exceeds/i.test(
+    message
+  );
+}
+function partLabel(index, total) {
+  return `part ${String(index + 1)} of ${String(total)}`;
+}
+
+// src/docevals/core/config.ts
+function anchorList(v) {
+  if (v === void 0) return void 0;
+  return Array.isArray(v) ? v : [v];
+}
+function normalizeEvalDef(raw) {
+  const examples = raw.examples === void 0 ? void 0 : {
+    ...anchorList(raw.examples.pass) !== void 0 && {
+      pass: anchorList(raw.examples.pass)
+    },
+    ...anchorList(raw.examples.fail) !== void 0 && {
+      fail: anchorList(raw.examples.fail)
+    }
+  };
+  return {
+    assertion: raw.assertion,
+    type: raw.type,
+    grader: raw.grader,
+    provider: raw.provider,
+    evidence: raw.evidence,
+    examples,
+    command: raw.command,
+    successExitCodes: raw["success-exit-codes"],
+    timeoutMs: raw["timeout-ms"],
+    generatedAssertionHash: raw["generated-assertion-hash"],
+    options: raw.options,
+    severity: raw.severity,
+    weight: raw.weight,
+    target: raw.target,
+    model: raw.model,
+    runs: raw.runs
+  };
+}
+var NAMESPACE = "docevals";
+var DEFAULT_CONFIG_FILENAME = "manni.config.yaml";
+var PRE_RENAME_ROOT_KEYS = [
+  "version",
+  "files",
+  "defaults",
+  "provider",
+  "judge",
+  "scripts",
+  "fill",
+  "evals",
+  "suites"
+];
+var ajv = new Ajv2020({ allErrors: true, allowUnionTypes: true, verbose: true });
+var validateConfig = ajv.compile(config_schema_default);
+function isAssertionHashWrapper(value) {
+  return value != null && typeof value === "object" && !Array.isArray(value) && "assertionHash" in value;
+}
+function findPreKebabKeys(node, path) {
+  if (node == null || typeof node !== "object" || Array.isArray(node)) return [];
+  const found = [];
+  for (const [key, value] of Object.entries(node)) {
+    if (/[a-z0-9][A-Z]/.test(key)) {
+      found.push({
+        at: `${path}.${key}`,
+        becomes: key.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase()
+      });
+    }
+    if (key === "generated" && isAssertionHashWrapper(value)) {
+      found.push({ at: `${path}.generated`, becomes: "generated-assertion-hash" });
+      continue;
+    }
+    found.push(...findPreKebabKeys(value, `${path}.${key}`));
+  }
+  return found;
+}
+var ENTRY_SECTIONS = ["evals", "criteria", "suites"];
+function camelCaseHint(key, parentSchema) {
+  const camel = key.replace(/-([a-z0-9])/g, (_, c) => c.toUpperCase());
+  if (camel === key || parentSchema === null || typeof parentSchema !== "object") return "";
+  const properties = parentSchema.properties;
+  return properties !== null && typeof properties === "object" && Object.hasOwn(properties, camel) ? `; did you mean "${camel}"?` : "";
+}
+function movedProviderHint(instancePath, keyword) {
+  return instancePath === `/${NAMESPACE}/provider` && keyword === "type" ? `; "provider" is now a provider name; per-provider settings moved to the top-level providers: map` : "";
+}
+function parseConfig(text, configPath) {
+  let raw;
+  try {
+    raw = parseYaml(text);
+  } catch (e) {
+    throw new DocevalsError(
+      `Invalid YAML in ${configPath}: ${errorMessage(e)}`
+    );
+  }
+  if (raw == null || typeof raw !== "object" || Array.isArray(raw)) {
+    throw new DocevalsError(`Invalid config in ${configPath}: root must be an object`);
+  }
+  const doc = raw;
+  const wrapped = NAMESPACE in doc;
+  const toError = (message) => new DocevalsError(message);
+  const collections = Object.hasOwn(doc, COLLECTIONS_KEY) ? parseCollections(doc[COLLECTIONS_KEY], configPath, toError) : [];
+  const providers = Object.hasOwn(doc, PROVIDERS_KEY) ? parseProviders(doc[PROVIDERS_KEY], configPath, dirname(resolve(configPath)), toError) : {};
+  return parseConfigSection(wrapped ? doc[NAMESPACE] : doc, configPath, wrapped, {
+    source: configPath,
+    collections,
+    providers
+  });
+}
+var COLLECTIONS_KEY = "collections";
+var CONFIG_REF = "https://hawkeyexl.github.io/manni/meta/reference/configuration/";
+var MOVED_KEY = "files";
+var MOVED_PROVIDERS_KEY = "providers";
+function parseConfigSection(value, configPath, wrapped, file = { source: configPath, collections: [] }) {
+  const raw = wrapped ? { [NAMESPACE]: value ?? {} } : value ?? {};
+  if (typeof raw !== "object" || Array.isArray(raw)) {
+    throw new DocevalsError(`Invalid config in ${configPath}: root must be an object`);
+  }
+  if (!(NAMESPACE in raw)) {
+    const stray = PRE_RENAME_ROOT_KEYS.filter((k) => k in raw);
+    if (stray.length > 0) {
+      throw new DocevalsError(
+        `Invalid config in ${configPath}: found ${stray.join(", ")} at the root, but no "${NAMESPACE}:" key.
+manni.config.yaml is shared across the manni family, so every docevals setting belongs under a top-level "${NAMESPACE}:" key. Indent the file's contents one level and add that key.`
+      );
+    }
+  }
+  const section = raw[NAMESPACE];
+  if (section !== null && typeof section === "object" && !Array.isArray(section) && Object.hasOwn(section, MOVED_KEY)) {
+    throw new DocevalsError(
+      `${file.source}: "${MOVED_KEY}" is no longer a docevals key. Document sets are declared once for every tool, under a top-level collections: list. See ${CONFIG_REF}#collections`
+    );
+  }
+  if (section !== null && typeof section === "object" && !Array.isArray(section) && Object.hasOwn(section, MOVED_PROVIDERS_KEY)) {
+    throw new DocevalsError(
+      `${file.source}: "${MOVED_PROVIDERS_KEY}" is no longer a docevals key. Provider settings are declared once for every tool, under a top-level providers: map. See ${CONFIG_REF}#providers`
+    );
+  }
+  const preKebab = ENTRY_SECTIONS.flatMap(
+    (name) => section !== null && typeof section === "object" && !Array.isArray(section) ? findPreKebabKeys(section[name], `${NAMESPACE}.${name}`) : []
+  );
+  if (preKebab.length > 0) {
+    throw new DocevalsError(
+      `Invalid config in ${configPath}: camelCase keys are not read in eval, criterion or suite entries.
+` + preKebab.map((p) => `  ${p.at} -> ${p.becomes}`).join("\n") + `
+Those entries are kebab-case, matching the frontmatter vocabulary.`
+    );
+  }
+  const ns = raw[NAMESPACE];
+  const scriptsSection = ns && typeof ns === "object" ? ns.scripts : void 0;
+  if (scriptsSection && typeof scriptsSection === "object" && "allow-frontmatter-commands" in scriptsSection) {
+    throw new DocevalsError(
+      `Invalid config in ${configPath}: scripts.allow-frontmatter-commands has been replaced by execution.allow.
+  Command evals run unless it narrows them; write \`execution: { allow: [] }\` to stop them.`
+    );
+  }
+  const unknownGrants = configuredGrants(ns).filter((g) => !isExecutionGrant(g));
+  if (unknownGrants.length > 0) {
+    throw new DocevalsError(
+      `Invalid config in ${configPath}: ${unknownGrantsMessage(unknownGrants)}`
+    );
+  }
+  if (!validateConfig(raw)) {
+    const details = (validateConfig.errors ?? []).map((e) => {
+      const extra = e.keyword === "additionalProperties" ? e.params.additionalProperty : void 0;
+      if (extra !== void 0) {
+        return `  ${e.instancePath || "/"}: unknown key "${extra}"${camelCaseHint(extra, e.parentSchema)}`;
+      }
+      return `  ${e.instancePath || "/"}: ${e.message ?? "is invalid"}${movedProviderHint(e.instancePath, e.keyword)}`;
+    }).join("\n");
+    throw new DocevalsError(`Invalid config in ${configPath}:
+${details}`);
+  }
+  const r = raw[NAMESPACE] ?? {};
+  const abs = resolve(configPath);
+  const dir = dirname(abs);
+  const provider = r.provider ?? null;
+  if (provider !== null) assertKnownProvider(provider, (message) => new DocevalsError(message));
+  const suites = {};
+  for (const [name, def] of Object.entries(r.suites ?? {})) {
+    suites[name] = {
+      targetPassRate: def["target-pass-rate"] ?? 1,
+      evals: def.evals ?? [],
+      criteria: def.criteria ?? []
+    };
+  }
+  const criteria = {};
+  for (const [name, def] of Object.entries(r.criteria ?? {})) {
+    criteria[name] = {
+      evals: def.evals ?? [],
+      combine: def.combine ?? "all",
+      weight: def.weight ?? 1
+    };
+  }
+  const config = {
+    collections: file.collections,
+    configSource: file.source,
+    defaults: {
+      suite: r.defaults?.suite ?? null,
+      failFast: r.defaults?.failFast ?? false,
+      concurrency: r.defaults?.concurrency ?? 4
+    },
+    provider,
+    model: r.model ?? null,
+    providers: file.providers ?? {},
+    baseline: r.baseline ?? null,
+    judge: {
+      ensembleRuns: r.judge?.ensembleRuns ?? 3,
+      // Falls back to the corpus-wide setting, so an unset value behaves
+      // exactly as it did before this knob existed. It is separable because
+      // the judge's right parallelism is not the deterministic graders': a
+      // local in-process model serves one context at a time (ADR 01039).
+      concurrency: r.judge?.concurrency ?? r.defaults?.concurrency ?? 4,
+      temperature: r.judge?.temperature ?? 0,
+      zones: {
+        autoPass: r.judge?.zones?.autoPass ?? 0.8,
+        autoFail: r.judge?.zones?.autoFail ?? 0.8
+      },
+      falsePositiveAlert: r.judge?.falsePositiveAlert ?? 0.15,
+      cacheDir: r.judge?.cacheDir ?? ".manni/docevals/cache",
+      maxTurns: r.judge?.maxTurns ?? null,
+      chunkChars: r.judge?.chunkChars ?? DEFAULT_CHUNK_CHARS
+    },
+    scripts: {
+      dir: r.scripts?.dir ?? "{docDir}/manni-docevals",
+      configDir: r.scripts?.configDir ?? "manni-docevals-scripts",
+      timeoutMs: r.scripts?.timeoutMs ?? 3e4
+    },
+    execution: { allow: [...r.execution?.allow ?? EXECUTION_GRANTS] },
+    fill: {
+      confidenceThreshold: r.fill?.confidenceThreshold ?? 0.7,
+      maxEvalsPerPage: r.fill?.maxEvalsPerPage ?? 3,
+      temperature: r.fill?.temperature ?? 0,
+      cacheDir: r.fill?.cacheDir ?? ".manni/docevals/cache/fill",
+      maxTurns: r.fill?.maxTurns ?? null,
+      chunkChars: r.fill?.chunkChars ?? DEFAULT_CHUNK_CHARS
+    },
+    evals: Object.fromEntries(
+      Object.entries(r.evals ?? {}).map(([name, def]) => [
+        name,
+        normalizeEvalDef(def)
+      ])
+    ),
+    criteria,
+    suites,
+    configPath: abs,
+    configDir: dir
+  };
+  for (const [suiteName, suite] of Object.entries(config.suites)) {
+    for (const evalName of suite.evals) {
+      if (!(evalName in config.evals)) {
+        throw new DocevalsError(
+          `Invalid config in ${configPath}: suite "${suiteName}" references undefined eval "${evalName}"`
+        );
+      }
+    }
+  }
+  for (const [critName, crit] of Object.entries(config.criteria)) {
+    for (const evalName of crit.evals) {
+      if (!(evalName in config.evals)) {
+        throw new DocevalsError(
+          `Invalid config in ${configPath}: criterion "${critName}" references undefined eval "${evalName}"`
+        );
+      }
+    }
+  }
+  for (const [suiteName, suite] of Object.entries(config.suites)) {
+    for (const critName of suite.criteria) {
+      if (!(critName in config.criteria)) {
+        throw new DocevalsError(
+          `Invalid config in ${configPath}: suite "${suiteName}" references undefined criterion "${critName}"`
+        );
+      }
+    }
+  }
+  if (config.defaults.suite && !(config.defaults.suite in config.suites)) {
+    throw new DocevalsError(
+      `Invalid config in ${configPath}: defaults.suite "${config.defaults.suite}" is not a defined suite`
+    );
+  }
+  return config;
+}
+function loadConfig(path, cwd = process.cwd()) {
+  const file = path ? readConfigFileSync(path, cwd, CONFIG_FILE) : findConfigFileSync(cwd, CONFIG_FILE);
+  if (file === null) return defaultConfig(cwd);
+  return parseConfigSection(file.value, file.path, file.wrapped, {
+    source: file.source,
+    collections: file.collections,
+    providers: file.providers ?? {}
+  });
+}
+function defaultConfig(cwd = process.cwd()) {
+  return {
+    ...parseConfigSection(null, resolve(cwd, DEFAULT_CONFIG_FILENAME), false),
+    configSource: null
+  };
+}
+function loadRunConfig(opts, cwd = process.cwd()) {
+  assertCollectionWithoutPaths(opts.collection, opts.paths);
+  return opts.noConfig ? defaultConfig(cwd) : loadConfig(opts.configPath, cwd);
+}
+function assertCollectionWithoutPaths(collection, paths) {
+  if ((collection ?? []).length > 0 && (paths ?? []).length > 0) {
+    throw new DocevalsError(
+      "--collection selects a configured collection; it cannot be combined with paths."
+    );
+  }
+}
+var CONFIG_FILE = {
+  section: NAMESPACE,
+  // docevals never shipped a file of its own under manni, so, like cite and
+  // a11y, it has no pre-family name to read.
+  legacyNames: [],
+  toError: (message) => new DocevalsError(message)
+};
+
+// src/docevals/core/discover.ts
+import { readFileSync, statSync } from "node:fs";
+import { resolve as resolve2, relative, extname } from "node:path";
+import fg from "fast-glob";
+var FENCES = [
+  { format: "yaml", open: /^---\r?\n/, isClose: (l) => l === "---" || l === "..." },
+  { format: "toml", open: /^\+\+\+\r?\n/, isClose: (l) => l === "+++" },
+  { format: "json", open: /^;;;\r?\n/, isClose: (l) => l === ";;;" }
+];
+function leadingFrontmatterFormat(content) {
+  const body = content.charCodeAt(0) === 65279 ? content.slice(1) : content;
+  return FENCES.find((f) => f.open.test(body))?.format;
+}
+function stripFrontmatterBlock(content) {
+  const body = content.charCodeAt(0) === 65279 ? content.slice(1) : content;
+  const fence = FENCES.find((f) => f.open.test(body));
+  if (!fence) return body;
+  const [openLine = "", ...rest] = body.split(/(?<=\n)/);
+  let offset = openLine.length;
+  for (const line of rest) {
+    offset += line.length;
+    if (fence.isClose(line.replace(/\r?\n$/, ""))) return body.slice(offset);
+  }
+  return body;
+}
+function formatForExtension(ext) {
+  switch (ext.toLowerCase()) {
+    case ".md":
+    case ".markdown":
+      return "markdown";
+    case ".mdx":
+      return "mdx";
+    default:
+      return "markdown";
+  }
+}
+function readPage(absPath, root) {
+  const content = readFileSync(absPath, "utf8");
+  const file = relative(root, absPath).replace(/\\/g, "/");
+  const format = formatForExtension(extname(absPath));
+  try {
+    const frontmatter = extractFrontmatter(content, format);
+    return {
+      file,
+      absPath,
+      content,
+      body: stripFrontmatterBlock(content),
+      frontmatter
+    };
+  } catch (e) {
+    return {
+      file,
+      absPath,
+      content,
+      body: stripFrontmatterBlock(content),
+      frontmatter: {
+        data: {},
+        present: false,
+        format,
+        lineFor: () => void 0
+      },
+      extractError: errorMessage(e)
+    };
+  }
+}
+var FAMILY_EXCLUDE = ["**/node_modules/**", "**/.git/**"];
+function runConfigOptions(paths, options) {
+  return {
+    paths,
+    ...options.config === void 0 ? {} : { configPath: options.config },
+    ...options.noConfig === void 0 ? {} : { noConfig: options.noConfig },
+    ...options.collection === void 0 ? {} : { collection: options.collection }
+  };
+}
+function documentSet(paths, options, verb) {
+  return {
+    paths,
+    verb,
+    ...options.collection === void 0 ? {} : { collection: options.collection },
+    ...options.exclude === void 0 ? {} : { exclude: options.exclude }
+  };
+}
+function toPatterns(input, base) {
+  if (fg.isDynamicPattern(input)) return [input.replace(/\\/g, "/")];
+  let isDirectory;
+  try {
+    isDirectory = statSync(resolve2(base, input)).isDirectory();
+  } catch {
+    return [input.replace(/\\/g, "/")];
+  }
+  const literal = fg.convertPathToPattern(input);
+  return [isDirectory ? `${literal.replace(/\/$/, "")}/**/*` : literal];
+}
+function glob(patterns, base, ignore) {
+  return fg.sync(patterns.flatMap((p) => toPatterns(p, base)), {
+    cwd: base,
+    ignore,
+    absolute: true,
+    dot: false,
+    onlyFiles: true
+  });
+}
+var SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".markdown", ".mdx"]);
+function discoverPages(config, options = {}, root = process.cwd()) {
+  const paths = options.paths ?? [];
+  const wanted = options.collection ?? [];
+  assertCollectionWithoutPaths(wanted, paths);
+  if (wanted.length > 0 && config.configSource === null) {
+    throw new DocevalsError("--collection needs a config file to select from.");
+  }
+  const collections = selectCollections(
+    config.collections,
+    wanted,
+    config.configSource ?? DEFAULT_CONFIG_FILENAME,
+    (message) => new DocevalsError(message)
+  );
+  const exclude = [...FAMILY_EXCLUDE, ...options.exclude ?? []];
+  let entries;
+  let patterns;
+  if (paths.length > 0) {
+    patterns = paths;
+    entries = glob(paths, root, exclude);
+  } else {
+    patterns = collections.flatMap((c) => c.paths);
+    if (patterns.length === 0) {
+      throw new DocevalsError(
+        `No files to ${options.verb ?? "evaluate"}. Pass paths/globs, or declare a collection under \`collections:\` in manni.config.yaml.`
+      );
+    }
+    entries = collections.flatMap(
+      (c) => glob(c.paths, config.configDir, [...exclude, ...c.exclude])
+    );
+  }
+  const files = [
+    ...new Set(entries.filter((p) => SUPPORTED_EXTENSIONS.has(extname(p).toLowerCase())))
+  ];
+  if (files.length === 0) {
+    throw new DocevalsError(
+      `No documentation pages found (patterns: ${patterns.join(", ")})`
+    );
+  }
+  return files.sort().map((p) => readPage(resolve2(p), root));
+}
+
+// src/docevals/core/external.ts
+import { resolve as resolve3 } from "node:path";
+var EVAL_KEYS = ["evals", "eval-suite", "eval-skip"];
+var EXTERNAL_KEYS = [
+  ...EVAL_KEYS,
+  "provenance",
+  "meta-provenance"
+];
+function urlManifestRefusal(source, collection, key) {
+  return `${source}: collection ${collection}: ${key} cannot come from a URL manifest, because docevals writes them.`;
+}
+function twoManifestsRefusal(label, a, b, key) {
+  return `${label} is in collections ${a} and ${b}, and both keep ${key} in a manifest.`;
+}
+var MARKED = new Set(EXTERNAL_KEYS);
+function owns(collection, manifest, key) {
+  return ownsKey(collection, manifest, key, MARKED);
+}
+function evalKeysOf(collection, manifest) {
+  return EVAL_KEYS.filter((k) => owns(collection, manifest, k));
+}
+async function loadExternalReader(config, base, pages) {
+  const collections = [];
+  const source = config.configSource ?? config.configPath;
+  for (const collection of config.collections) {
+    const owning = collection.externalMetadata.filter(
+      (m) => EXTERNAL_KEYS.some((k) => owns(collection, m, k))
+    );
+    if (owning.length === 0) continue;
+    for (const manifest of owning) {
+      if (classifyRef(manifest.file).kind !== "url") continue;
+      const [owned] = evalKeysOf(collection, manifest);
+      if (owned !== void 0) {
+        throw new DocevalsError(
+          urlManifestRefusal(source, collection.name, owned)
+        );
+      }
+    }
+    collections.push({ ...collection, externalMetadata: owning });
+  }
+  if (collections.length === 0) return null;
+  const index = await loadExternalMetadata(collections, {
+    configDir: config.configDir,
+    base,
+    // A `{page}` entry names one manifest per page (proposal 0058), so it
+    // reads a file only for the pages it is handed. Without them it reads
+    // nothing, and every value the corpus keeps beside its pages goes missing
+    // while the ownership that hides the page's own copy stays.
+    ...pages === void 0 ? {} : { pages }
+  });
+  if (index === null) return null;
+  return reader(index, collections, config.configDir, base, marksFor(config, base));
+}
+function marksFor(config, base) {
+  return familyMarks({ configPath: config.configPath, configDir: config.configDir, cwd: base });
+}
+function ownersOf(collections, members, key) {
+  return collections.filter(
+    (c) => members.includes(c.name) && c.externalMetadata.some((m) => owns(c, m, key))
+  ).map((c) => c.name);
+}
+function reader(index, collections, configDir, base, marks) {
+  return {
+    async forPage(page) {
+      if (page.extractError !== void 0) return page;
+      const members = memberOf(collections, configDir, base, page.file);
+      if (members.length === 0) return page;
+      for (const key of EVAL_KEYS) {
+        const [a, b] = ownersOf(collections, members, key);
+        if (a !== void 0 && b !== void 0) {
+          throw new DocevalsError(twoManifestsRefusal(page.file, a, b, key));
+        }
+      }
+      const merged = await mergeWithMarks(
+        page.file,
+        page.frontmatter,
+        index,
+        members,
+        base,
+        { marks: await marks() }
+      );
+      return {
+        ...page,
+        frontmatter: merged.extracted,
+        external: {
+          locate: merged.locate,
+          // Only the keys this tool reads and writes. A manifest owning a
+          // sibling tool's key that the page also carries is `meta validate`'s
+          // finding, and saying it twice would make the same fix look like two.
+          collisions: merged.collisions.filter(
+            (c) => EVAL_KEYS.includes(c.key)
+          ),
+          evalManifests: suppliers(merged.locate, base)
+        }
+      };
+    }
+  };
+}
+function suppliers(locate, base) {
+  const files = /* @__PURE__ */ new Set();
+  for (const key of EVAL_KEYS) {
+    const at = locate(`/${key}`);
+    if (at !== void 0) files.add(resolve3(base, at.file));
+  }
+  return [...files];
+}
+async function withExternalMetadata(pages, config, base) {
+  const merge = await loadExternalReader(
+    config,
+    base,
+    pages.map((p) => p.absPath)
+  );
+  if (merge === null) return pages;
+  return Promise.all(pages.map((p) => merge.forPage(p)));
+}
+
+// src/docevals/core/resolve.ts
+import { Ajv2020 as Ajv20202 } from "ajv/dist/2020.js";
+
+// src/docevals/schema.ts
+var frontmatterSchema = __default;
+var FRONTMATTER_SCHEMA_ID = __default.$id;
+
+// src/docevals/graders/command.ts
+import { dirname as dirname2 } from "node:path";
+function substitute(cmd, file) {
+  return cmd.map((part) => part.replaceAll("{file}", file));
+}
+async function gradeOne(ctx, target) {
+  const { plan, eval: ev } = target;
+  if (!ev.command) {
+    return [
+      {
+        evalName: ev.name,
+        file: plan.page.file,
+        message: "No command to run (script not yet generated \u2014 run `manni docevals generate` or `manni docevals run` with a provider configured)",
+        severity: ev.severity,
+        diagnostic: true
+      }
+    ];
+  }
+  const cwd = ev.source === "page" ? dirname2(plan.page.absPath) : ctx.config.configDir;
+  const result = await ctx.exec(substitute(ev.command, plan.page.absPath), {
+    cwd,
+    timeoutMs: ev.timeoutMs ?? ctx.config.scripts.timeoutMs,
+    env: { MANNI_DOCEVALS_FILE: plan.page.absPath }
+  });
+  if (result.spawnError) {
+    return [
+      {
+        evalName: ev.name,
+        file: plan.page.file,
+        message: `Failed to run command "${ev.command[0] ?? "(empty)"}": ${result.spawnError}`,
+        severity: ev.severity,
+        diagnostic: true
+      }
+    ];
+  }
+  if (result.timedOut) {
+    return [
+      {
+        evalName: ev.name,
+        file: plan.page.file,
+        message: `Command timed out after ${ev.timeoutMs ?? ctx.config.scripts.timeoutMs}ms`,
+        severity: ev.severity,
+        diagnostic: true
+      }
+    ];
+  }
+  if (result.code != null && ev.successExitCodes.includes(result.code)) {
+    return [];
+  }
+  const tail = outputTail(result);
+  return [
+    {
+      evalName: ev.name,
+      file: plan.page.file,
+      // "Exit code N" is the message users and docs already know; only the
+      // null case — killed by a signal, which is what a timeout produces —
+      // needs different words, because "Exit code null" describes nothing.
+      message: (result.code === null ? "Killed before exiting" : `Exit code ${result.code}`) + (tail ? `: ${tail}` : ""),
+      severity: ev.severity
+    }
+  ];
+}
+var commandGrader = {
+  kind: "command",
+  mode: "per-file",
+  async grade(ctx) {
+    const findings = [];
+    for (const target of ctx.targets) {
+      findings.push(...await gradeOne(ctx, target));
+    }
+    return findings;
+  }
+};
+
+// src/docevals/graders/options.ts
+function firstError(...checks) {
+  return checks.find((check) => check !== void 0);
+}
+function knownKeys(options, allowed) {
+  const unknown = Object.keys(options).filter((k) => !allowed.includes(k));
+  if (unknown.length === 0) return void 0;
+  const listed = [...allowed].sort().join(", ");
+  return `unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")} \u2014 this grader accepts: ${listed}`;
+}
+function requiredString(options, key) {
+  const value = options[key];
+  if (typeof value !== "string" || value.length === 0) {
+    return `options.${key} is required`;
+  }
+  return void 0;
+}
+function optionalString(options, key) {
+  const value = options[key];
+  if (value === void 0) return void 0;
+  if (typeof value !== "string" || value.length === 0) {
+    return `options.${key} must be a non-empty string`;
+  }
+  return void 0;
+}
+
+// src/docevals/core/target.ts
+import { readFileSync as readFileSync2 } from "node:fs";
+import { dirname as dirname3, isAbsolute, relative as relative2, resolve as resolve4, sep } from "node:path";
+import { stringify as stringifyYaml } from "yaml";
+var DEFAULT_TARGET = "body";
+function readTarget(target, plan) {
+  const t = target ?? DEFAULT_TARGET;
+  if (t === "body") return { ok: true, text: plan.page.body, label: "body" };
+  if (t === "raw") return { ok: true, text: plan.page.content, label: "raw" };
+  if (t === "frontmatter") {
+    const data = plan.page.frontmatter.data;
+    return {
+      ok: true,
+      text: stringifyYaml(data ?? {}),
+      label: "frontmatter"
+    };
+  }
+  const pageDir = dirname3(plan.page.absPath);
+  if (isAbsolute(t.path)) {
+    return {
+      ok: false,
+      reason: `target file "${t.path}" is an absolute path; name it relative to the page`
+    };
+  }
+  const abs = resolve4(pageDir, t.path);
+  const rel = relative2(pageDir, abs);
+  if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
+    return {
+      ok: false,
+      reason: `target file "${t.path}" resolves outside the page's directory`
+    };
+  }
+  try {
+    return { ok: true, text: readFileSync2(abs, "utf8"), label: `file ${t.path}` };
+  } catch (err) {
+    const code = err instanceof Error ? err.code : void 0;
+    return {
+      ok: false,
+      reason: `target file "${t.path}" could not be read (${code ?? "unknown error"})`
+    };
+  }
+}
+
+// src/docevals/graders/native/regex.ts
+var MATCH_RE = /^(contains|not-contains|count:\d+)$/;
+var FLAGS_RE = /^[dgimsuvy]*$/;
+function lineAt(text, index) {
+  let line = 1;
+  for (let i = 0; i < index && i < text.length; i++) {
+    if (text[i] === "\n") line++;
+  }
+  return line;
+}
+function linesAbove(target, page) {
+  if (target !== void 0 && target !== "body") return 0;
+  if (!page.content.endsWith(page.body)) return 0;
+  return lineAt(page.content, page.content.length - page.body.length) - 1;
+}
+var regexGrader = {
+  kind: "tool:regex",
+  // It calls `readTarget` directly, so it serves the whole vocabulary — the
+  // one deterministic grader that does. Declaring it here is what exempts it
+  // from the engine's body-only guard (ADR 01033).
+  targets: ["body", "raw", "frontmatter", "file"],
+  validateOptions(options) {
+    const match = options.match;
+    return firstError(
+      knownKeys(options, ["pattern", "flags", "match"]),
+      requiredString(options, "pattern"),
+      optionalString(options, "flags"),
+      typeof options.flags === "string" && !FLAGS_RE.test(options.flags) ? "options.flags must be JS RegExp flags (d g i m s u v y)" : void 0,
+      match !== void 0 && (typeof match !== "string" || !MATCH_RE.test(match)) ? "options.match must be contains, not-contains, or count:N" : void 0,
+      // A pattern that will not compile is a typo, not a finding: reporting it
+      // as a failed assertion would blame the page for the eval's own bug.
+      (() => {
+        if (typeof options.pattern !== "string") return void 0;
+        try {
+          new RegExp(options.pattern, options.flags ?? "");
+          return void 0;
+        } catch (err) {
+          return `options.pattern is not a valid regular expression (${errorMessage(err)})`;
+        }
+      })()
+    );
+  },
+  mode: "per-file",
+  grade(ctx) {
+    return Promise.resolve().then(() => {
+      const findings = [];
+      for (const { plan, eval: ev } of ctx.targets) {
+        const opts = ev.options;
+        const pattern = opts.pattern ?? "";
+        const flags = opts.flags ?? "";
+        const match = opts.match ?? "contains";
+        const selected = readTarget(ev.target, plan);
+        if (!selected.ok) {
+          findings.push({
+            evalName: ev.name,
+            file: plan.page.file,
+            ruleId: "regex/unreadable-target",
+            message: selected.reason,
+            severity: ev.severity,
+            line: 1
+          });
+          continue;
+        }
+        const text = selected.text;
+        const offset = linesAbove(ev.target, plan.page);
+        const all = [...text.matchAll(new RegExp(pattern, flags.includes("g") ? flags : `${flags}g`))];
+        const first = all[0];
+        if (match === "contains" && all.length === 0) {
+          findings.push({
+            evalName: ev.name,
+            file: plan.page.file,
+            ruleId: "regex/not-found",
+            message: `Pattern /${pattern}/${flags} not found in ${selected.label}`,
+            severity: ev.severity,
+            line: 1
+          });
+        } else if (match === "not-contains" && first !== void 0) {
+          findings.push({
+            evalName: ev.name,
+            file: plan.page.file,
+            ruleId: "regex/found",
+            message: `Pattern /${pattern}/${flags} found in ${selected.label}, expected absent`,
+            severity: ev.severity,
+            line: offset + lineAt(text, first.index)
+          });
+        } else if (match.startsWith("count:")) {
+          const want = Number(match.slice("count:".length));
+          if (all.length !== want) {
+            findings.push({
+              evalName: ev.name,
+              file: plan.page.file,
+              ruleId: "regex/count",
+              message: `Pattern /${pattern}/${flags} matched ${String(all.length)} time(s) in ${selected.label}, expected ${String(want)}`,
+              severity: ev.severity,
+              line: first === void 0 ? 1 : offset + lineAt(text, first.index)
+            });
+          }
+        }
+      }
+      return findings;
+    });
+  }
+};
+
+// src/docevals/graders/registry.ts
+var GRADERS = new Map(
+  [commandGrader, regexGrader].map((g) => [g.kind, g])
+);
+var UNMAPPED_KINDS = ["ai", "human"];
+var BUILT_IN_ORDER = ["ai", "command", "human", "tool:regex"];
+function graderFor(kind) {
+  return GRADERS.get(kind);
+}
+function listGraderKinds() {
+  return [.../* @__PURE__ */ new Set([...BUILT_IN_ORDER, ...GRADERS.keys()])];
+}
+function isRegisteredGrader(kind) {
+  return UNMAPPED_KINDS.includes(kind) || GRADERS.has(kind);
+}
+function registerGrader(grader) {
+  GRADERS.set(grader.kind, grader);
+}
+
+// src/docevals/core/resolve.ts
+var ajv2 = new Ajv20202({ allErrors: true, allowUnionTypes: true, strict: false });
+var validateFrontmatter = ajv2.compile(frontmatterSchema);
+function evalEntries(raw) {
+  if (raw === void 0) return [];
+  return typeof raw === "string" ? [raw] : raw;
+}
+function fromDef(name, suite, def, source, location) {
+  return {
+    location,
+    name,
+    suite,
+    assertion: def.assertion,
+    type: def.type ?? "regression",
+    grader: def.grader ?? "ai",
+    provider: def.provider,
+    evidence: def.evidence,
+    examples: def.examples,
+    command: def.command,
+    successExitCodes: def.successExitCodes ?? [0],
+    timeoutMs: def.timeoutMs,
+    generatedAssertionHash: def.generatedAssertionHash,
+    options: def.options ?? {},
+    severity: def.severity ?? "error",
+    weight: def.weight ?? 1,
+    target: def.target,
+    model: def.model,
+    runs: def.runs,
+    source,
+    skip: false
+  };
+}
+function shorthandName(index, taken) {
+  const base = `assertion-${index + 1}`;
+  if (!taken.has(base)) return base;
+  for (let n = 2; ; n++) {
+    const candidate = `${base}-${n}`;
+    if (!taken.has(candidate)) return candidate;
+  }
+}
+var RESERVED_KEYS = ["eval-suite", "eval-skip"];
+function describeError(e) {
+  if (e.keyword === "false schema" && e.instancePath.startsWith("/eval-")) {
+    const key = e.instancePath.slice(1);
+    return `unknown key "${key}". The "eval-" prefix is reserved, and the only settings under it are ${RESERVED_KEYS.join(", ")} \u2014 so a typo is an error here rather than a key nothing reads.`;
+  }
+  return e.message ?? "is invalid";
+}
+function locationOf(page, pointer) {
+  const at = declaredAt(page.file, pointer, metadataOf(page));
+  return at.file === page.file ? { line: at.line ?? 1 } : at;
+}
+function metadataOf(page) {
+  return {
+    extracted: page.frontmatter,
+    ...page.external === void 0 ? {} : { locate: page.external.locate }
+  };
+}
+function declaredLocation(page, pointer, key) {
+  if (!(key in page.frontmatter.data)) return { file: page.file };
+  return declaredAt(page.file, pointer, metadataOf(page));
+}
+function resolvePage(page, config) {
+  const problems = [];
+  const empty = {
+    page,
+    skip: false,
+    suite: null,
+    evals: [],
+    problems
+  };
+  if (page.extractError) {
+    problems.push({ message: page.extractError, level: "error", line: 1 });
+    return empty;
+  }
+  for (const c of page.external?.collisions ?? []) {
+    problems.push({
+      message: `"${c.key}" is owned by manifest ${c.file} (collection ${c.collection}); remove it from the document`,
+      level: "error",
+      line: page.frontmatter.lineFor(`/${c.key}`) ?? 1
+    });
+  }
+  const data = page.frontmatter.data;
+  if (!validateFrontmatter(data)) {
+    for (const e of validateFrontmatter.errors ?? []) {
+      problems.push({
+        message: `frontmatter${e.instancePath}: ${describeError(e)}`,
+        level: "error",
+        ...locationOf(page, e.instancePath)
+      });
+    }
+    return empty;
+  }
+  const fm = data;
+  const pageSkip = fm["eval-skip"] ?? false;
+  const declaredSuite = fm["eval-suite"];
+  const suiteName = declaredSuite ?? config.defaults.suite;
+  if (declaredSuite && !(declaredSuite in config.suites)) {
+    problems.push({
+      message: `Unknown suite "${declaredSuite}" (not defined in ${config.configPath})`,
+      level: "error",
+      ...locationOf(page, "/eval-suite")
+    });
+    return { ...empty, skip: pageSkip };
+  }
+  const resolved = /* @__PURE__ */ new Map();
+  if (suiteName) {
+    const suite = config.suites[suiteName];
+    const attached = declaredLocation(page, "/eval-suite", "eval-suite");
+    for (const name of suite?.evals ?? []) {
+      const def = config.evals[name];
+      if (def) resolved.set(name, fromDef(name, suiteName, def, "config", attached));
+    }
+  }
+  const reportSuite = suiteName ?? "default";
+  const claimed = new Set(resolved.keys());
+  for (const entry of evalEntries(fm.evals)) {
+    if (typeof entry === "string") continue;
+    claimed.add("use" in entry ? entry.use : entry.id);
+  }
+  for (const [i, entry] of evalEntries(fm.evals).entries()) {
+    const linePtr = `/evals/${String(i)}`;
+    const entryAt = declaredLocation(page, linePtr, "evals");
+    if (typeof entry === "string") {
+      if (entry in config.evals) {
+        problems.push({
+          message: `String shorthand "${entry}" matches an eval defined in ${config.configPath}, but a string is an assertion now, not a reference. Write "use: ${entry}" to run that eval.`,
+          level: "warning",
+          ...locationOf(page, linePtr)
+        });
+      }
+      const name = shorthandName(i, claimed);
+      claimed.add(name);
+      resolved.set(
+        name,
+        fromDef(name, reportSuite, { assertion: entry }, "page", entryAt)
+      );
+      continue;
+    }
+    if ("use" in entry) {
+      const ref = entry;
+      const def = config.evals[ref.use];
+      if (!def) {
+        problems.push({
+          message: `Unknown eval "${ref.use}" (not defined in ${config.configPath})`,
+          level: "error",
+          ...locationOf(page, linePtr)
+        });
+        continue;
+      }
+      const base = resolved.get(ref.use) ?? fromDef(ref.use, reportSuite, def, "config", entryAt);
+      resolved.set(ref.use, {
+        ...base,
+        location: entryAt,
+        type: ref.type ?? base.type,
+        severity: ref.severity ?? base.severity,
+        weight: ref.weight ?? base.weight,
+        options: { ...base.options, ...ref.options ?? {} },
+        skip: ref.skip ?? base.skip
+      });
+      continue;
+    }
+    const inline = entry;
+    if (resolved.has(inline.id) && resolved.get(inline.id)?.source === "page") {
+      problems.push({
+        message: `Duplicate eval id "${inline.id}" on page \u2014 the later entry replaces the earlier one`,
+        level: "error",
+        ...locationOf(page, linePtr)
+      });
+    }
+    const ev = fromDef(inline.id, reportSuite, normalizeEvalDef(inline), "page", entryAt);
+    ev.skip = inline.skip ?? false;
+    resolved.set(inline.id, ev);
+    if ("severity-map" in inline && isRegisteredGrader(ev.grader)) {
+      warn(
+        `${page.file}: eval "${inline.id}" sets severity-map, which no registered grader reads; it has no effect.`
+      );
+    }
+    if (ev.grader === "ai" && !inline.examples) {
+      problems.push({
+        message: `Eval "${inline.id}": ai-graded evals work best with examples.pass/examples.fail`,
+        level: "warning",
+        ...locationOf(page, linePtr)
+      });
+    }
+  }
+  return {
+    page,
+    skip: pageSkip,
+    suite: suiteName,
+    evals: [...resolved.values()],
+    problems
+  };
+}
+function resolvePages(pages, config) {
+  return pages.map((p) => resolvePage(p, config));
+}
+
+// src/docevals/core/feasibility.ts
+function unregistered(where, name, grader) {
+  return new DocevalsError(
+    `${where}: eval "${name}" names grader "${grader}", which is not registered. Registered graders: ${listGraderKinds().join(", ")}.`
+  );
+}
+function assertRegisteredGraders(plans, config) {
+  const configFile = config.configSource ?? config.configPath;
+  for (const [name, def] of Object.entries(config.evals)) {
+    const grader = def.grader ?? "ai";
+    if (!isRegisteredGrader(grader)) throw unregistered(configFile, name, grader);
+  }
+  for (const plan of plans) {
+    for (const ev of plan.evals) {
+      if (ev.source === "page" && !isRegisteredGrader(ev.grader)) {
+        throw unregistered(plan.page.file, ev.name, ev.grader);
+      }
+    }
+  }
+}
+function checkFeasibility(plans, options) {
+  const problems = [];
+  for (const plan of plans) {
+    if (plan.skip) continue;
+    for (const ev of plan.evals) {
+      if (ev.skip) continue;
+      const isAi = ev.grader === "ai";
+      if (isAi && options.deterministicOnly === true) continue;
+      if (!isAi && ev.grader !== "human" && options.aiOnly === true) continue;
+      const where = `Eval "${ev.name}"`;
+      const grader = graderFor(ev.grader);
+      const invalid = grader?.validateOptions?.(ev.options);
+      if (invalid !== void 0) {
+        problems.push({
+          file: plan.page.file,
+          level: "error",
+          message: `${where}: ${ev.grader} ${invalid}`
+        });
+      }
+      if (ev.grader === "command" && ev.source === "page" && !options.allowFrontmatterCommands && !ev.command && !options.canGenerate) {
+        problems.push({
+          file: plan.page.file,
+          level: "error",
+          message: `${where}: a page-declared command eval with no command, while frontmatter commands are not granted (execution.allow: [frontmatter-commands]) and generation is off \u2014 it can only ever be skipped`
+        });
+      }
+    }
+  }
+  return problems;
+}
+
+// src/docevals/core/baseline.ts
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync as readFileSync3, renameSync, writeFileSync } from "node:fs";
+import { dirname as dirname4, relative as relative3, resolve as resolve5 } from "node:path";
+var DEFAULT_BASELINE_PATH = ".manni-docevals-baseline.json";
+var BASELINE_VERSION = 1;
+var FINGERPRINT_RE = /^[0-9a-f]{16}$/;
+function canonicalFilePath(file, ctx) {
+  if (!ctx) return file;
+  const from = ctx.runBase ?? ctx.base;
+  return relative3(ctx.base, resolve5(from, file)).replace(/\\/g, "/");
+}
+function fingerprint(f) {
+  const parts = [f.evalName, f.ruleId ?? ""];
+  return createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 16);
+}
+function serializeBaseline(baseline) {
+  const entries = /* @__PURE__ */ Object.create(null);
+  for (const file of Object.keys(baseline.entries).sort()) {
+    entries[file] = [...baseline.entries[file] ?? []].sort();
+  }
+  return `${JSON.stringify(
+    { version: baseline.version, generatedWith: baseline.generatedWith, entries },
+    null,
+    2
+  )}
+`;
+}
+function bad(source, detail) {
+  throw new DocevalsError(`Baseline "${source}": ${detail}`);
+}
+function parseBaseline(text, source) {
+  let raw;
+  try {
+    raw = JSON.parse(text.replace(/^/, ""));
+  } catch (err) {
+    bad(source, `invalid JSON: ${errorMessage(err)}`);
+  }
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    bad(source, "top level must be an object.");
+  }
+  const obj = raw;
+  if (obj.version !== BASELINE_VERSION) {
+    bad(
+      source,
+      `unsupported version ${JSON.stringify(obj.version)} (this manni docevals writes version ${BASELINE_VERSION}). Re-record it with \`manni docevals run --write-baseline\`.`
+    );
+  }
+  const rawEntries = obj.entries;
+  if (typeof rawEntries !== "object" || rawEntries === null || Array.isArray(rawEntries)) {
+    bad(source, '"entries" must be an object mapping file paths to fingerprints.');
+  }
+  const entries = /* @__PURE__ */ Object.create(null);
+  for (const [file, value] of Object.entries(rawEntries)) {
+    if (!Array.isArray(value) || value.some((v) => typeof v !== "string")) {
+      bad(source, `entries["${file}"] must be a list of fingerprint strings.`);
+    }
+    const malformed = value.find((v) => !FINGERPRINT_RE.test(v));
+    if (malformed !== void 0) {
+      bad(
+        source,
+        `entries["${file}"] contains ${JSON.stringify(malformed)}, which is not a fingerprint (16 lowercase hex characters).`
+      );
+    }
+    entries[file] = value;
+  }
+  return {
+    version: BASELINE_VERSION,
+    generatedWith: typeof obj.generatedWith === "string" ? obj.generatedWith : "",
+    entries
+  };
+}
+function readBaseline(absPath, source) {
+  let text;
+  try {
+    text = readFileSync3(absPath, "utf8");
+  } catch (err) {
+    if (err.code === "ENOENT") return null;
+    throw new DocevalsError(
+      `Baseline "${source}" could not be read: ${errorMessage(err)}`
+    );
+  }
+  return parseBaseline(text, source);
+}
+function writeBaselineFile(absPath, baseline, source) {
+  try {
+    mkdirSync(dirname4(absPath), { recursive: true });
+    const tmp = `${absPath}.tmp`;
+    writeFileSync(tmp, serializeBaseline(baseline));
+    renameSync(tmp, absPath);
+  } catch (err) {
+    throw new DocevalsError(
+      `Baseline "${source}" could not be written: ${errorMessage(err)}`
+    );
+  }
+}
+function buildBaseline(results, generatedWith, ctx) {
+  const entries = /* @__PURE__ */ Object.create(null);
+  for (const r of results) {
+    const key = canonicalFilePath(r.file, ctx);
+    for (const f of r.findings ?? []) {
+      if (f.diagnostic === true) continue;
+      const list = entries[key] ?? [];
+      const print = fingerprint(f);
+      if (!list.includes(print)) list.push(print);
+      entries[key] = list;
+    }
+  }
+  return { version: BASELINE_VERSION, generatedWith, entries };
+}
+function applyBaseline(results, baseline, ctx) {
+  let suppressed = 0;
+  const seenByFile = /* @__PURE__ */ new Map();
+  const touched = /* @__PURE__ */ new Set();
+  const applied = results.map((r) => {
+    const key = canonicalFilePath(r.file, ctx);
+    if (r.outcome === "pass" || r.outcome === "fail") touched.add(key);
+    const findings = r.findings;
+    if (!findings || findings.length === 0) return r;
+    const known = baseline.entries[key];
+    if (!known) return r;
+    const recordedHere = new Set(known);
+    const seen = seenByFile.get(key) ?? /* @__PURE__ */ new Set();
+    const fresh = [];
+    for (const f of findings) {
+      const print = fingerprint(f);
+      if (recordedHere.has(print)) {
+        seen.add(print);
+        suppressed += 1;
+      } else {
+        fresh.push(f);
+      }
+    }
+    seenByFile.set(key, seen);
+    if (fresh.length === findings.length) return r;
+    const hasError = fresh.some(
+      (f) => f.severity === "error" || f.diagnostic === true
+    );
+    return {
+      ...r,
+      outcome: hasError ? "fail" : "pass",
+      ...fresh.length > 0 ? { findings: fresh } : { findings: void 0 },
+      baselined: findings.length - fresh.length
+    };
+  });
+  let recorded = 0;
+  let stale = 0;
+  for (const key of touched) {
+    const known = baseline.entries[key];
+    if (!known) continue;
+    const recordedHere = new Set(known);
+    recorded += recordedHere.size;
+    stale += recordedHere.size - (seenByFile.get(key)?.size ?? 0);
+  }
+  return { results: applied, recorded, suppressed, stale };
+}
+function diffBaselines(previous, next) {
+  const flatten = (b) => {
+    const out = /* @__PURE__ */ new Set();
+    if (!b) return out;
+    for (const [file, prints] of Object.entries(b.entries)) {
+      for (const p of prints) out.add(`${file}\0${p}`);
+    }
+    return out;
+  };
+  const before = flatten(previous);
+  const after = flatten(next);
+  let added = 0;
+  let removed = 0;
+  for (const k of after) if (!before.has(k)) added += 1;
+  for (const k of before) if (!after.has(k)) removed += 1;
+  return { added, removed };
+}
+function countFingerprints(baseline) {
+  let n = 0;
+  for (const prints of Object.values(baseline.entries)) n += prints.length;
+  return n;
+}
+
+// src/docevals/core/since.ts
+import { resolve as resolve6 } from "node:path";
+var GIT_TIMEOUT_MS = 3e4;
+function changedKey(absPath) {
+  const abs = resolve6(absPath);
+  return process.platform === "win32" ? abs.toLowerCase() : abs;
+}
+function stderrOf(result) {
+  return outputTail(result) || "(no output)";
+}
+async function git(exec, args, root) {
+  const result = await exec(args, { cwd: root, timeoutMs: GIT_TIMEOUT_MS });
+  if (result.spawnError) {
+    throw new DocevalsError(
+      `--since needs git on PATH: could not run git (${result.spawnError}). Install git, or drop --since to evaluate the whole corpus.`
+    );
+  }
+  if (result.timedOut) {
+    throw new DocevalsError(
+      `--since timed out: \`${args.join(" ")}\` took longer than ${GIT_TIMEOUT_MS / 1e3}s in ${root}.`
+    );
+  }
+  return result;
+}
+async function changedFilesSince(ref, root, exec) {
+  if (ref.startsWith("-")) {
+    throw new DocevalsError(
+      `--since "${ref}" is not a ref: a value starting with "-" is read by git as an option, which would silently produce an empty diff.`
+    );
+  }
+  if (ref.trim() === "") {
+    throw new DocevalsError(
+      `--since was given an empty ref. Git would read that as HEAD and scope the run to nothing, which exits 0 having checked nothing. If this came from a CI expression, the variable is unset on this event.`
+    );
+  }
+  const top = await git(exec, ["git", "rev-parse", "--show-toplevel"], root);
+  if (top.code !== 0) {
+    throw new DocevalsError(
+      `--since requires a git repository, and ${root} is not a git repository (or git cannot read it). git said: ${stderrOf(top)}`
+    );
+  }
+  const topLevel = top.stdout.trim();
+  const diff = await git(
+    exec,
+    ["git", "--no-pager", "diff", "--name-only", "-z", `${ref}...HEAD`],
+    root
+  );
+  if (diff.code !== 0) {
+    throw new DocevalsError(
+      `--since ${ref}: git could not resolve "${ref}...HEAD". In CI this is usually a shallow clone \u2014 actions/checkout defaults to fetch-depth: 1, which fetches no other branch, so set fetch-depth: 0 (or fetch ${ref} explicitly) before using --since. git said: ${stderrOf(diff)}`
+    );
+  }
+  const changed = /* @__PURE__ */ new Set();
+  for (const rel of diff.stdout.split("\0")) {
+    if (rel === "") continue;
+    changed.add(changedKey(resolve6(topLevel, rel)));
+  }
+  return changed;
+}
+
+// src/docevals/core/newer-than.ts
+import { statSync as statSync2 } from "node:fs";
+import { relative as relative4, resolve as resolve7 } from "node:path";
+var GIT_TIMEOUT_MS2 = 3e4;
+async function git2(exec, args, cwd) {
+  const result = await exec(args, { cwd, timeoutMs: GIT_TIMEOUT_MS2 });
+  if (result.timedOut) {
+    throw new DocevalsError(
+      `--newer-than timed out: \`${args.join(" ")}\` took longer than ${String(GIT_TIMEOUT_MS2 / 1e3)}s in ${cwd}.`
+    );
+  }
+  return result;
+}
+function keysOf(stdout, topLevel) {
+  const keys = /* @__PURE__ */ new Set();
+  for (const rel of stdout.split("\0")) {
+    if (rel !== "") keys.add(changedKey(resolve7(topLevel, rel)));
+  }
+  return keys;
+}
+async function repoState(root, exec) {
+  const top = await git2(exec, ["git", "rev-parse", "--show-toplevel"], root);
+  if (top.spawnError !== void 0 || top.code !== 0) return null;
+  const topLevel = top.stdout.trim();
+  if (topLevel === "") return null;
+  const tracked = await git2(exec, ["git", "ls-files", "-z", "--full-name"], topLevel);
+  if (tracked.code !== 0) return null;
+  const diff = await git2(
+    exec,
+    ["git", "--no-pager", "diff", "--name-only", "-z", "HEAD"],
+    topLevel
+  );
+  const trackedKeys = keysOf(tracked.stdout, topLevel);
+  return {
+    topLevel,
+    tracked: trackedKeys,
+    modified: diff.code === 0 ? keysOf(diff.stdout, topLevel) : trackedKeys
+  };
+}
+function newestMtime(files) {
+  let newest = Number.NEGATIVE_INFINITY;
+  for (const file of files) {
+    try {
+      newest = Math.max(newest, statSync2(file).mtimeMs);
+    } catch {
+    }
+  }
+  return newest;
+}
+async function lastCommit(files, repo, exec) {
+  const paths = files.map((f) => relative4(repo.topLevel, f).replace(/\\/g, "/"));
+  const log = await git2(
+    exec,
+    ["git", "--no-pager", "log", "-1", "--format=%ct", "--", ...paths],
+    repo.topLevel
+  );
+  const seconds = Number.parseInt(log.stdout.trim(), 10);
+  return log.code === 0 && Number.isFinite(seconds) ? seconds * 1e3 : void 0;
+}
+async function pageAges(pages, root, exec) {
+  const repo = await repoState(root, exec);
+  const ages = [];
+  for (const { page, manifests } of pages) {
+    const files = [page, ...manifests];
+    const committed = repo !== null && files.every((f) => {
+      const key = changedKey(f);
+      return repo.tracked.has(key) && !repo.modified.has(key);
+    });
+    const fromGit = committed ? await lastCommit(files, repo, exec) : void 0;
+    ages.push(fromGit ?? newestMtime(files));
+  }
+  return ages;
+}
+
+// src/docevals/graders/types.ts
+function sortedForKey(value) {
+  if (value === void 0 || value === null) return value;
+  if (Array.isArray(value)) return value.map(sortedForKey);
+  if (typeof value !== "object") return value;
+  const entries = value;
+  const out = {};
+  for (const k of Object.keys(entries).sort()) {
+    out[k] = sortedForKey(entries[k]);
+  }
+  return out;
+}
+function groupTargetsByEval(targets) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const t of targets) {
+    const key = JSON.stringify([
+      t.eval.name,
+      // Key order, not just content: `resolve.ts` rebuilds `options` per page
+      // by spread, so insertion order follows each page's own YAML. Two pages
+      // declaring the same options in a different order used to hash
+      // differently and land in separate groups, so a batch grader made one
+      // invocation per page instead of one per configuration. Sorting makes
+      // the key describe the configuration rather than how it was typed.
+      sortedForKey(t.eval.options),
+      t.eval.timeoutMs ?? null,
+      t.eval.severity
+    ]);
+    const list = groups.get(key) ?? [];
+    list.push(t);
+    groups.set(key, list);
+  }
+  return [...groups.values()];
+}
+
+// src/docevals/judge/cache.ts
+import { buildCacheKey, JsonCache, sha256 } from "@hawkeyexl/inference";
+
+// src/docevals/judge/prompt.ts
+var PROMPT_VERSION = 4;
+var JUDGE_SYSTEM_PROMPT = [
+  "You are a meticulous technical documentation judge. You evaluate whether a",
+  "documentation page satisfies a specific assertion.",
+  "",
+  "Rules:",
+  "- Judge ONLY the stated assertion against the supplied page content. Do not",
+  "  invent requirements the assertion does not state.",
+  '- "match" is "pass" only when the assertion is fully satisfied by the page.',
+  '- Use "partial" when the page partially satisfies the assertion.',
+  '- Use "fail" when the page does not satisfy the assertion.',
+  '- Quote the specific page text you relied on in "observed". If the page',
+  "  lacks relevant content, say so explicitly.",
+  "- Be conservative with confidence: reserve values above 0.9 for verdicts a",
+  "  careful human reviewer would certainly agree with.",
+  "Respond with a JSON object matching the provided schema."
+].join("\n");
+function cleanBody(body) {
+  const out = [];
+  let fence = null;
+  for (const line of body.split("\n")) {
+    const marker = /^(\s*)(```+|~~~+)/.exec(line)?.[2];
+    if (fence) {
+      out.push(line);
+      if (marker?.startsWith(fence)) fence = null;
+      continue;
+    }
+    if (marker !== void 0) {
+      fence = marker;
+      out.push(line);
+      continue;
+    }
+    out.push(
+      line.replace(/\{\/\*[\s\S]*?\*\/\}/g, "").replace(/<!--[\s\S]*?-->/g, "").replace(/^(import|export)\s.*$/, "")
+    );
+  }
+  return out.join("\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+function buildUserContent(ev, body, targetLabel = "body") {
+  const parts = [
+    `# Assertion`,
+    ev.assertion ?? ""
+  ];
+  if (ev.evidence) {
+    parts.push("", `# Where to look`, ev.evidence);
+  }
+  if (ev.examples?.pass || ev.examples?.fail) {
+    parts.push("", "# Anchors");
+    for (const anchor of ev.examples.pass ?? []) {
+      parts.push(`A passing page: ${anchor}`);
+    }
+    for (const anchor of ev.examples.fail ?? []) {
+      parts.push(`A failing page: ${anchor}`);
+    }
+  }
+  const isBody = targetLabel === "body";
+  parts.push(
+    "",
+    isBody ? "# Page content" : `# Page content (${targetLabel})`,
+    "",
+    isBody ? cleanBody(body) : body
+  );
+  return parts.join("\n");
+}
+var EVIDENCE_SYSTEM_PROMPT = [
+  "You are gathering evidence from one part of a documentation page so that a",
+  "judge can later evaluate an assertion about the whole page.",
+  "",
+  "Rules:",
+  "- Do NOT decide whether the assertion holds. You are seeing one part; the",
+  "  part that settles it may be elsewhere.",
+  "- Quote page text verbatim. Quote anything that supports the assertion and",
+  "  anything that contradicts it.",
+  "- Quote only what bears on this assertion. An empty list is the right",
+  "  answer for a part that says nothing about it.",
+  "Respond with a JSON object matching the provided schema."
+].join("\n");
+var EVIDENCE_SCHEMA = {
+  type: "object",
+  required: ["supports", "contradicts"],
+  properties: {
+    supports: { type: "array", items: { type: "string" } },
+    contradicts: { type: "array", items: { type: "string" } }
+  },
+  additionalProperties: false
+};
+function buildEvidenceUser(ev, chunk, part) {
+  return [
+    "# Assertion (do not judge it \u2014 only gather evidence)",
+    ev.assertion ?? "",
+    ...ev.evidence ? ["", "# Where to look", ev.evidence] : [],
+    "",
+    `# Page content (${partLabel(part.index, part.total)})`,
+    "",
+    chunk
+  ].join("\n");
+}
+function renderEvidence(parts, total) {
+  const supports = parts.flatMap((p) => p.supports);
+  const contradicts = parts.flatMap((p) => p.contradicts);
+  const lines = [
+    `The page was too long to read in one pass, so it was read in ${String(total)} parts`,
+    "and the passages bearing on this assertion were collected below. Judge the",
+    "assertion against this evidence. Absence of a quotation is weak evidence of",
+    "absence in the page: parts that bore on nothing contributed nothing.",
+    "",
+    "## Passages supporting the assertion",
+    ...supports.length > 0 ? supports.map((q) => `- ${q}`) : ["(none)"],
+    "",
+    "## Passages contradicting the assertion",
+    ...contradicts.length > 0 ? contradicts.map((q) => `- ${q}`) : ["(none)"]
+  ];
+  return lines.join("\n");
+}
+
+// src/docevals/judge/cache.ts
+var VerdictCache = class _VerdictCache extends JsonCache {
+  /** An ensemble is cacheable only if every run produced a verdict. */
+  static usable(runs) {
+    return runs.every((r) => r.error === void 0 && r.verdict !== void 0);
+  }
+  set(key, value) {
+    if (!_VerdictCache.usable(value)) return;
+    super.set(key, value);
+  }
+  /**
+   * Reading applies the same predicate, which is what makes the property
+   * total rather than merely forward-looking.
+   *
+   * Guarding only the write cannot heal an entry this class did not write —
+   * and those exist: the 65 errored ensembles that prompted ADR 01038 were on
+   * disk before it, and had to be deleted by hand. A committed cache is
+   * exactly the population at risk, because CI replays it with no provider
+   * reachable, so a poisoned entry there is a verdict no future run can
+   * dislodge. Treating it as a miss costs one re-judge and converges;
+   * inheriting `get` unchanged never does.
+   *
+   * The predicate also matches the library's own definition rather than ours:
+   * `computeConsensus` counts a run with no `verdict` as an error vote, so an
+   * entry with neither `verdict` nor `error` — reachable through a truncated
+   * write to these hand-editable JSON files — would otherwise replay as a
+   * permanent human-review with nothing explaining why.
+   */
+  get(key) {
+    const hit = super.get(key);
+    if (hit === void 0) return void 0;
+    return _VerdictCache.usable(hit) ? hit : void 0;
+  }
+};
+function judgeCacheBody(chunkChars, text) {
+  return `chunk${String(chunkChars)}
+${text}`;
+}
+function cacheKey(provider, model, runs, temperature, body, ev) {
+  const evalFingerprint = JSON.stringify({
+    assertion: ev.assertion,
+    evidence: ev.evidence,
+    examples: ev.examples,
+    type: ev.type
+  });
+  return buildCacheKey([
+    provider,
+    model,
+    `v${PROMPT_VERSION}`,
+    `r${runs}`,
+    `t${temperature}`,
+    // Pre-hashed: page bodies are large and key parts should stay short.
+    sha256(body),
+    sha256(evalFingerprint)
+  ]);
+}
+
+// src/docevals/judge/self-preference.ts
+var remedy = (evalId) => `, and it is also the judge. Self-judging favors the author; give "${evalId}" a model: of its own.`;
+function selfPreferenceOf(plan, ev, model) {
+  const data = plan.page.frontmatter.data;
+  const meta = metaProvenanceEntries(data["meta-provenance"]).filter(
+    (e) => e["generated-by"] === model
+  );
+  const target = ev.target ?? DEFAULT_TARGET;
+  const readsBody = target === "body" || target === "raw";
+  const readsFields = target === "frontmatter" || target === "raw";
+  if (readsBody && provenanceEntries(data.provenance).some((e) => e["generated-by"] === model)) {
+    return {
+      axis: "content",
+      model,
+      message: `provenance names ${model} for the body this eval grades${remedy(ev.name)}`
+    };
+  }
+  if (readsFields && meta.some((e) => e.fields.length > 0)) {
+    return {
+      axis: "content",
+      model,
+      message: `meta-provenance names ${model} for the fields this eval grades${remedy(ev.name)}`
+    };
+  }
+  if (meta.some((e) => e.evals.includes(ev.name))) {
+    return {
+      axis: "criterion",
+      model,
+      message: `meta-provenance says ${model} proposed "${ev.name}"${remedy(ev.name)}`
+    };
+  }
+  return void 0;
+}
+
+// src/docevals/core/engine.ts
+import { resolve as resolve8 } from "node:path";
+var GENERATION_UNAVAILABLE = "no command and script generation unavailable (configure a provider or run `manni docevals generate`)";
+function unsupportedTarget(ev) {
+  const target = ev.target;
+  if (target === void 0 || target === "body") return void 0;
+  const grader = graderFor(ev.grader);
+  const named = typeof target === "string" ? target : target.path;
+  if (grader?.targets?.includes(typeof target === "string" ? target : "file")) {
+    return void 0;
+  }
+  return `grader ${ev.grader} cannot read target "${named}" \u2014 it grades the page body. Use the ai grader for this target, or drop the target.`;
+}
+function skippedResult(plan, ev, reason) {
+  return {
+    evalName: ev.name,
+    type: ev.type,
+    grader: ev.grader,
+    file: plan.page.file,
+    outcome: "skipped",
+    skipReason: reason,
+    durationMs: 0
+  };
+}
+function resultKey(file, evalName) {
+  return JSON.stringify([file, evalName]);
+}
+function groupFindings(findings) {
+  const map = /* @__PURE__ */ new Map();
+  for (const f of findings) {
+    const key = resultKey(f.file, f.evalName);
+    const list = map.get(key) ?? [];
+    list.push(f);
+    map.set(key, list);
+  }
+  return map;
+}
+function stampSuites(results, plans) {
+  const suiteOf = /* @__PURE__ */ new Map();
+  const weightOf = /* @__PURE__ */ new Map();
+  const locationOf2 = /* @__PURE__ */ new Map();
+  for (const plan of plans) {
+    for (const ev of plan.evals) {
+      const key = resultKey(plan.page.file, ev.name);
+      suiteOf.set(key, ev.suite);
+      weightOf.set(key, ev.weight);
+      locationOf2.set(key, ev.location);
+    }
+  }
+  for (const r of results) {
+    const key = resultKey(r.file, r.evalName);
+    r.suite = suiteOf.get(key) ?? "default";
+    r.weight = weightOf.get(key) ?? 1;
+    r.location = locationOf2.get(key) ?? { file: r.file };
+  }
+}
+function resolveBaseline(results, config, options, cwd, problems) {
+  const wants = options.writeBaseline !== void 0 && options.writeBaseline !== false;
+  if (options.baseline === false && !wants) return { results };
+  const readFallback = config.baseline ?? (options.baseline === true ? DEFAULT_BASELINE_PATH : null);
+  const applyFrom = options.baseline === false ? null : typeof options.baseline === "string" ? options.baseline : readFallback;
+  const writeTo = !wants ? null : typeof options.writeBaseline === "string" ? options.writeBaseline : config.baseline ?? DEFAULT_BASELINE_PATH;
+  const configured = writeTo ?? applyFrom;
+  if (configured === null) return { results };
+  const ctx = { base: config.configDir, runBase: cwd };
+  const read = (from) => {
+    try {
+      return readBaseline(resolve8(config.configDir, from), from);
+    } catch (e) {
+      if (!wants) throw e;
+      problems.push({
+        file: from,
+        message: `${errorMessage(e)} \u2014 re-recording over it.`,
+        level: "warning"
+      });
+      return null;
+    }
+  };
+  const previous = applyFrom === null ? null : read(applyFrom);
+  const overwriting = writeTo === null ? null : writeTo === applyFrom ? previous : read(writeTo);
+  let out = results;
+  let recorded = 0;
+  let suppressed = 0;
+  let stale = 0;
+  if (previous && options.baseline !== false) {
+    const applied = applyBaseline(results, previous, ctx);
+    out = applied.results;
+    recorded = applied.recorded;
+    suppressed = applied.suppressed;
+    stale = applied.stale;
+  }
+  if (!wants || writeTo === null) {
+    return {
+      results: out,
+      summary: { path: configured, recorded, suppressed, stale }
+    };
+  }
+  const next = buildBaseline(results, options.toolVersion ?? "unknown", ctx);
+  writeBaselineFile(resolve8(config.configDir, writeTo), next, writeTo);
+  const { added, removed } = diffBaselines(overwriting, next);
+  const accepted = applyBaseline(results, next, ctx);
+  if (config.baseline !== writeTo) {
+    problems.push({
+      file: writeTo,
+      message: `Recorded ${writeTo}, but \`baseline:\` in ${config.configPath} is ${config.baseline === null ? "not set" : `"${config.baseline}"`}. An ordinary run will not read this file \u2014 point the key at it, or pass --baseline ${writeTo}.`,
+      level: "warning"
+    });
+  }
+  return {
+    results: accepted.results,
+    summary: {
+      path: configured,
+      recorded,
+      suppressed,
+      stale,
+      written: { added, removed, total: countFingerprints(next) }
+    }
+  };
+}
+function applySelection(plans, config, options, requireRunnable = true) {
+  const names = options.evalNames?.filter((n) => n.trim() !== "") ?? [];
+  const suite = options.suite;
+  if (names.length === 0 && suite === void 0) return false;
+  if (suite !== void 0 && !(suite in config.suites)) {
+    throw new DocevalsError(
+      `--suite "${suite}" is not a defined suite. Defined: ${Object.keys(config.suites).sort().join(", ") || "(none)"}`
+    );
+  }
+  const wanted = new Set(names);
+  let matched = 0;
+  for (const plan of plans) {
+    const runnable = !plan.skip && !plan.problems.some((p) => p.level === "error");
+    plan.evals = plan.evals.filter((ev) => {
+      const byName = wanted.size === 0 || wanted.has(ev.name);
+      const bySuite = suite === void 0 || ev.suite === suite;
+      const keep = byName && bySuite;
+      if (keep && (!requireRunnable || runnable && !ev.skip)) matched += 1;
+      return keep;
+    });
+  }
+  if (matched === 0) {
+    const asked = [
+      names.length > 0 ? `--eval ${names.join(", ")}` : "",
+      suite === void 0 ? "" : `--suite ${suite}`
+    ].filter(Boolean).join(" and ");
+    throw new DocevalsError(
+      `${asked} matched no evals that would run on the pages selected. Nothing would have been checked \u2014 the name may be wrong, or every page carrying it may be skipped. Run \`manni docevals list\` with the same filter to see the resolved plan.`
+    );
+  }
+  return true;
+}
+function scopeManifests(plan) {
+  return plan.page.external?.evalManifests ?? [];
+}
+function sinceSelects(plan, changed) {
+  return [plan.page.absPath, ...scopeManifests(plan)].some(
+    (f) => changed.has(changedKey(f))
+  );
+}
+function applyScope(plans, keep) {
+  let pagesSelected = 0;
+  for (const plan of plans) {
+    if (keep(plan)) {
+      pagesSelected += 1;
+      continue;
+    }
+    plan.evals = [];
+  }
+  return { pagesSelected };
+}
+async function recentPages(plans, windowMs, cwd, exec) {
+  const floor = Date.now() - windowMs;
+  const ages = await pageAges(
+    plans.map((p) => ({ page: p.page.absPath, manifests: scopeManifests(p) })),
+    cwd,
+    exec
+  );
+  return new Set(plans.filter((_, i) => (ages[i] ?? Number.NEGATIVE_INFINITY) >= floor));
+}
+function summarizeSuites(results, config, partial = false) {
+  const bySuite = /* @__PURE__ */ new Map();
+  for (const r of results) {
+    const suite = r.suite ?? "default";
+    const list = bySuite.get(suite) ?? [];
+    list.push(r);
+    bySuite.set(suite, list);
+  }
+  const summaries = [];
+  for (const [suite, rs] of [...bySuite.entries()].sort()) {
+    const passed = rs.filter((r) => r.outcome === "pass").length;
+    const failed = rs.filter((r) => r.outcome === "fail").length;
+    const errored = rs.filter((r) => r.outcome === "error").length;
+    const needsReview = rs.filter((r) => r.outcome === "needs-review").length;
+    const skipped = rs.filter((r) => r.outcome === "skipped").length;
+    const graded = (r) => r.outcome === "pass" || r.outcome === "fail" || r.outcome === "error";
+    const suiteCriteria = config.suites[suite]?.criteria ?? [];
+    const contributions = [];
+    let critPassed = 0;
+    let critFailed = 0;
+    let critSuspended = 0;
+    const absorbed = /* @__PURE__ */ new Set();
+    const byKey = /* @__PURE__ */ new Map();
+    const filesByEval = /* @__PURE__ */ new Map();
+    for (const r of rs) {
+      byKey.set(resultKey(r.file, r.evalName), r);
+      const files = filesByEval.get(r.evalName) ?? /* @__PURE__ */ new Set();
+      files.add(r.file);
+      filesByEval.set(r.evalName, files);
+    }
+    for (const critName of suiteCriteria) {
+      const def = config.criteria[critName];
+      if (!def) continue;
+      const pages = /* @__PURE__ */ new Set();
+      for (const name of def.evals) {
+        for (const file of filesByEval.get(name) ?? []) pages.add(file);
+      }
+      for (const file of pages) {
+        const members = def.evals.map(
+          (name) => byKey.get(resultKey(file, name))
+        );
+        if (members.some((m) => m === void 0 || !graded(m))) {
+          critSuspended++;
+          continue;
+        }
+        const passes = members.map((m) => m?.outcome === "pass");
+        const passed2 = def.combine === "any" ? passes.some(Boolean) : passes.every(Boolean);
+        if (passed2) critPassed++;
+        else critFailed++;
+        contributions.push({ weight: def.weight, passed: passed2 });
+        for (const m of members) {
+          if (m) absorbed.add(resultKey(m.file, m.evalName));
+        }
+      }
+    }
+    for (const r of rs) {
+      if (!graded(r) || absorbed.has(resultKey(r.file, r.evalName))) continue;
+      contributions.push({ weight: r.weight ?? 1, passed: r.outcome === "pass" });
+    }
+    const totalWeight = contributions.reduce((sum, c) => sum + c.weight, 0);
+    const passRate = totalWeight > 0 ? contributions.filter((c) => c.passed).reduce((sum, c) => sum + c.weight, 0) / totalWeight : 1;
+    const targetPassRate = config.suites[suite]?.targetPassRate ?? 1;
+    summaries.push({
+      suite,
+      total: rs.length,
+      passed,
+      failed,
+      needsReview,
+      skipped,
+      errored,
+      passRate,
+      targetPassRate,
+      // A suite target is a claim about a body of checks. A filtered run
+      // measured part of it, so it reports the numbers and withholds the
+      // verdict — erring toward false confidence is what gets a gate removed
+      // rather than fixed (ADR 01018).
+      meetsTarget: partial ? false : passRate >= targetPassRate,
+      ...partial ? { partial: true } : {},
+      ...suiteCriteria.length > 0 ? {
+        criteria: {
+          total: critPassed + critFailed + critSuspended,
+          passed: critPassed,
+          failed: critFailed,
+          suspended: critSuspended
+        }
+      } : {}
+    });
+  }
+  return summaries;
+}
+async function runEvals(options = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  const paths = options.paths ?? [];
+  const config = options.config ?? loadRunConfig(
+    {
+      paths,
+      ...options.configPath === void 0 ? {} : { configPath: options.configPath },
+      ...options.noConfig === void 0 ? {} : { noConfig: options.noConfig },
+      ...options.collection === void 0 ? {} : { collection: options.collection }
+    },
+    cwd
+  );
+  const exec = options.exec ?? realExec;
+  const pages = await withExternalMetadata(
+    discoverPages(
+      config,
+      {
+        paths,
+        verb: "evaluate",
+        ...options.collection === void 0 ? {} : { collection: options.collection },
+        ...options.exclude === void 0 ? {} : { exclude: options.exclude }
+      },
+      cwd
+    ),
+    config,
+    cwd
+  );
+  if (options.writeBaseline !== void 0 && options.writeBaseline !== false && ((options.evalNames?.some((n) => n.trim() !== "") ?? false) || options.suite !== void 0 || options.since !== void 0 || options.newerThan !== void 0)) {
+    throw new DocevalsError(
+      "--write-baseline records the whole corpus, so it cannot be combined with --eval, --suite, --since or --newer-than: the re-record would drop every finding the narrowing excluded. Re-run without it to record."
+    );
+  }
+  const sinceRef = options.since;
+  const newerThan = options.newerThan;
+  const windowMs = newerThan === void 0 ? void 0 : parseDuration(newerThan, (message) => new DocevalsError(message));
+  const changed = sinceRef === void 0 ? null : await changedFilesSince(sinceRef, cwd, exec);
+  const plans = resolvePages(pages, config);
+  assertRegisteredGraders(plans, config);
+  const unskipped = plans.filter((p) => !p.skip);
+  if (unskipped.length > 0 && unskipped.every((p) => p.evals.length === 0) && !plans.some((p) => p.problems.some((pr) => pr.level === "error"))) {
+    throw new DocevalsError(
+      `No evals resolved for any of the ${unskipped.length} page(s) this run would check. Nothing would have been checked, so this run would have reported success without checking anything.
+Point \`defaults.suite\` at a suite in ${config.configPath}, or give the pages an \`eval-suite\` or \`evals\` frontmatter key. Run \`manni docevals list\` to see the resolved plan.`,
+      "nothing-resolved"
+    );
+  }
+  const filtered = applySelection(plans, config, options);
+  let scope = null;
+  const judgeOptions = options.judgeOptions ?? {};
+  const problems = plans.flatMap(
+    (p) => p.problems.map((pr) => ({
+      // A value a manifest supplied is reported against that manifest, at its
+      // line there: the page has nothing at that pointer to fix.
+      file: pr.file ?? p.page.file,
+      message: pr.message,
+      level: pr.level,
+      line: pr.line
+    }))
+  );
+  const results = [];
+  const deterministicTargets = [];
+  const aiTargets = [];
+  const generationTargets = [];
+  const generatedPaths = [];
+  const generatedThisRun = /* @__PURE__ */ new Set();
+  const granted = grantsFor(config.execution.allow, {
+    ...options.allowExecution === void 0 ? {} : { allowExecution: options.allowExecution },
+    ...options.execution === void 0 ? {} : { execution: options.execution }
+  });
+  const allowFrontmatterCommands = granted.has("frontmatter-commands");
+  problems.push(
+    ...checkFeasibility(plans, {
+      allowFrontmatterCommands,
+      canGenerate: options.generate !== false && options.generateScripts != null,
+      ...options.deterministicOnly === true ? { deterministicOnly: true } : {},
+      ...options.aiOnly === true ? { aiOnly: true } : {}
+    })
+  );
+  if (changed !== null || windowMs !== void 0) {
+    const recent = windowMs === void 0 ? null : await recentPages(plans, windowMs, cwd, exec);
+    scope = {
+      pagesTotal: plans.length,
+      ...applyScope(
+        plans,
+        (plan) => (changed === null || sinceSelects(plan, changed)) && (recent === null || recent.has(plan))
+      )
+    };
+  }
+  for (const plan of plans) {
+    if (plan.problems.some((p) => p.level === "error")) continue;
+    for (const ev of plan.evals) {
+      if (plan.skip) {
+        results.push(skippedResult(plan, ev, "page skipped (evals.skip)"));
+        continue;
+      }
+      if (ev.skip) {
+        results.push(skippedResult(plan, ev, "eval skipped"));
+        continue;
+      }
+      if (ev.grader === "human") {
+        results.push({
+          evalName: ev.name,
+          type: ev.type,
+          grader: ev.grader,
+          file: plan.page.file,
+          outcome: "needs-review",
+          durationMs: 0
+        });
+        continue;
+      }
+      if (ev.grader === "ai") {
+        if (options.deterministicOnly) {
+          results.push(skippedResult(plan, ev, "judge skipped (--deterministic-only)"));
+        } else if (!options.judge) {
+          results.push(skippedResult(plan, ev, "judge unavailable (no provider)"));
+        } else {
+          aiTargets.push({ plan, eval: ev });
+        }
+        continue;
+      }
+      if (options.aiOnly) {
+        results.push(skippedResult(plan, ev, "deterministic evals skipped (--ai-only)"));
+        continue;
+      }
+      const pageAuthoredArgv = ev.source === "page" && (ev.grader === "command" || Array.isArray(ev.options.command));
+      if (pageAuthoredArgv && !allowFrontmatterCommands) {
+        results.push(
+          skippedResult(
+            plan,
+            ev,
+            NOT_GRANTED_REASON
+          )
+        );
+        continue;
+      }
+      if (ev.grader === "command" && !ev.command) {
+        generationTargets.push({ plan, eval: ev });
+        continue;
+      }
+      if (ev.grader === "command" && ev.command && ev.generatedAssertionHash && ev.assertion && ev.generatedAssertionHash !== sha256(ev.assertion)) {
+        if (options.generate !== false && options.generateScripts) {
+          generationTargets.push({ plan, eval: ev });
+          continue;
+        }
+        problems.push({
+          file: plan.page.file,
+          message: `Eval "${ev.name}": assertion changed since its script was generated \u2014 run \`manni docevals generate\` to regenerate`,
+          level: "warning"
+        });
+      }
+      const unsupported = unsupportedTarget(ev);
+      if (unsupported !== void 0) {
+        results.push({
+          evalName: ev.name,
+          type: ev.type,
+          grader: ev.grader,
+          file: plan.page.file,
+          outcome: "error",
+          skipReason: unsupported,
+          durationMs: 0
+        });
+        continue;
+      }
+      deterministicTargets.push({ plan, eval: ev });
+    }
+  }
+  if (generationTargets.length > 0) {
+    if (options.generate !== false && options.generateScripts) {
+      const gen = await options.generateScripts(
+        generationTargets,
+        config,
+        judgeOptions
+      );
+      generatedPaths.push(...gen.generatedPaths);
+      const refusedBy = new Map(
+        (gen.refusals ?? []).map((r) => [resultKey(r.file, r.evalName), r.message])
+      );
+      for (const t of generationTargets) {
+        const refused = refusedBy.get(resultKey(t.plan.page.file, t.eval.name));
+        if (!t.eval.command && refused !== void 0) {
+          results.push({
+            evalName: t.eval.name,
+            type: t.eval.type,
+            grader: t.eval.grader,
+            file: t.plan.page.file,
+            outcome: "error",
+            skipReason: refused,
+            durationMs: 0
+          });
+        } else if (!t.eval.command && gen.unavailable !== void 0) {
+          results.push({
+            evalName: t.eval.name,
+            type: t.eval.type,
+            grader: t.eval.grader,
+            file: t.plan.page.file,
+            outcome: "error",
+            skipReason: `${GENERATION_UNAVAILABLE}: ${gen.unavailable}`,
+            durationMs: 0
+          });
+        } else if (t.eval.command) {
+          deterministicTargets.push(t);
+          generatedThisRun.add(resultKey(t.plan.page.file, t.eval.name));
+        } else {
+          results.push({
+            evalName: t.eval.name,
+            type: t.eval.type,
+            grader: t.eval.grader,
+            file: t.plan.page.file,
+            outcome: "error",
+            skipReason: "script generation failed",
+            generated: false,
+            durationMs: 0
+          });
+        }
+      }
+    } else {
+      for (const t of generationTargets) {
+        results.push({
+          evalName: t.eval.name,
+          type: t.eval.type,
+          grader: t.eval.grader,
+          file: t.plan.page.file,
+          outcome: "error",
+          skipReason: GENERATION_UNAVAILABLE,
+          durationMs: 0
+        });
+      }
+    }
+  }
+  const byKind = /* @__PURE__ */ new Map();
+  for (const t of deterministicTargets) {
+    const list = byKind.get(t.eval.grader) ?? [];
+    list.push(t);
+    byKind.set(t.eval.grader, list);
+  }
+  for (const [kind, targets] of byKind) {
+    const grader = graderFor(kind);
+    if (!grader) {
+      for (const t of targets) {
+        results.push({
+          evalName: t.eval.name,
+          type: t.eval.type,
+          grader: t.eval.grader,
+          file: t.plan.page.file,
+          outcome: "error",
+          skipReason: `Unknown grader kind "${kind}"`,
+          durationMs: 0
+        });
+      }
+      continue;
+    }
+    for (const group of groupTargetsByEval(targets)) {
+      const start = Date.now();
+      let findings;
+      try {
+        findings = await grader.grade({ targets: group, config, root: cwd, exec });
+      } catch (e) {
+        const reason = errorMessage(e);
+        for (const target of group) {
+          results.push({
+            evalName: target.eval.name,
+            suite: target.eval.suite,
+            type: target.eval.type,
+            grader: target.eval.grader,
+            file: target.plan.page.file,
+            outcome: "error",
+            // Names the eval, not just the kind. With one catch per kind the
+            // message described whichever eval happened to throw while sitting
+            // on every eval's result, so the name in the result and the name
+            // in the message disagreed.
+            skipReason: `grader ${kind} failed for eval "${target.eval.name}": ${reason}`,
+            durationMs: Date.now() - start
+          });
+        }
+        continue;
+      }
+      const durationMs = Date.now() - start;
+      const grouped = groupFindings(findings);
+      for (const t of group) {
+        const key = resultKey(t.plan.page.file, t.eval.name);
+        const own = grouped.get(key) ?? [];
+        const hasError = own.some(
+          (f) => f.severity === "error" || f.diagnostic === true
+        );
+        results.push({
+          evalName: t.eval.name,
+          type: t.eval.type,
+          grader: t.eval.grader,
+          file: t.plan.page.file,
+          outcome: hasError ? "fail" : "pass",
+          findings: own.length > 0 ? own : void 0,
+          generated: generatedThisRun.has(key) ? true : void 0,
+          durationMs: Math.round(durationMs / group.length)
+        });
+      }
+    }
+  }
+  let effectiveAiTargets = aiTargets;
+  if (config.defaults.failFast) {
+    const failedPages = new Set(
+      results.filter((r) => r.outcome === "fail" || r.outcome === "error").map((r) => r.file)
+    );
+    effectiveAiTargets = [];
+    for (const t of aiTargets) {
+      if (failedPages.has(t.plan.page.file)) {
+        results.push(
+          skippedResult(t.plan, t.eval, "deterministic-precondition-failed")
+        );
+      } else {
+        effectiveAiTargets.push(t);
+      }
+    }
+  }
+  if (effectiveAiTargets.length > 0 && options.judge) {
+    results.push(
+      ...await options.judge(effectiveAiTargets, config, judgeOptions)
+    );
+  }
+  const budgetSkipped = results.filter((r) => isTurnBudgetSkip(r.skipReason));
+  const firstBudgetSkipped = budgetSkipped[0];
+  if (firstBudgetSkipped !== void 0) {
+    problems.push({
+      file: firstBudgetSkipped.file,
+      message: `${budgetSkipped.length} eval(s) were not judged: the turn budget ran out. This run covered less than it was asked to \u2014 raise --max-turns or narrow the run.`,
+      level: "warning"
+    });
+  }
+  if (!results.some((r) => r.outcome !== "skipped")) {
+    problems.push({
+      file: config.configPath,
+      message: `This run graded nothing \u2014 no eval reached a verdict, so it established nothing about the corpus. ` + (results.length > 0 ? `All ${results.length} resolved eval(s) were skipped; check the skip reasons above \u2014 a page-level \`eval-skip\`, a grader-class flag, or a missing provider.` : (
+        // With zero results the cause is not always the same, and naming the
+        // wrong one sends the reader to the wrong file. An error-level
+        // resolution problem makes the plan loop skip that page entirely and
+        // also suppresses the empty-plan throw above, so "skipped or scoped
+        // out" would contradict the errors printed right beside it.
+        problems.some((p) => p.level === "error") ? `Every discovered page was dropped by an error-level resolution problem \u2014 fix those first; they are listed above.` : `Every discovered page was skipped or scoped out of the run.`
+      )),
+      level: "warning"
+    });
+  }
+  const baselineOutcome = resolveBaseline(results, config, options, cwd, problems);
+  const baselinedResults = [...baselineOutcome.results];
+  results.length = 0;
+  results.push(...baselinedResults);
+  stampSuites(results, plans);
+  for (const r of results) {
+    if (!r.selfPreference) continue;
+    const plan = plans.find((p) => p.page.file === r.file);
+    const ev = plan?.evals.find((e) => e.name === r.evalName);
+    const finding = plan && ev ? selfPreferenceOf(plan, ev, r.selfPreference.model) : void 0;
+    if (!finding) continue;
+    problems.push({ file: r.file, level: "warning", message: finding.message });
+  }
+  const scopeLostCoverage = scope !== null && scope.pagesSelected < plans.length;
+  const suites = summarizeSuites(results, config, filtered || scopeLostCoverage);
+  const judged = results.filter((r) => r.consensus != null);
+  const totalTokens = judged.reduce(
+    (n, r) => n + (r.consensus?.runs.reduce(
+      (m, run) => m + (run.usage ? run.usage.inputTokens + run.usage.outputTokens : 0),
+      0
+    ) ?? 0),
+    0
+  );
+  const hasFailure = results.some((r) => r.outcome === "fail" || r.outcome === "error") || suites.some((s) => s.partial !== true && !s.meetsTarget) || problems.some((p) => p.level === "error") || options.failOnReview === true && results.some((r) => r.outcome === "needs-review");
+  return {
+    pages: plans.length,
+    evalResults: results,
+    suites,
+    usage: {
+      totalTokens,
+      cachedEvals: judged.filter(
+        (r) => r.consensus?.runs.every((run) => run.cached) === true
+      ).length,
+      judgedEvals: judged.length
+    },
+    generated: generatedPaths,
+    exitCode: hasFailure ? 1 : 0,
+    problems,
+    ...baselineOutcome.summary ? { baseline: baselineOutcome.summary } : {},
+    ...scope && sinceRef !== void 0 ? { since: { ref: sinceRef, ...scope } } : {},
+    ...scope && newerThan !== void 0 ? { newerThan: { duration: newerThan, ...scope } } : {}
+  };
+}
+
+export {
+  splitBody,
+  looksLikeOverflow,
+  partLabel,
+  DEFAULT_CONFIG_FILENAME,
+  parseConfig,
+  loadConfig,
+  loadRunConfig,
+  leadingFrontmatterFormat,
+  stripFrontmatterBlock,
+  readPage,
+  runConfigOptions,
+  documentSet,
+  discoverPages,
+  EVAL_KEYS,
+  EXTERNAL_KEYS,
+  urlManifestRefusal,
+  loadExternalReader,
+  withExternalMetadata,
+  frontmatterSchema,
+  FRONTMATTER_SCHEMA_ID,
+  readTarget,
+  graderFor,
+  listGraderKinds,
+  registerGrader,
+  resolvePage,
+  resolvePages,
+  assertRegisteredGraders,
+  JUDGE_SYSTEM_PROMPT,
+  buildUserContent,
+  EVIDENCE_SYSTEM_PROMPT,
+  EVIDENCE_SCHEMA,
+  buildEvidenceUser,
+  renderEvidence,
+  sha256,
+  VerdictCache,
+  judgeCacheBody,
+  cacheKey,
+  selfPreferenceOf,
+  applySelection,
+  runEvals
+};
+//# sourceMappingURL=chunk-NWZ2SLYL.js.map