@mutagent/evaluator 0.1.0-alpha.2

package/.claude/skills/mutagent-evaluator/scripts/cold-start-sampler.ts

@@ -0,0 +1,267 @@
+/**
+ * scripts/cold-start-sampler.ts — EV-5.3 the balanced COLD-START sampler.
+ * ---------------------------------------------------------------------------
+ * PRE-SUITE, there is NO LLM judge yet: the EV-042 determiner (`determine-
+ * outcome.ts`) needs a criteria suite to run, and the suite doesn't exist yet.
+ * So the ONLY ✓/✗ labels available at cold start are:
+ *   (a) the agent/skill DEFINITION's success ORACLE, and
+ *   (b) deterministic MECHANICAL signals — the four cold-label families
+ *       ERROR · FEEDBACK · SCORE · TERMINAL (a subset of `ext.signals` plus the
+ *       §9.4.2 terminal/`incomplete` marker). No provider, no judge tokens.
+ *
+ * This is DISTINCT from `sample-traces.ts` `balancedSample`, which balances over
+ * the LLM-determiner's `OutcomeVerdict` labels (a WARM, post-judge input). This
+ * module is the COLD front-door: it manufactures a balanced ✓/✗ bootstrap suite
+ * from mechanical labels alone so the first judge pass is trained on BOTH classes.
+ *
+ * The pipeline (in order):
+ *   stratify → SL-5 prevalence floor → worthiness/scenario dedup →
+ *   MIN-BOTH (a guaranteed minimum from BOTH pools — never a success-only suite) →
+ *   oversample the ✗-pool when rare (and flag the suite LOW-CONFIDENCE).
+ *
+ * PURE + deterministic: no `Math.random`, no clock, no network. A given
+ * (traces, opts) always yields the identical suite (reproducible cold suites —
+ * C-PIN-adjacent). Never fabricates a trace: a scarcer class contributes all it
+ * has and no more.
+ */
+
+// ── Mechanical cold labels (NO LLM judge) ───────────────────────────────────
+
+/** The cold outcome — the ONLY two pools a balanced suite is built from. */
+export const ColdOutcome = { Pass: "pass", Fail: "fail" } as const;
+export type ColdOutcomeValue = (typeof ColdOutcome)[keyof typeof ColdOutcome];
+
+/**
+ * The four MECHANICAL cold-label signal families available pre-suite. Mirrors the
+ * subset of UniTF `ext.signals` (`hasError`/`hasApiErrors`/`hasChatFeedback`/
+ * `negativeReaction`/`hasScore`/`minScore`) plus the §9.4.2 terminal marker
+ * (`incomplete`). All optional/additive: an absent field is UNKNOWN, never a
+ * false — the oracle reads only what is present.
+ */
+export interface MechanicalSignals {
+  /** ERROR family — a run/tool error was recorded. */
+  hasError?: boolean;
+  /** ERROR family — provider/API errors were recorded. */
+  hasApiErrors?: boolean;
+  /** FEEDBACK family — a user chat reaction exists on the trace. */
+  hasChatFeedback?: boolean;
+  /** FEEDBACK family — that reaction was negative (frustration/swear hit). */
+  negativeReaction?: boolean;
+  /** SCORE family — a numeric score exists. */
+  hasScore?: boolean;
+  /** SCORE family — the lowest score seen (compared to `scoreThreshold`). */
+  minScore?: number;
+  /** TERMINAL family — the session was cut off / reached no terminal event. */
+  incomplete?: boolean;
+}
+
+/** One candidate trace at cold start: its id + mechanical signals + dedup keys. */
+export interface ColdTrace {
+  id: string;
+  signals: MechanicalSignals;
+  /** `ext.classification.scenario` — the dedup key (one rep per scenario). */
+  scenario?: string;
+  /** `ext.signals.worthinessScore` [0,1] — dedup keeps the worthiest rep. */
+  worthiness?: number;
+}
+
+/**
+ * The DEFINITION success oracle: maps a cold trace → ✓/✗, or `null` when the
+ * mechanical signals are insufficient to decide. A `null` (UNKNOWN) trace is kept
+ * for coverage but NEVER forced into a pool — it can't help balance and must not
+ * fabricate a label. The default is {@link defaultMechanicalOracle}; a subject
+ * whose DEFINITION encodes a richer success rule injects its own here.
+ */
+export type SuccessOracle = (t: ColdTrace) => ColdOutcomeValue | null;
+
+/** Default SCORE-family pass/fail cut. */
+export const DEFAULT_SCORE_THRESHOLD = 0.5;
+
+/**
+ * The default MECHANICAL oracle over the four cold-label families. A trace is ✗
+ * when ANY negative mechanical signal fired — an error, a NEGATIVE user reaction,
+ * a below-threshold score, or a non-terminal/truncated session — and ✓ otherwise.
+ * PURE + total (never returns `null`): a subject that needs an UNKNOWN verdict
+ * injects its own oracle. No LLM, no provider.
+ */
+export function defaultMechanicalOracle(
+  scoreThreshold: number = DEFAULT_SCORE_THRESHOLD,
+): SuccessOracle {
+  return (t) => {
+    const s = t.signals;
+    const errored = s.hasError === true || s.hasApiErrors === true;
+    const negFeedback = s.hasChatFeedback === true && s.negativeReaction === true;
+    const lowScore =
+      s.hasScore === true && typeof s.minScore === "number" && s.minScore < scoreThreshold;
+    const truncated = s.incomplete === true;
+    return errored || negFeedback || lowScore || truncated ? ColdOutcome.Fail : ColdOutcome.Pass;
+  };
+}
+
+// ── stratify ────────────────────────────────────────────────────────────────
+
+/** The ✓/✗ pools plus the UNKNOWN remainder (oracle abstained). Order-preserving. */
+export interface Stratified {
+  pass: ColdTrace[];
+  fail: ColdTrace[];
+  unknown: ColdTrace[];
+}
+
+/**
+ * Partition traces into the ✓/✗ pools via the oracle (UNKNOWN → `unknown`).
+ * PURE + deterministic; preserves input order within each pool.
+ */
+export function stratify(traces: ColdTrace[], oracle: SuccessOracle): Stratified {
+  const pass: ColdTrace[] = [];
+  const fail: ColdTrace[] = [];
+  const unknown: ColdTrace[] = [];
+  for (const t of traces) {
+    const v = oracle(t);
+    if (v === ColdOutcome.Pass) pass.push(t);
+    else if (v === ColdOutcome.Fail) fail.push(t);
+    else unknown.push(t);
+  }
+  return { pass, fail, unknown };
+}
+
+// ── worthiness / scenario dedup ─────────────────────────────────────────────
+
+/**
+ * Keep ONE representative per `scenario` — the highest `worthiness` (ties → the
+ * first seen). Traces with NO scenario have no dedup key, so they are all kept
+ * (they can't be proven duplicates). Deterministic + order-preserving over the
+ * surviving representatives. Never reorders by worthiness — only DROPS lower-
+ * worthiness same-scenario duplicates.
+ */
+export function dedupByScenario(pool: ColdTrace[]): ColdTrace[] {
+  // First pass: pick the winning id per scenario (highest worthiness, first-seen tie).
+  const winner = new Map<string, ColdTrace>();
+  for (const t of pool) {
+    if (t.scenario === undefined) continue;
+    const cur = winner.get(t.scenario);
+    if (cur === undefined || (t.worthiness ?? 0) > (cur.worthiness ?? 0)) {
+      winner.set(t.scenario, t);
+    }
+  }
+  // Second pass: emit in original order, keeping non-scenario traces + each
+  // scenario's winner exactly once.
+  const emittedScenario = new Set<string>();
+  const out: ColdTrace[] = [];
+  for (const t of pool) {
+    if (t.scenario === undefined) {
+      out.push(t);
+      continue;
+    }
+    if (winner.get(t.scenario) === t && !emittedScenario.has(t.scenario)) {
+      emittedScenario.add(t.scenario);
+      out.push(t);
+    }
+  }
+  return out;
+}
+
+// ── coldStartSample — the balanced cold suite ───────────────────────────────
+
+export interface ColdStartOptions {
+  /** Target suite size (the sampler stays at or under this, MIN-BOTH excepted). */
+  size: number;
+  /**
+   * MIN-BOTH — the guaranteed minimum drawn from EACH non-empty pool (default 1).
+   * When honoring both minimums would exceed `size`, MIN-BOTH WINS: a balanced
+   * bootstrap suite is worth more than a hard size cap, and a success-only suite
+   * is never acceptable.
+   */
+  minBoth?: number;
+  /**
+   * SL-5 PREVALENCE FLOOR — the minimum number of ✗-pool traces retained even when
+   * the ✗ class is scarce (default = `minBoth`). The fail budget never drops below
+   * `min(prevalenceFloor, availableFails)`, so a rare-but-present failure signal is
+   * never balanced away.
+   */
+  prevalenceFloor?: number;
+  /** SCORE-family pass/fail cut for the default oracle (default 0.5). */
+  scoreThreshold?: number;
+  /** The DEFINITION success oracle (default `defaultMechanicalOracle(scoreThreshold)`). */
+  oracle?: SuccessOracle;
+  /** Worthiness/scenario dedup (default true). */
+  dedup?: boolean;
+}
+
+export interface ColdStartSuite {
+  /** The selected cold suite (deduped, balanced): ✓ representatives then ✗. */
+  selected: ColdTrace[];
+  passCount: number;
+  failCount: number;
+  /** true when the ✗-pool was too rare to fill its balanced half → all fails taken. */
+  oversampledFailPool: boolean;
+  /**
+   * true when the suite rests on TOO-FEW negatives — the ✗-pool was oversampled or
+   * couldn't fill its balanced half (or is empty). Downstream MUST treat the first
+   * cold judge pass as LOW-CONFIDENCE and re-balance once real verdicts exist.
+   */
+  lowConfidence: boolean;
+  /**
+   * The MIN-BOTH invariant: BOTH non-empty pools are represented in `selected`.
+   * A success-only suite (or a fail-only suite) ⇒ false. When both pools have
+   * members this is ALWAYS true.
+   */
+  minBothHeld: boolean;
+}
+
+/**
+ * Build a balanced cold-start suite from mechanical labels. Runs the full EV-5.3
+ * pipeline: stratify → SL-5 prevalence floor → scenario/worthiness dedup →
+ * MIN-BOTH → oversample-when-rare. PURE + deterministic; never fabricates.
+ */
+export function coldStartSample(traces: ColdTrace[], opts: ColdStartOptions): ColdStartSuite {
+  const size = Math.max(0, Math.floor(opts.size));
+  const minBoth = Math.max(0, Math.floor(opts.minBoth ?? 1));
+  const oracle = opts.oracle ?? defaultMechanicalOracle(opts.scoreThreshold);
+
+  const strat = stratify(traces, oracle);
+  const pass = opts.dedup === false ? strat.pass : dedupByScenario(strat.pass);
+  const fail = opts.dedup === false ? strat.fail : dedupByScenario(strat.fail);
+
+  const perSide = Math.floor(size / 2);
+  // SL-5 prevalence floor: fails we must keep even when scarce (capped at availability).
+  const prevalenceFloor = Math.max(0, Math.floor(opts.prevalenceFloor ?? minBoth));
+  const failFloor = Math.min(Math.max(prevalenceFloor, minBoth), fail.length);
+
+  let takeFail: number;
+  let takePass: number;
+  let oversampledFailPool = false;
+
+  if (fail.length > 0 && fail.length < perSide) {
+    // ✗-pool RARE — oversample: take EVERY available fail, fill the rest with ✓.
+    oversampledFailPool = true;
+    takeFail = fail.length;
+    takePass = Math.min(pass.length, Math.max(0, size - takeFail));
+  } else {
+    // Balanced take, but never below the SL-5 floor for fails.
+    takeFail = Math.min(fail.length, Math.max(perSide, failFloor));
+    takePass = Math.min(pass.length, Math.max(0, size - takeFail));
+    // If ✓ is scarce and budget remains, let ✗ backfill up to `size`.
+    if (takePass + takeFail < size) {
+      takeFail = Math.min(fail.length, size - takePass);
+    }
+  }
+
+  // MIN-BOTH enforcement — never drop a NON-EMPTY pool below min(minBoth, available).
+  // This is the "never a success-only suite" guarantee; it can push the total over
+  // `size` only in the degenerate small-`size` case, which is intentional.
+  if (fail.length > 0) takeFail = Math.max(takeFail, Math.min(minBoth, fail.length));
+  if (pass.length > 0) takePass = Math.max(takePass, Math.min(minBoth, pass.length));
+
+  const selectedPass = pass.slice(0, takePass);
+  const selectedFail = fail.slice(0, takeFail);
+  const selected = [...selectedPass, ...selectedFail];
+
+  const passCount = selectedPass.length;
+  const failCount = selectedFail.length;
+  const minBothHeld = passCount > 0 && failCount > 0;
+  // Low-confidence when negatives are thin: oversampled, or the ✗-pool couldn't
+  // fill its balanced half, or there are no negatives at all.
+  const lowConfidence = oversampledFailPool || fail.length === 0 || failCount < perSide;
+
+  return { selected, passCount, failCount, oversampledFailPool, lowConfidence, minBothHeld };
+}

package/.claude/skills/mutagent-evaluator/scripts/config/load.ts

@@ -0,0 +1,307 @@
+/**
+ * scripts/config/load.ts — the evaluator's config-loader (v0.2.0).
+ * ---------------------------------------------------------------------------
+ * Read + YAML-parse the UNIFIED local `.mutagent/config.yaml` and RESOLVE the
+ * two things the evaluator needs to run:
+ *
+ *   source — bound BY ROLE. The evaluator is a source-CONSUMER, so it binds
+ *            `global.sources`. A SINGLE entry auto-binds; MULTIPLE ⇒
+ *            disambiguation deferred (Fork A) → returned as `multiple` (NOT a
+ *            hard fail — DISCOVER needs a source, code/dataset runs don't).
+ *   judge  — the pinned judge MODEL, resolved from `global.models.judge_model`
+ *            ?? `global.models.default`. `judge_runtime` (renamed from
+ *            `substrate` in v0.2.0) selects HOW judges run.
+ *
+ * These feed the CLI's config→flag precedence (an explicit `--model` /
+ * `--traces` flag WINS; the config fills the gap). This module does NOT apply
+ * that precedence — it exposes the resolved config values; the caller (cli/prep.ts)
+ * merges them under flag precedence.
+ *
+ * LEGACY (Fork B — hard-cut): a pre-v0.2.0 config (wrong `config_version` OR a
+ * legacy top-level key `shared`/`stages`/top-level `diagnostics`/`evaluator`) is
+ * DETECTED, never parsed at runtime — the loader returns
+ * `{ status: "migration-required", … }` so the caller routes to the migration
+ * DIRECTIVE. No old-shape parse ever happens.
+ *
+ * Determinism (coding-rules): `loadEvaluatorConfig` reads an INJECTED path —
+ * never resolved here. Path resolution (the LOCAL `.mutagent/`) happens only in
+ * the thin CLI, so tests stay deterministic against committed fixtures. Pure:
+ * one fs read + a YAML parse; no clock, no random, no network. Never throws.
+ *
+ * SECRETS: config holds only credential REF names (never values); resolution is
+ * a separate use-time step (resolve-credential.ts).
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { parse as parseYaml } from "yaml";
+import {
+  CONFIG_VERSION,
+  LEGACY_CONFIG_VERSIONS,
+  type GlobalSource,
+  type GlobalTarget,
+  type ContextLink,
+  validateGlobal,
+  validateLifecycleEvaluator,
+} from "./schema.ts";
+
+/** The unified config path under a project root: `<root>/.mutagent/config.yaml`. */
+export function configPathFor(projectRoot: string): string {
+  return resolve(projectRoot, ".mutagent", "config.yaml");
+}
+
+/** The v0.1.0 top-level keys that must NOT appear in a v0.2.0 config. */
+const LEGACY_TOP_LEVEL_KEYS = [
+  "shared",
+  "stages",
+  "diagnostics",
+  "evaluator",
+] as const;
+
+/**
+ * Detect a legacy (pre-v0.2.0) config WITHOUT parsing its old shape. Two signals,
+ * either one fires: a wrong/absent `config_version`, or a legacy top-level key
+ * (`shared`/`stages`/a top-level `diagnostics`/`evaluator` skill section). Pure;
+ * tolerant of a non-object input (returns []). Mirrors the orchestrator's
+ * detectLegacyConfig (parity port — no cross-import).
+ */
+export function detectLegacyConfig(obj: unknown): string[] {
+  if (obj === null || typeof obj !== "object") return [];
+  const o = obj as Record<string, unknown>;
+  const markers: string[] = [];
+
+  const version = o.config_version;
+  if (version !== CONFIG_VERSION) {
+    if (
+      typeof version === "string" &&
+      (LEGACY_CONFIG_VERSIONS as readonly string[]).includes(version)
+    ) {
+      markers.push(
+        `config_version: "${version}" (legacy — expected "${CONFIG_VERSION}")`,
+      );
+    } else if (version === undefined) {
+      markers.push(`config_version: absent (expected "${CONFIG_VERSION}")`);
+    }
+    // A non-legacy, non-current version (e.g. a future "0.3.0") is NOT a legacy
+    // marker — structural validation reports it instead.
+  }
+
+  for (const key of LEGACY_TOP_LEVEL_KEYS) {
+    if (key in o) markers.push(`legacy top-level key '${key}'`);
+  }
+  return markers;
+}
+
+/**
+ * How the evaluator's source was bound (or why it couldn't be). The SELECTION
+ * contract (v0.2.1 — parity with orchestrator + diagnostics):
+ *   bound            — exactly one source resolved (single entry · `default:true` · `--source` hit).
+ *   none             — zero sources (DISCOVER needs a pick; code/dataset runs don't need a source).
+ *   needs-selection  — >1 source, none `default:true`, no `--source` ⇒ the caller must PROMPT the
+ *                      operator to pick (NON-fatal — the source EXISTS, only the pick is missing).
+ *   multiple-defaults— >1 source with `default:true` — an AMBIGUOUS config the operator must fix.
+ *   unknown-name     — an explicit `--source <name>` that matched no catalog entry.
+ */
+export type SourceBinding =
+  | { kind: "bound"; source: GlobalSource }
+  | { kind: "none" }
+  | { kind: "needs-selection"; sources: GlobalSource[] }
+  | { kind: "multiple-defaults"; sources: GlobalSource[] }
+  | { kind: "unknown-name"; name: string };
+
+/** How the judge model resolved (model-intent-sacred: never a silent swap). */
+export type JudgeModelResolution =
+  | { kind: "resolved"; model: string; from: "judge_model" | "default" }
+  | { kind: "unresolved" };
+
+export interface EvaluatorConfigResolved {
+  /** the raw config_version literal read from the file. */
+  configVersion: string;
+  /** source bound BY ROLE from `global.sources` (single auto-binds). */
+  source: SourceBinding;
+  /** the pinned judge model from `global.models.{judge_model,default}`. */
+  judgeModel: JudgeModelResolution;
+  /** `lifecycle.evaluator.judge_runtime` (renamed from `substrate`), if set. */
+  judgeRuntime: string | undefined;
+  /** `global.context[]` — PROJECT-WIDE context links (every run loads these). */
+  globalContext: ContextLink[];
+  /** `lifecycle.evaluator.context[]` — the evaluator's stage-specific links. */
+  evaluatorContext: ContextLink[];
+  /** present targets (evaluator is judge-only — carried for completeness only). */
+  targets: GlobalTarget[];
+}
+
+export type LoadEvaluatorConfigResult =
+  | { status: "ok"; exists: true; config: EvaluatorConfigResolved }
+  | { status: "absent"; exists: false }
+  | { status: "migration-required"; exists: true; markers: string[]; error: string }
+  | { status: "invalid"; exists: true; errors: string[] };
+
+/**
+ * Bind a source by role from `global.sources`. Precedence (SELECTION contract —
+ * EXACT parity with orchestrator + diagnostics):
+ *
+ *   1. explicit `selectName` (`--source`) WINS — find by name; miss ⇒ `unknown-name`.
+ *   2. 0 sources                  ⇒ `none`.
+ *   3. 1 source                   ⇒ `bound` (auto-bind).
+ *   4. >1 sources:
+ *        exactly one `default:true` ⇒ `bound` (silent default).
+ *        >1 `default:true`          ⇒ `multiple-defaults` (ambiguous — operator fixes config).
+ *        0  `default:true`          ⇒ `needs-selection` (caller PROMPTS the operator).
+ *
+ * Pure; no clock/random/network. NON-fatal on `needs-selection` — the source
+ * exists, only the disambiguation pick is deferred to the run-start ASK.
+ */
+function bindSourceByRole(
+  sources: GlobalSource[],
+  selectName?: string,
+): SourceBinding {
+  // (1) Explicit override — `--source <name>` WINS over any default / auto-bind.
+  if (selectName !== undefined && selectName !== "") {
+    const hit = sources.find((s) => s.name === selectName);
+    return hit !== undefined
+      ? { kind: "bound", source: hit }
+      : { kind: "unknown-name", name: selectName };
+  }
+  // (2) / (3) — zero or a single entry.
+  if (sources.length === 0) return { kind: "none" };
+  if (sources.length === 1) return { kind: "bound", source: sources[0] as GlobalSource };
+  // (4) — multiple entries; disambiguate by `default:true`.
+  const defaults = sources.filter((s) => s.default === true);
+  if (defaults.length === 1) return { kind: "bound", source: defaults[0] as GlobalSource };
+  if (defaults.length > 1) return { kind: "multiple-defaults", sources };
+  return { kind: "needs-selection", sources };
+}
+
+/**
+ * Resolve the judge model (model-intent-sacred): `judge_model` wins, else
+ * `default`, else unresolved. The CALLER refuses (or falls back to `--model`)
+ * on `unresolved` — this pure resolver never invents a model.
+ */
+function resolveJudgeModel(models: {
+  default?: string;
+  judge_model?: string;
+}): JudgeModelResolution {
+  if (models.judge_model !== undefined && models.judge_model !== "") {
+    return { kind: "resolved", model: models.judge_model, from: "judge_model" };
+  }
+  if (models.default !== undefined && models.default !== "") {
+    return { kind: "resolved", model: models.default, from: "default" };
+  }
+  return { kind: "unresolved" };
+}
+
+/** Options for {@link loadEvaluatorConfig}. */
+export interface LoadEvaluatorConfigOptions {
+  /**
+   * Run-time source override (`--source <name>`). WINS over `default:true` /
+   * auto-bind; a miss ⇒ `{ kind: "unknown-name" }`. Absent ⇒ default precedence.
+   */
+  selectSourceName?: string;
+}
+
+/**
+ * Load + resolve the evaluator's config from an INJECTED path. Guarded (mirrors
+ * the orchestrator loadConfig): a missing file → `absent`; malformed YAML or a
+ * failed structural check → `invalid`; a LEGACY config → `migration-required`
+ * (detected BEFORE structural validation — never parse the old shape). Otherwise
+ * `ok` with the resolved source + judge model + context links. Never throws.
+ *
+ * `opts.selectSourceName` (`--source`) threads the run-time source override into
+ * `bindSourceByRole` (SELECTION contract) — it WINS over `default:true`.
+ */
+export function loadEvaluatorConfig(
+  configPath: string,
+  opts: LoadEvaluatorConfigOptions = {},
+): LoadEvaluatorConfigResult {
+  if (!existsSync(configPath)) {
+    return { status: "absent", exists: false };
+  }
+
+  let raw: string;
+  try {
+    raw = readFileSync(configPath, "utf8");
+  } catch (err) {
+    return { status: "invalid", exists: true, errors: [`cannot read ${configPath}: ${String(err)}`] };
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = parseYaml(raw);
+  } catch (err) {
+    return { status: "invalid", exists: true, errors: [`malformed YAML in ${configPath}: ${String(err)}`] };
+  }
+
+  // Legacy detection FIRST — never parse the old shape at runtime (Fork B).
+  const legacyMarkers = detectLegacyConfig(parsed);
+  if (legacyMarkers.length > 0) {
+    return {
+      status: "migration-required",
+      exists: true,
+      markers: legacyMarkers,
+      error:
+        `legacy config detected in ${configPath} — migration required ` +
+        `(see references/config-migration.md): ${legacyMarkers.join("; ")}`,
+    };
+  }
+
+  const root = (parsed ?? {}) as Record<string, unknown>;
+  const globalBlock = (root.global ?? {}) as Record<string, unknown>;
+  const lifecycle = (root.lifecycle ?? {}) as Record<string, unknown>;
+  const evaluatorSection = (lifecycle.evaluator ?? {}) as Record<string, unknown>;
+
+  // Structural validation of the two blocks the evaluator reads.
+  const gRes = validateGlobal(globalBlock);
+  if (!gRes.ok) return { status: "invalid", exists: true, errors: gRes.errors.map((e) => `global${e}`) };
+  const eRes = validateLifecycleEvaluator(evaluatorSection);
+  if (!eRes.ok) {
+    return { status: "invalid", exists: true, errors: eRes.errors.map((e) => `lifecycle.evaluator${e}`) };
+  }
+
+  const sources = (globalBlock.sources as GlobalSource[] | undefined) ?? [];
+  const targets = (globalBlock.targets as GlobalTarget[] | undefined) ?? [];
+  const models = (globalBlock.models as { default?: string; judge_model?: string } | undefined) ?? {};
+  const globalContext = (globalBlock.context as ContextLink[] | undefined) ?? [];
+  const evaluatorContext = (evaluatorSection.context as ContextLink[] | undefined) ?? [];
+  const judgeRuntime =
+    typeof evaluatorSection.judge_runtime === "string" && evaluatorSection.judge_runtime !== ""
+      ? evaluatorSection.judge_runtime
+      : undefined;
+
+  return {
+    status: "ok",
+    exists: true,
+    config: {
+      configVersion: typeof root.config_version === "string" ? root.config_version : "",
+      source: bindSourceByRole(sources, opts.selectSourceName),
+      judgeModel: resolveJudgeModel(models),
+      judgeRuntime,
+      globalContext,
+      evaluatorContext,
+      targets,
+    },
+  };
+}
+
+// ── CLI — thin wrapper (resolve the LOCAL config; print the resolution) ────────
+// bun scripts/config/load.ts [project-root] [--source <name>]   (root defaults to cwd)
+// Exit 0 = ok / absent; exit 1 = invalid / migration-required.
+
+function runCli(argv: string[]): number {
+  const sourceIdx = argv.indexOf("--source");
+  const selectSourceName =
+    sourceIdx >= 0 && sourceIdx + 1 < argv.length ? argv[sourceIdx + 1] : undefined;
+  // First positional after the script path that isn't the --source value.
+  const positional = argv[2] !== undefined && argv[2] !== "--source" ? argv[2] : undefined;
+  const projectRoot = positional ?? process.cwd();
+  const result = loadEvaluatorConfig(configPathFor(projectRoot), { selectSourceName });
+  process.stdout.write(JSON.stringify(result, null, 2) + "\n");
+  return result.status === "invalid" || result.status === "migration-required" ? 1 : 0;
+}
+
+const isMain =
+  typeof import.meta !== "undefined" &&
+  (import.meta as unknown as { main?: boolean }).main === true;
+if (isMain) {
+  const argv = typeof Bun !== "undefined" ? Bun.argv : process.argv;
+  process.exit(runCli(argv));
+}