claude-overnight 1.51.3 → 1.54.0

package/README.md

@@ -572,6 +572,20 @@ A fixed-plan `tasks.json` (without `flexiblePlan: true`) bypasses orchestration
 | `1` | Some tasks failed |
 | `2` | All failed or none completed |
 
+## Prompt evolution (server-side)
+
+The `src/prompt-evolution/` engine and `claude-overnight-evolve` CLI power a self-evolution pipeline that optimises prompts (the planner prompt here, MCP-browser's supervisor prompts, or any prompt in a user's repo) via Pareto-frontier mutation with LLM-as-judge and heuristic scoring.
+
+**Multi-hour runs aren't meant for your laptop.** Three ways to run it:
+
+1. **`npx claude-overnight-evolve …`** — quickest. Fine for smoke tests or short runs; needs `ANTHROPIC_API_KEY` in env and keeps running only as long as your shell is open. Output: `~/.claude-overnight/prompt-evolution/<runId>/`.
+2. **Self-hosted Docker** — [`self-host/`](self-host/README.md) ships a tiny runner image + optional HTTP server (enqueue + read-back) you can run on any VPS. Laptop can be off.
+3. **Fornace hosted** — already have a fornace project? `POST /api/projects/:id/prompt-evolution/enqueue` runs the same engine in your project's container. Body: `{ prompt, target, evalModel, generations, population, env?, anthropicApiKey?, anthropicBaseUrl?, anthropicModel? }`. Poll `GET /:runId` for status + inline `report.md`. See the [fornace integration doc § 2.6](https://github.com/Fornace/MCP-Browser/blob/main/docs/integration.md#26-prompt-evolution--apiprojectsidprompt-evolution).
+
+Experiment credentials — any Anthropic-compatible provider (Anthropic direct, OpenRouter, Kimi, DashScope, a local proxy) — are injected via env vars: `ANTHROPIC_BASE_URL`, `ANTHROPIC_API_KEY`, `EVAL_MODEL`, `MUTATE_MODEL`. Self-host reads them from `self-host/.env` (or per-run `env:` in the enqueue body).
+
+Full design: [docs/prompt-evolution-research.md](docs/prompt-evolution-research.md).
+
 ## License
 
 MIT

package/dist/bin/evolve.d.ts

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+/**
+ * `claude-overnight-evolve` — CLI for the prompt-evolution engine.
+ *
+ * Ships with the npm package (compiled to dist/bin/evolve.js). The MCP-browser
+ * platform runs this binary inside a per-project `raw`-mode container via
+ * `docker exec`. See docs/prompt-evolution-research.md.
+ *
+ * Examples:
+ *   claude-overnight-evolve --prompt 10_planning/10-3_plan --eval-model claude-haiku-4-5 --generations 3
+ *   claude-overnight-evolve --target mcp-browser --prompt-kind plan-supervision --eval-model kimi-k2-6
+ *
+ * Requires ANTHROPIC_API_KEY (or ANTHROPIC_AUTH_TOKEN) in env. When `--target
+ * mcp-browser` is used the cwd must be the MCP-browser repo root (so
+ * `platform/supervisor/gemini-client.ts` resolves), or pass the file via
+ * `MCP_BROWSER_GEMINI_CLIENT`.
+ */
+export {};

package/dist/bin/evolve.js

@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+/**
+ * `claude-overnight-evolve` — CLI for the prompt-evolution engine.
+ *
+ * Ships with the npm package (compiled to dist/bin/evolve.js). The MCP-browser
+ * platform runs this binary inside a per-project `raw`-mode container via
+ * `docker exec`. See docs/prompt-evolution-research.md.
+ *
+ * Examples:
+ *   claude-overnight-evolve --prompt 10_planning/10-3_plan --eval-model claude-haiku-4-5 --generations 3
+ *   claude-overnight-evolve --target mcp-browser --prompt-kind plan-supervision --eval-model kimi-k2-6
+ *
+ * Requires ANTHROPIC_API_KEY (or ANTHROPIC_AUTH_TOKEN) in env. When `--target
+ * mcp-browser` is used the cwd must be the MCP-browser repo root (so
+ * `platform/supervisor/gemini-client.ts` resolves), or pass the file via
+ * `MCP_BROWSER_GEMINI_CLIENT`.
+ */
+import { evolvePrompt } from "../prompt-evolution/index.js";
+import { PLAN_CASES } from "../prompt-evolution/fixtures/plan-cases.js";
+import { scenariosToCases, PLANNING_SCENARIOS, REVIEW_SCENARIOS, SUPERVISION_SCENARIOS, STUCK_SCENARIOS, hydrateCases, extractPrompt, } from "../prompt-evolution/adapters/mcp-browser.js";
+function help() {
+    process.stdout.write(`Usage: claude-overnight-evolve [options]
+
+Options:
+  --target <name>         claude-overnight | mcp-browser (default: claude-overnight)
+  --prompt <path>         Prompt file path (claude-overnight)
+  --prompt-kind <kind>    MCP-browser prompt kind: planning | review | evolution |
+                          goal-refinement | plan-supervision | simple-supervision | stuck-analysis
+  --eval-model <model>    Fast model for evaluation (default: claude-haiku-4-5)
+  --mutate-model <model>  Smarter model for mutation (defaults to eval-model)
+  --generations <n>       Number of evolution generations (default: 10)
+  --population <n>        Max population size (default: 8)
+  --plateau <n>           Stop early if no improvement for N generations (default: 3)
+  --cases <suite>         Benchmark suite: plan | mcp-planning | mcp-review |
+                          mcp-supervision | mcp-stuck (default: plan)
+  --base-url <url>        API base URL override
+  --auth-token <token>    Auth token override
+  --run-id <id>           Preset run id (default: auto-generated)
+`);
+    process.exit(0);
+}
+function parseArgs() {
+    const args = process.argv.slice(2);
+    if (args.includes("--help") || args.includes("-h"))
+        help();
+    const opts = {
+        target: "claude-overnight",
+        prompt: "10_planning/10-3_plan",
+        promptKind: "",
+        evalModel: process.env.EVAL_MODEL ?? "claude-haiku-4-5",
+        mutateModel: process.env.MUTATE_MODEL,
+        generations: 10,
+        population: 8,
+        plateau: 3,
+        cases: "",
+        baseUrl: process.env.ANTHROPIC_BASE_URL,
+        authToken: process.env.ANTHROPIC_AUTH_TOKEN ?? process.env.ANTHROPIC_API_KEY,
+    };
+    for (let i = 0; i < args.length; i++) {
+        const v = args[i + 1];
+        switch (args[i]) {
+            case "--target":
+                opts.target = v;
+                i++;
+                break;
+            case "--prompt":
+                opts.prompt = v;
+                i++;
+                break;
+            case "--prompt-kind":
+                opts.promptKind = v;
+                i++;
+                break;
+            case "--eval-model":
+                opts.evalModel = v;
+                i++;
+                break;
+            case "--mutate-model":
+                opts.mutateModel = v;
+                i++;
+                break;
+            case "--generations":
+                opts.generations = parseInt(v, 10);
+                i++;
+                break;
+            case "--population":
+                opts.population = parseInt(v, 10);
+                i++;
+                break;
+            case "--plateau":
+                opts.plateau = parseInt(v, 10);
+                i++;
+                break;
+            case "--cases":
+                opts.cases = v;
+                i++;
+                break;
+            case "--base-url":
+                opts.baseUrl = v;
+                i++;
+                break;
+            case "--auth-token":
+                opts.authToken = v;
+                i++;
+                break;
+            case "--run-id":
+                opts.runId = v;
+                i++;
+                break;
+        }
+    }
+    if (opts.target === "mcp-browser" && !opts.cases) {
+        opts.cases = `mcp-${opts.promptKind || "planning"}`;
+    }
+    if (!opts.cases)
+        opts.cases = "plan";
+    return opts;
+}
+async function main() {
+    const opts = parseArgs();
+    let cases;
+    let promptPath = opts.prompt;
+    let seedText;
+    if (opts.target === "mcp-browser") {
+        const kind = (opts.promptKind || "planning");
+        const scenarioMap = {
+            planning: PLANNING_SCENARIOS,
+            review: REVIEW_SCENARIOS,
+            evolution: [],
+            "goal-refinement": [],
+            "plan-supervision": SUPERVISION_SCENARIOS,
+            "simple-supervision": SUPERVISION_SCENARIOS,
+            "stuck-analysis": STUCK_SCENARIOS,
+        };
+        cases = hydrateCases(scenariosToCases(kind, scenarioMap[kind]));
+        promptPath = `mcp-browser/${kind}`;
+        seedText = extractPrompt(kind);
+    }
+    else {
+        if (opts.cases === "plan")
+            cases = PLAN_CASES;
+        else
+            throw new Error(`Unknown case suite: ${opts.cases}`);
+    }
+    console.log(`Evolution config:`);
+    console.log(`  target:      ${opts.target}`);
+    console.log(`  prompt:      ${promptPath}`);
+    console.log(`  evalModel:   ${opts.evalModel}`);
+    console.log(`  mutateModel: ${opts.mutateModel ?? opts.evalModel}`);
+    console.log(`  generations: ${opts.generations}`);
+    console.log(`  population:  ${opts.population}`);
+    console.log(`  plateau:     ${opts.plateau}`);
+    console.log(`  cases:       ${cases.length} (${opts.cases})`);
+    console.log("");
+    const result = await evolvePrompt({
+        promptPath,
+        cases,
+        evalModel: opts.evalModel,
+        mutateModel: opts.mutateModel,
+        generations: opts.generations,
+        populationCap: opts.population,
+        plateauGenerations: opts.plateau,
+        baseUrl: opts.baseUrl,
+        authToken: opts.authToken,
+        seedText,
+        target: opts.target,
+        runId: opts.runId,
+        onLog: (text) => console.log(text),
+    });
+    console.log("\n=== BEST VARIANT ===");
+    console.log(`id:         ${result.bestVariant.variantId}`);
+    console.log(`generation: ${result.bestVariant.generation}`);
+    console.log(`gmean:      ${(result.bestVariant.gmean * 100).toFixed(1)}%`);
+    console.log(`parse:      ${(result.bestVariant.aggregate.parse * 100).toFixed(1)}%`);
+    console.log(`schema:     ${(result.bestVariant.aggregate.schema * 100).toFixed(1)}%`);
+    console.log(`content:    ${(result.bestVariant.aggregate.content * 100).toFixed(1)}%`);
+    console.log(`cost:       ${(result.bestVariant.aggregate.costEfficiency * 100).toFixed(1)}%`);
+    console.log(`speed:      ${(result.bestVariant.aggregate.speed * 100).toFixed(1)}%`);
+    console.log("\n--- Prompt text ---");
+    console.log(result.bestVariant.text);
+}
+main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});

package/dist/core/_version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "1.51.3";
+export declare const VERSION = "1.53.0";

package/dist/core/_version.js

@@ -1,2 +1,2 @@
 // Auto-generated by build — do not edit manually.
-export const VERSION = "1.51.3";
+export const VERSION = "1.53.0";

package/dist/prompt-evolution/adapters/mcp-browser.js

@@ -12,10 +12,28 @@
  */
 import { readFileSync } from "node:fs";
 import { resolve } from "node:path";
-const GEMINI_CLIENT_PATH = resolve("/Users/francesco/works/repos/MCP-browser/platform/supervisor/gemini-client.ts");
+/**
+ * Resolve the path to `gemini-client.ts` inside whichever MCP-browser checkout
+ * is in scope. Order:
+ *   1. `MCP_BROWSER_GEMINI_CLIENT` env var — explicit override.
+ *   2. `MCP_BROWSER_REPO` env var — repo root, relative file is appended.
+ *   3. cwd — expected shape on fornace.net (inside the project's raw container
+ *      the repo is cloned at `/workspace`, so `process.cwd()` resolves it).
+ *
+ * NEVER hardcode an absolute host path — this runs in a container on the
+ * server and on any contributor's laptop.
+ */
+function resolveGeminiClientPath() {
+    const override = process.env.MCP_BROWSER_GEMINI_CLIENT;
+    if (override)
+        return resolve(override);
+    const repo = process.env.MCP_BROWSER_REPO ?? process.cwd();
+    return resolve(repo, "platform/supervisor/gemini-client.ts");
+}
 /** Extract a const prompt string from gemini-client.ts by name */
 export function extractPrompt(kind) {
-    const source = readFileSync(GEMINI_CLIENT_PATH, "utf-8");
+    const path = resolveGeminiClientPath();
+    const source = readFileSync(path, "utf-8");
     const nameMap = {
         planning: "PLANNING_PROMPT",
         review: "REVIEW_PROMPT",
@@ -29,7 +47,7 @@ export function extractPrompt(kind) {
     const pattern = new RegExp(`const ${constName} = \`([\\s\\S]*?)\`;`);
     const m = source.match(pattern);
     if (!m)
-        throw new Error(`Prompt ${constName} not found in ${GEMINI_CLIENT_PATH}`);
+        throw new Error(`Prompt ${constName} not found in ${path}`);
     return m[1].trim();
 }
 /** Build a synthetic user prompt for a given kind and scenario */

package/dist/prompt-evolution/evaluator.js

@@ -76,12 +76,15 @@ async function runSingle(job, opts) {
     const baseUrl = (opts.baseUrl ?? process.env.ANTHROPIC_BASE_URL ?? "https://api.anthropic.com").replace(/\/$/, "");
     const authToken = opts.authToken ?? process.env.ANTHROPIC_AUTH_TOKEN ?? process.env.ANTHROPIC_API_KEY ?? "";
     const isAnthropic = /^https?:\/\/(api\.)?anthropic\.com/i.test(baseUrl);
+    const isKimi = /kimi\.com/i.test(baseUrl);
     let body;
     let endpoint;
     let headers = {
         "Content-Type": "application/json",
         "Authorization": `Bearer ${authToken}`,
     };
+    if (isKimi)
+        headers["User-Agent"] = "Kilo-Code/1.0";
     if (isAnthropic) {
         // Anthropic native format
         endpoint = `${baseUrl}/v1/messages`;

package/dist/prompt-evolution/index.d.ts

@@ -17,7 +17,7 @@
  *   4. Mutate worst-performing variants using failure traces
  *   5. Repeat
  */
-import type { BenchmarkCase, VariantRow, LearningEntry } from "./types.js";
+import type { BenchmarkCase, EvolutionResult } from "./types.js";
 export interface EvolveOpts {
     /** Prompt file path, e.g. "10_planning/10-3_plan" or "mcp-browser/planning" */
     promptPath: string;
@@ -31,6 +31,8 @@ export interface EvolveOpts {
     generations?: number;
     /** Population size cap */
     populationCap?: number;
+    /** Stop early if no improvement for N generations (default: 3) */
+    plateauGenerations?: number;
     /** Current canon gmean (0 if none) */
     canonGmean?: number;
     /** Optional logging callback */
@@ -41,10 +43,9 @@ export interface EvolveOpts {
     authToken?: string;
     /** Optional seed prompt text (for non-file prompts like MCP-browser) */
     seedText?: string;
-}
-export interface EvolutionResult {
-    bestVariant: VariantRow;
-    allRows: VariantRow[];
-    learningLog: LearningEntry[];
+    /** Target project label for persistence */
+    target?: string;
+    /** Run ID override (auto-generated if omitted) */
+    runId?: string;
 }
 export declare function evolvePrompt(opts: EvolveOpts): Promise<EvolutionResult>;

package/dist/prompt-evolution/index.js

@@ -21,12 +21,31 @@ import { renderPrompt } from "../prompts/load.js";
 import { buildMatrix, renderVariant } from "./evaluator.js";
 import { mutate } from "./mutator.js";
 import { curate, formatMatrix } from "./curator.js";
+import { initRun, appendMatrix, appendLearning, snapshotPrompts, finalizeRun } from "./persistence.js";
+import { generateReport } from "./report.js";
 export async function evolvePrompt(opts) {
     const log = opts.onLog ?? ((t) => process.stdout.write(t + "\n"));
-    const generations = opts.generations ?? 3;
-    const populationCap = opts.populationCap ?? 6;
+    const generations = opts.generations ?? 10;
+    const populationCap = opts.populationCap ?? 8;
+    const plateauGenerations = opts.plateauGenerations ?? 3;
     const mutateModel = opts.mutateModel ?? opts.evalModel;
     const canonGmean = opts.canonGmean ?? 0;
+    const target = opts.target ?? "claude-overnight";
+    const runId = opts.runId ?? `run_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 6)}`;
+    // ── 0. Initialise persistence ──
+    initRun({
+        runId,
+        promptPath: opts.promptPath,
+        target,
+        evalModel: opts.evalModel,
+        mutateModel,
+        generations,
+        populationCap,
+        startedAt: new Date().toISOString(),
+        status: "running",
+        caseNames: opts.cases.map((c) => c.name),
+    });
+    log(`Run directory: ~/.claude-overnight/prompt-evolution/${runId}/`);
     // ── 1. Seed population from existing variants or seed text ──
     let population = opts.seedText
         ? [{ id: "default", promptPath: opts.promptPath, generation: 0, text: opts.seedText }]
@@ -34,6 +53,8 @@ export async function evolvePrompt(opts) {
     log(`Seeded ${population.length} variants from ${opts.promptPath}`);
     const learningLog = [];
     let bestOverall = null;
+    const generationMatrices = [];
+    let generationsWithoutImprovement = 0;
     for (let gen = 0; gen < generations; gen++) {
         log(`\n=== Generation ${gen + 1}/${generations} | Population: ${population.length} ===`);
         // ── 2. Evaluate ──
@@ -47,11 +68,18 @@ export async function evolvePrompt(opts) {
             },
         };
         const matrix = await buildMatrix(population, opts.cases, evalOpts);
+        generationMatrices.push(matrix);
+        snapshotPrompts(runId, matrix);
+        appendMatrix(runId, gen, matrix);
         log(formatMatrix(matrix, opts.cases.map((c) => c.name)));
         // Track best
         const genBest = matrix.reduce((a, b) => (a.gmean > b.gmean ? a : b));
-        if (!bestOverall || genBest.gmean > bestOverall.gmean) {
+        if (!bestOverall || genBest.gmean > bestOverall.gmean + 0.001) {
             bestOverall = genBest;
+            generationsWithoutImprovement = 0;
+        }
+        else {
+            generationsWithoutImprovement++;
         }
         // ── 3. Curate ──
         const curateOpts = {
@@ -71,6 +99,12 @@ export async function evolvePrompt(opts) {
         }));
         // ── 5. Mutate to refill ──
         const targetSize = Math.min(populationCap, keptRows.length + 2);
+        const newEntries = [];
+        // Early stopping check
+        if (generationsWithoutImprovement >= plateauGenerations && gen >= 2) {
+            log(`\n=== Early stop: no improvement for ${generationsWithoutImprovement} generations ===`);
+            break;
+        }
         if (nextPop.length < targetSize && gen < generations - 1) {
             const mutantsNeeded = targetSize - nextPop.length;
             log(`Generating ${mutantsNeeded} mutant(s)...`);
@@ -115,12 +149,14 @@ export async function evolvePrompt(opts) {
                         generation: mutant.generation,
                         text: mutant.text,
                     });
-                    learningLog.push({
+                    const entry = {
                         generation: gen,
                         mutationSummary: mutant.mutationSummary,
                         fitnessDelta: 0, // filled next gen
                         status: "neutral",
-                    });
+                    };
+                    learningLog.push(entry);
+                    newEntries.push(entry);
                     log(`  Mutant ${mutant.variantId} ← ${parent.variantId}: ${mutant.mutationSummary}`);
                 }
                 catch (err) {
@@ -129,6 +165,8 @@ export async function evolvePrompt(opts) {
                 }
             }
         }
+        if (newEntries.length)
+            appendLearning(runId, newEntries);
         population = nextPop;
     }
     // Final evaluation of surviving population
@@ -139,13 +177,36 @@ export async function evolvePrompt(opts) {
         authToken: opts.authToken,
         concurrency: 4,
     });
+    generationMatrices.push(finalMatrix);
+    snapshotPrompts(runId, finalMatrix);
+    appendMatrix(runId, generations, finalMatrix);
     log(formatMatrix(finalMatrix, opts.cases.map((c) => c.name)));
     const best = finalMatrix.reduce((a, b) => (a.gmean > b.gmean ? a : b));
-    if (bestOverall && bestOverall.gmean > best.gmean) {
-        // Return the historical best even if it didn't survive final cull
-        return { bestVariant: bestOverall, allRows: finalMatrix, learningLog };
-    }
-    return { bestVariant: best, allRows: finalMatrix, learningLog };
+    const historicalBest = bestOverall && bestOverall.gmean > best.gmean ? bestOverall : best;
+    const result = {
+        bestVariant: historicalBest,
+        allRows: finalMatrix,
+        learningLog,
+        runId,
+    };
+    // Generate and save report
+    const baselineText = generationMatrices[0]?.find((r) => r.variantId === "default")?.text;
+    const report = generateReport({
+        runId,
+        promptPath: opts.promptPath,
+        target,
+        evalModel: opts.evalModel,
+        generations,
+        baselineText,
+    }, result, generationMatrices);
+    const { writeFileSync } = await import("node:fs");
+    const { runDir } = await import("./persistence.js");
+    const reportPath = `${runDir(runId)}/report.md`;
+    writeFileSync(reportPath, report);
+    result.reportPath = reportPath;
+    finalizeRun(runId, result);
+    log(`\nReport saved: ${reportPath}`);
+    return result;
 }
 // ── Helpers ──
 function seedPopulation(promptPath) {

package/dist/prompt-evolution/llm-judge.d.ts

@@ -0,0 +1,40 @@
+/**
+ * LLM-as-judge scoring for prompt evolution.
+ *
+ * Inspired by Hermes Agent's autoresearch skill and self-evolution repo:
+ *   - Structured rubric (5 criteria × 1-5 scale, normalised to 0-1)
+ *   - The judge sees the prompt, the case, and the model's raw output
+ *   - Returns both a scalar score and human-readable justification
+ *
+ * When to use:
+ *   - Content criteria that are too fuzzy for deterministic regex
+ *     (e.g. "was the plan creative?", "did the response follow the spirit of the prompt?")
+ *   - Final validation gate before promoting a variant to canon
+ *
+ * Cost: ~1 judge call per case per generation (~$0.002-0.01 each).
+ */
+import type { BenchmarkCase, ScoreDimensions } from "./types.js";
+export interface JudgeOpts {
+    model: string;
+    baseUrl?: string;
+    authToken?: string;
+    maxTokens?: number;
+    timeoutMs?: number;
+}
+export interface JudgeResult {
+    /** 0-1 overall quality score */
+    score: number;
+    /** Dimension breakdown matching ScoreDimensions keys */
+    dimensions: Partial<ScoreDimensions>;
+    /** Human-readable rubric justification */
+    justification: string;
+}
+/**
+ * Score a single (case, output) pair with an LLM judge.
+ *
+ * The judge prompt is carefully structured to be reproducible:
+ *   - Exact rubric with 1-5 Likert scale definitions
+ *   - One-shot example in the prompt text
+ *   - Forced JSON output schema
+ */
+export declare function judgeOutput(rawOutput: string, c: BenchmarkCase, opts: JudgeOpts): Promise<JudgeResult>;

package/dist/prompt-evolution/llm-judge.js

@@ -0,0 +1,158 @@
+/**
+ * LLM-as-judge scoring for prompt evolution.
+ *
+ * Inspired by Hermes Agent's autoresearch skill and self-evolution repo:
+ *   - Structured rubric (5 criteria × 1-5 scale, normalised to 0-1)
+ *   - The judge sees the prompt, the case, and the model's raw output
+ *   - Returns both a scalar score and human-readable justification
+ *
+ * When to use:
+ *   - Content criteria that are too fuzzy for deterministic regex
+ *     (e.g. "was the plan creative?", "did the response follow the spirit of the prompt?")
+ *   - Final validation gate before promoting a variant to canon
+ *
+ * Cost: ~1 judge call per case per generation (~$0.002-0.01 each).
+ */
+const DEFAULT_RUBRIC = [
+    { name: "parse", question: "Is the output well-formed and parseable (valid JSON if expected, clear structure otherwise)?" },
+    { name: "schema", question: "Does the output contain all required fields / follow the expected schema?" },
+    { name: "content", question: "Is the content accurate, relevant, and satisfying the user's intent?" },
+    { name: "concision", question: "Is the response concise without omitting necessary detail?" },
+    { name: "instruction", question: "Does the output follow the explicit and implicit instructions in the system prompt?" },
+];
+/**
+ * Score a single (case, output) pair with an LLM judge.
+ *
+ * The judge prompt is carefully structured to be reproducible:
+ *   - Exact rubric with 1-5 Likert scale definitions
+ *   - One-shot example in the prompt text
+ *   - Forced JSON output schema
+ */
+export async function judgeOutput(rawOutput, c, opts) {
+    const baseUrl = (opts.baseUrl ?? process.env.ANTHROPIC_BASE_URL ?? "https://api.anthropic.com").replace(/\/$/, "");
+    const authToken = opts.authToken ?? process.env.ANTHROPIC_AUTH_TOKEN ?? process.env.ANTHROPIC_API_KEY ?? "";
+    const isKimi = /kimi\.com/i.test(baseUrl);
+    const prompt = buildJudgePrompt(rawOutput, c);
+    const body = JSON.stringify({
+        model: opts.model,
+        max_tokens: opts.maxTokens ?? 2048,
+        messages: [{ role: "user", content: prompt }],
+    });
+    const isAnthropic = /^https?:\/\/(api\.)?anthropic\.com/i.test(baseUrl);
+    const endpoint = isAnthropic ? `${baseUrl}/v1/messages` : `${baseUrl}/v1/chat/completions`;
+    const headers = {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${authToken}`,
+    };
+    if (isAnthropic)
+        headers["anthropic-version"] = "2023-06-01";
+    if (isKimi)
+        headers["User-Agent"] = "Kilo-Code/1.0";
+    const res = await fetch(endpoint, {
+        method: "POST",
+        headers,
+        body,
+        signal: AbortSignal.timeout(opts.timeoutMs ?? 60_000),
+    });
+    if (!res.ok) {
+        const text = await res.text().catch(() => "");
+        throw new Error(`Judge HTTP ${res.status}: ${text.slice(0, 200)}`);
+    }
+    let raw = "";
+    if (isAnthropic) {
+        const data = await res.json();
+        raw = data.content?.map((c) => c.text ?? "").join("") ?? "";
+    }
+    else {
+        const data = await res.json();
+        raw = data.choices?.[0]?.message?.content ?? "";
+    }
+    return parseJudgeOutput(raw);
+}
+function buildJudgePrompt(rawOutput, c) {
+    const rubricLines = DEFAULT_RUBRIC
+        .map((r, i) => `${i + 1}. **${r.name}**: ${r.question}`)
+        .join("\n");
+    return `You are an expert prompt-evaluation judge. Your task is to score a language-model output against a benchmark case using a strict rubric.
+
+## Benchmark Case
+
+- **Name**: ${c.name}
+- **Prompt path**: ${c.promptPath}
+- **Expected criteria**:
+${Object.entries(c.criteria).map(([k, v]) => `  - ${k}: ${v}`).join("\n")}
+
+## Model Output
+
+\`\`\`
+${rawOutput.slice(0, 3000)}
+\`\`\`
+
+## Rubric
+
+Score each criterion on a 1-5 Likert scale:
+- 5 = Excellent / exceeds expectations
+- 4 = Good / meets expectations
+- 3 = Acceptable / minor issues
+- 2 = Poor / significant issues
+- 1 = Unacceptable / fails completely
+
+${rubricLines}
+
+## Response Format
+
+Respond ONLY with a JSON object in this exact shape (no markdown fences, no extra text):
+
+{"parse":5,"schema":5,"content":4,"concision":4,"instruction":5,"justification":"Brief justification here."}
+`;
+}
+function parseJudgeOutput(raw) {
+    // Strip fences
+    const cleaned = raw
+        .replace(/^\`\`\`(?:json)?\s*\n?/i, "")
+        .replace(/\n?\`\`\`\s*$/i, "")
+        .trim();
+    let obj;
+    try {
+        obj = JSON.parse(cleaned);
+    }
+    catch {
+        // Try to extract first JSON object
+        const m = cleaned.match(/\{[\s\S]*\}/);
+        if (!m) {
+            return { score: 0.5, dimensions: {}, justification: "Judge returned unparseable JSON. Falling back to neutral." };
+        }
+        try {
+            obj = JSON.parse(m[0]);
+        }
+        catch {
+            return { score: 0.5, dimensions: {}, justification: "Judge returned unparseable JSON. Falling back to neutral." };
+        }
+    }
+    const getNum = (k) => {
+        const v = obj[k];
+        if (typeof v === "number")
+            return Math.max(1, Math.min(5, v));
+        return 3; // neutral default
+    };
+    const parse = (getNum("parse") - 1) / 4;
+    const schema = (getNum("schema") - 1) / 4;
+    const content = (getNum("content") - 1) / 4;
+    const concision = (getNum("concision") - 1) / 4;
+    const instruction = (getNum("instruction") - 1) / 4;
+    // Map concision → costEfficiency, instruction → speed (proxy for "follows instructions quickly")
+    const dimensions = {
+        parse,
+        schema,
+        content,
+        costEfficiency: concision,
+        speed: instruction,
+    };
+    const vals = [parse, schema, content, concision, instruction];
+    const score = vals.reduce((a, b) => a + b, 0) / vals.length;
+    return {
+        score,
+        dimensions,
+        justification: typeof obj.justification === "string" ? obj.justification : "(no justification)",
+    };
+}