dsh-continual-evolve 0.11.0 → 0.12.1

package/lib/autocase.d.ts

@@ -1,38 +0,0 @@
-/** The dedicated container benchmark every auto-case lands in (sanitizeId-safe). */
-export declare const AUTO_CASE_BENCHMARK_ID = "auto_regression";
-/** Which failed-evolution trigger produced the capture. */
-export type AutoCaseSource = "benchmark_rejection" | "gate_no_consent";
-export interface AutoCaseInput {
-    baseDir: string;
-    /**
-     * Resolved rubric key — captures must decrypt under the installation's
-     * real key. Omitting it silently encrypts with the dev fallback, making
-     * the scaffold unreadable to the real key; callers resolve once and pass.
-     */
-    rubricKey?: Buffer | undefined;
-    source: AutoCaseSource;
-    sessionId?: string | undefined;
-    /** One line for what was being attempted (candidate label / proposal summary). */
-    summary: string;
-    /** Machine-captured failure reasons (decision reasons / gate rationale). */
-    reasons: readonly string[];
-    /** Refinement id when the attempt produced one (e.g. a rolled-back candidate). */
-    refinementId?: string | undefined;
-    now?: number | undefined;
-}
-export interface AutoCaseCapture {
-    bid: string;
-    caseId: string;
-}
-/** The draft statement: a structured scaffold, not an evaluable task yet. */
-export declare function renderAutoCaseStatement(input: AutoCaseInput, stamp: string): string;
-/** The draft rubric: explicitly not scoreable until a human rewrites it. */
-export declare function renderAutoCaseRubric(input: AutoCaseInput): string;
-/**
- * Capture one failed evolution attempt as a draft case in the container
- * benchmark, creating the container on first use. Id uniqueness comes from
- * the millisecond stamp; two captures in the same millisecond throw (the
- * caller's containment turns that into a warning, never a lost trigger).
- */
-export declare function captureAutoCase(input: AutoCaseInput): AutoCaseCapture;
-//# sourceMappingURL=autocase.d.ts.map

package/lib/autocase.js

@@ -1,76 +0,0 @@
-/**
- * Auto-case capture (P1, 2026-08-28): failed evolution attempts become DRAFT
- * benchmark cases — regression assets for the benchmark loop instead of
- * discarded failure notes.
- *
- * Captures land in a dedicated container benchmark (auto-regression), NEVER
- * in a user benchmark: a benchmark run evaluates every case in its container
- * without a status filter, and an auto-case's rubric is a mechanical scaffold
- * that must never be scored. Humans promote a capture into a real benchmark
- * by re-authoring statement and rubric there, then casecheck → pilot →
- * freeze. The container benchmark keeps the staging area visible and
- * rollback-free (no engine involvement anywhere).
- *
- * No LLM call: the capture is deterministic. Callers own containment — a
- * failed capture must never break the trigger path (benchmark run, gate).
- */
-import { addCase, createBenchmark, loadBenchmark } from "./benchmark.js";
-/** The dedicated container benchmark every auto-case lands in (sanitizeId-safe). */
-export const AUTO_CASE_BENCHMARK_ID = "auto_regression";
-/** Id-safe compact timestamp (20260828T131934000Z style), unique per capture. */
-function idStamp(now) {
-    return new Date(now)
-        .toISOString()
-        .replace(/[-:]/g, "")
-        .replace(".", "")
-        .replace("Z", "Z");
-}
-/** The draft statement: a structured scaffold, not an evaluable task yet. */
-export function renderAutoCaseStatement(input, stamp) {
-    const lines = [
-        "# Auto-generated regression scaffold",
-        "",
-        `- source: ${input.source}`,
-        `- captured: ${stamp}`,
-        `- session: ${input.sessionId ?? "(unknown)"}`,
-        ...(input.refinementId ? [`- refinement: ${input.refinementId}`] : []),
-        `- attempted: ${input.summary}`,
-        "- failure reasons:",
-        ...(input.reasons.length > 0 ? input.reasons.map((reason) => `  - ${reason}`) : ["  - (none captured)"]),
-        "",
-        "This case was captured mechanically from a failed evolution attempt.",
-        "It lives in the auto-regression container and never enters a user",
-        "benchmark automatically. To promote it: re-author the statement and",
-        "rubric in a real benchmark, then casecheck → pilot → freeze.",
-    ];
-    return `${lines.join("\n")}\n`;
-}
-/** The draft rubric: explicitly not scoreable until a human rewrites it. */
-export function renderAutoCaseRubric(input) {
-    return [
-        "Draft scaffold — the real rubric is authored at calibration time.",
-        `Scoring signal captured from the failure: ${input.reasons.join("; ") || "(none)"}.`,
-        "Scoring against this scaffold is meaningless until a human rewrites it;",
-        "the case stays in the auto-regression container and out of real runs.",
-    ].join("\n");
-}
-/**
- * Capture one failed evolution attempt as a draft case in the container
- * benchmark, creating the container on first use. Id uniqueness comes from
- * the millisecond stamp; two captures in the same millisecond throw (the
- * caller's containment turns that into a warning, never a lost trigger).
- */
-export function captureAutoCase(input) {
-    if (!loadBenchmark(input.baseDir, AUTO_CASE_BENCHMARK_ID)) {
-        createBenchmark(input.baseDir, {
-            title: AUTO_CASE_BENCHMARK_ID,
-            description: "Draft regression scaffolds captured mechanically from failed evolution attempts. Not evaluated; promote captures into a real benchmark by hand (re-author statement/rubric, then casecheck → pilot → freeze).",
-        });
-    }
-    const stamp = idStamp(input.now ?? Date.now());
-    // Stamp first: sanitizeId truncates to 40 chars, and the stamp must
-    // survive the truncation for per-millisecond capture uniqueness.
-    const added = addCase(input.baseDir, AUTO_CASE_BENCHMARK_ID, `auto ${stamp} ${input.source}`, renderAutoCaseStatement(input, stamp), renderAutoCaseRubric(input), input.rubricKey);
-    return { bid: AUTO_CASE_BENCHMARK_ID, caseId: added.id };
-}
-//# sourceMappingURL=autocase.js.map

package/lib/benchmark-command.d.ts

@@ -1,9 +0,0 @@
-/**
- * The `/evolve benchmark` subcommand handler. Extracted from command.ts (P2-2).
- */
-import type { Context } from "@deepseek-ai/cordis";
-import type { CommandInvocation, CommandResult } from "@deepseek-ai/dsh-commands";
-import type { EvolutionEngine } from "./service.js";
-import type { CommandRuntimeOptions } from "./command-util.js";
-export declare function executeBenchmarkCommand(ctx: Context, engine: EvolutionEngine, invocation: CommandInvocation, rest: string[], runtime: CommandRuntimeOptions): Promise<CommandResult>;
-//# sourceMappingURL=benchmark-command.d.ts.map

package/lib/benchmark-command.js

@@ -1,357 +0,0 @@
-import { formatHarnessStateForPrompt } from "./render.js";
-import { stripAngleBrackets } from "./command-util.js";
-import { addCase, caseCheckProblems, createBenchmark, listBenchmarks, listCases, loadBenchmark, loadCaseMeta, loadScoreboard, rollbackRejectedCandidate, saveCaseMeta, saveScoreboard, transitionCaseStatus } from "./benchmark.js";
-import { AUTO_CASE_BENCHMARK_ID, captureAutoCase } from "./autocase.js";
-import { decide, decisionReport, entryFromCells, flagMaterialDrift } from "./score.js";
-import { evaluateState } from "./evaluate.js";
-function success(text) {
-    return { kind: "success", text };
-}
-function error(text) {
-    return { kind: "error", text };
-}
-function parsePositiveInt(value, what) {
-    const n = Number(value);
-    if (!Number.isInteger(n) || n < 1) {
-        throw new Error(`${what} must be a positive integer, got "${value}"`);
-    }
-    return n;
-}
-/** " (N failed)" suffix when an evaluation entry carries failed cells. */
-function failedTextOf(entry) {
-    const failed = entry.cells.filter((cell) => cell.status === "failed").length;
-    return failed > 0 ? ` (${failed} failed)` : "";
-}
-const BENCHMARK_USAGE = `Usage:
-  /evolve benchmark new <title>                          create a benchmark (runs=1)
-  /evolve benchmark add-case <bid> <title> <statement> <rubric>
-  /evolve benchmark list                                 list benchmarks + reference status
-  /evolve benchmark status <bid>                         show scoreboard + decisions
-  /evolve benchmark reset <bid>                          clear the scoreboard (fresh reference)
-  /evolve benchmark run <bid>                            evaluate current state as the reference
-  /evolve benchmark run <bid> candidate <refinementId>   evaluate the post-refinement state and decide
-  /evolve benchmark casecheck <bid>                      quality-gate check all cases
-  /evolve benchmark pilot <bid> <cid>                    single pilot run for calibration
-  /evolve benchmark freeze <bid> <cid>                   freeze a case as formal baseline
-  /evolve benchmark meta <bid> <cid> [field value ...]   set case metadata (capability/distinguisher/shortcuts)`;
-export async function executeBenchmarkCommand(ctx, engine, invocation, rest, runtime) {
-    const sub = rest[0] ?? "";
-    const args = rest.slice(1);
-    const sessionId = invocation.agent.id;
-    const baseDir = engine.baseDir;
-    switch (sub) {
-        case "":
-        case "help":
-            return success(BENCHMARK_USAGE);
-        case "new": {
-            const title = args[0] ?? "";
-            if (!title) {
-                return error(`benchmark new requires a title.\n${BENCHMARK_USAGE}`);
-            }
-            const runs = args[1] !== undefined ? parsePositiveInt(args[1], "runs") : undefined;
-            const definition = createBenchmark(baseDir, { title, ...(runs !== undefined ? { runs } : {}) });
-            return success(`benchmark ${definition.id} created (runs=${definition.runs}, passThreshold=${definition.passThreshold})\nAdd cases with: /evolve benchmark add-case ${definition.id} "<title>" "<statement>" "<rubric>"`);
-        }
-        case "list": {
-            const benchmarks = listBenchmarks(baseDir);
-            if (benchmarks.length === 0) {
-                return success("(no benchmarks yet — use /evolve benchmark new <title>)");
-            }
-            const lines = benchmarks.map((b) => {
-                const cases = listCases(baseDir, b.id);
-                const board = loadScoreboard(baseDir, b.id);
-                const ref = board.reference ? ` ref=${board.reference.overall ?? "?"}` : " no-reference";
-                return `- ${b.id} (${cases.length} cases, runs=${b.runs})${ref}`;
-            });
-            return success(lines.join("\n"));
-        }
-        case "add-case": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const title = args[1] ?? "";
-            const statement = args[2] ?? "";
-            const rubric = args[3] ?? "";
-            if (!bid || !title || !statement || !rubric) {
-                return error(`benchmark add-case needs <bid> <title> <statement> <rubric>.\n${BENCHMARK_USAGE}`);
-            }
-            if (!loadBenchmark(baseDir, bid)) {
-                return error(`benchmark ${bid} not found`);
-            }
-            const caseItem = addCase(baseDir, bid, title, statement, rubric, runtime.rubricKey);
-            return success(`case ${caseItem.id} added to ${bid} (status: draft)`);
-        }
-        case "reset": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            if (!bid) {
-                return error(`benchmark reset needs a <bid>.\n${BENCHMARK_USAGE}`);
-            }
-            if (!loadBenchmark(baseDir, bid)) {
-                return error(`benchmark ${bid} not found`);
-            }
-            saveScoreboard(baseDir, bid, { candidates: [], decisions: [] });
-            return success(`scoreboard reset for ${bid} — run /evolve benchmark run ${bid} to record a fresh reference`);
-        }
-        case "status": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const board = loadScoreboard(baseDir, bid);
-            const lines = [];
-            if (board.reference) {
-                lines.push(`reference "${board.reference.label}": overall=${board.reference.overall ?? "?"} cells=${board.reference.cells.length}${failedTextOf(board.reference)}`);
-            }
-            else {
-                lines.push("(no reference evaluation yet)");
-            }
-            for (const c of board.candidates) {
-                lines.push(`candidate "${c.label}": overall=${c.overall ?? "?"} cells=${c.cells.length}${failedTextOf(c)}${c.refinementId ? ` (${c.refinementId})` : ""}`);
-            }
-            for (const d of board.decisions) {
-                lines.push(`decision: ${d.accepted ? "ACCEPTED" : "rejected"} ${d.candidateLabel} — ${d.reasons.join("; ") || "ok"}`);
-            }
-            return success(lines.join("\n"));
-        }
-        case "run": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const candidateId = args.includes("candidate") ? stripAngleBrackets(args[args.indexOf("candidate") + 1] ?? "") : undefined;
-            const definition = loadBenchmark(baseDir, bid);
-            if (!definition) {
-                return error(`benchmark ${bid} not found`);
-            }
-            const cases = listCases(baseDir, bid);
-            if (cases.length === 0) {
-                return error(`benchmark ${bid} has no cases — use /evolve benchmark add-case`);
-            }
-            const board = loadScoreboard(baseDir, bid);
-            const label = candidateId ? `candidate:${candidateId}` : "reference";
-            if (!candidateId && board.reference) {
-                return error(`reference already evaluated (${board.reference.overall ?? "?"}); evaluate a candidate instead: /evolve benchmark run ${bid} candidate <refinementId>`);
-            }
-            const overview = formatHarnessStateForPrompt(engine.load("local", sessionId));
-            const outcome = await evaluateState(ctx, invocation.agent, {
-                cases,
-                rubricKey: runtime.rubricKey,
-                runs: definition.runs,
-                passThreshold: definition.passThreshold,
-                harnessOverview: overview,
-                label,
-                signal: invocation.signal,
-            });
-            // Gap A3 (version_changed semantics): when a reference exists, re-check
-            // the candidate's cells for material drift — a case whose statement/rubric
-            // hash differs from the reference run is re-marked failed (never counted
-            // as a score, can reject the round via the failure-cell protocol).
-            const cells = candidateId && board.reference ? flagMaterialDrift(board.reference, outcome.cells) : outcome.cells;
-            const entry = entryFromCells(label, cells, candidateId);
-            const failedCells = cells.filter((cell) => cell.status === "failed").length;
-            const lines = [
-                `evaluation "${label}": ${outcome.cells.length} cells${failedCells > 0 ? `, ${failedCells} failed` : ""}, overall=${entry.overall ?? "?"}`,
-                ...Object.entries(entry.aggregate)
-                    // totalDurationMs is metadata, not a case (FAQ #11: the same
-                    // leak class decide/decisionReport already fixed — this
-                    // display path was the remaining one; review audit S4).
-                    .filter(([key]) => key !== "overall" && key !== "failed" && key !== "total" && key !== "totalDurationMs")
-                    .map(([key, value]) => `  ${key}: ${value ?? "?"}`),
-            ];
-            if (failedCells > 0) {
-                for (const cell of cells.filter((cell) => cell.status === "failed")) {
-                    lines.push(`  [failed] ${cell.caseId} r${cell.run}: ${cell.notes}`);
-                }
-            }
-            if (candidateId) {
-                if (!board.reference) {
-                    lines.push("(no reference yet — this run only recorded the candidate)");
-                    board.candidates.push(entry);
-                }
-                else {
-                    const decision = decide(board.reference, entry, {
-                        passThreshold: definition.passThreshold,
-                        regressionTolerance: 0,
-                        maxFailedCells: 0,
-                    });
-                    board.candidates.push(entry);
-                    board.decisions.push({
-                        candidateLabel: label,
-                        refinementId: candidateId,
-                        accepted: decision.accepted,
-                        reasons: decision.reasons,
-                        createdAt: new Date().toISOString(),
-                    });
-                    lines.push(...decisionReport(board.reference, entry, decision));
-                    if (!decision.accepted) {
-                        lines.push(`Consider rolling back the candidate: /evolve rollback <${candidateId}>`);
-                        if (runtime.autoRollbackOnReject) {
-                            const outcome = rollbackRejectedCandidate(engine, sessionId, candidateId);
-                            lines.push(outcome.message);
-                        }
-                        // P1 auto-case capture: the rejected candidate becomes a
-                        // draft regression scaffold. Contained — a failed capture
-                        // must never break the benchmark run's report.
-                        if (runtime.autoCase) {
-                            try {
-                                const captured = captureAutoCase({
-                                    baseDir,
-                                    rubricKey: runtime.rubricKey,
-                                    source: "benchmark_rejection",
-                                    sessionId,
-                                    summary: `${bid} candidate ${candidateId}`,
-                                    reasons: decision.reasons,
-                                    refinementId: candidateId,
-                                });
-                                lines.push(`auto-case captured: ${AUTO_CASE_BENCHMARK_ID}/${captured.caseId} (draft — promote manually if it earns a place)`);
-                            }
-                            catch (cause) {
-                                ctx.logger("continual-evolve").warn(`auto-case capture failed: ${cause instanceof Error ? cause.message : String(cause)}`);
-                            }
-                        }
-                    }
-                }
-            }
-            else {
-                board.reference = entry;
-                lines.push("reference evaluation recorded as the baseline");
-            }
-            saveScoreboard(baseDir, bid, board);
-            return success(lines.join("\n"));
-        }
-        case "casecheck": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            if (!bid) {
-                return error(`benchmark casecheck needs a <bid>.\n${BENCHMARK_USAGE}`);
-            }
-            const definition = loadBenchmark(baseDir, bid);
-            if (!definition) {
-                return error(`benchmark ${bid} not found`);
-            }
-            const cases = listCases(baseDir, bid);
-            if (cases.length === 0) {
-                return error(`benchmark ${bid} has no cases`);
-            }
-            const lines = [];
-            let totalProblems = 0;
-            for (const c of cases) {
-                const problems = caseCheckProblems(baseDir, bid, c.id);
-                const meta = loadCaseMeta(baseDir, bid, c.id);
-                const status = meta?.status ?? "draft";
-                if (problems.length === 0) {
-                    lines.push(`  ${c.id} [${status}] ✓`);
-                }
-                else {
-                    lines.push(`  ${c.id} [${status}] ✗ (${problems.length} problem${problems.length > 1 ? "s" : ""}):`);
-                    for (const p of problems) {
-                        lines.push(`    - ${p}`);
-                    }
-                    totalProblems += problems.length;
-                }
-            }
-            const verdict = totalProblems === 0 ? "✓ all cases pass quality gate" : `✗ ${totalProblems} problem${totalProblems > 1 ? "s" : ""} found`;
-            return success(`casecheck ${bid}: ${verdict}\n${cases.length} cases checked\n${lines.join("\n")}`);
-        }
-        case "pilot": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const cid = stripAngleBrackets(args[1] ?? "");
-            if (!bid || !cid) {
-                return error(`benchmark pilot needs <bid> <cid>.\n${BENCHMARK_USAGE}`);
-            }
-            const definition = loadBenchmark(baseDir, bid);
-            if (!definition) {
-                return error(`benchmark ${bid} not found`);
-            }
-            const cases = listCases(baseDir, bid);
-            const target = cases.find((c) => c.id === cid);
-            if (!target) {
-                return error(`case ${cid} not found in ${bid}`);
-            }
-            // Transition to calibrating if currently draft.
-            const meta = loadCaseMeta(baseDir, bid, cid);
-            if (meta?.status === "frozen") {
-                return error(`case ${cid} is frozen and cannot be calibrated`);
-            }
-            if (meta?.status !== "calibrating") {
-                transitionCaseStatus(baseDir, bid, cid, "calibrating");
-            }
-            // Run a single evaluation on just this case (1 run).
-            const overview = formatHarnessStateForPrompt(engine.load("local", sessionId));
-            const outcome = await evaluateState(ctx, invocation.agent, {
-                cases: [target],
-                rubricKey: runtime.rubricKey,
-                runs: 1,
-                passThreshold: definition.passThreshold,
-                harnessOverview: overview,
-                label: `pilot:${cid}`,
-                signal: invocation.signal,
-            });
-            const cell = outcome.cells[0];
-            const scoreText = cell?.status === "ok" ? `${cell.score} (${cell.passed ? "passed" : "below threshold"})` : `failed: ${cell?.notes ?? "unknown"}`;
-            // Record in calibration history.
-            const updatedMeta = loadCaseMeta(baseDir, bid, cid) ?? { status: "calibrating", capability: "", distinguisher: "", shortcuts: "", calibrationHistory: [] };
-            // Older or externally-written meta.json may lack the history array.
-            updatedMeta.calibrationHistory ??= [];
-            updatedMeta.calibrationHistory.push({
-                runAt: new Date().toISOString(),
-                score: cell?.status === "ok" ? cell.score : 0,
-                passed: cell?.passed ?? false,
-                notes: cell?.notes ?? "",
-                modified: false,
-            });
-            saveCaseMeta(baseDir, bid, cid, updatedMeta);
-            const lines = [
-                `pilot ${cid}: ${scoreText}`,
-                `status: calibrating`,
-                `calibration runs: ${updatedMeta.calibrationHistory.length}`,
-            ];
-            if (cell?.status === "ok" && cell.sessionId) {
-                lines.push(`session: ${cell.sessionId}`);
-            }
-            lines.push(`next: set meta fields, then /evolve benchmark freeze ${bid} ${cid}`);
-            return success(lines.join("\n"));
-        }
-        case "freeze": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const cid = stripAngleBrackets(args[1] ?? "");
-            if (!bid || !cid) {
-                return error(`benchmark freeze needs <bid> <cid>.\n${BENCHMARK_USAGE}`);
-            }
-            const definition = loadBenchmark(baseDir, bid);
-            if (!definition) {
-                return error(`benchmark ${bid} not found`);
-            }
-            const meta = loadCaseMeta(baseDir, bid, cid);
-            if (!meta) {
-                return error(`case ${cid} not found in ${bid}`);
-            }
-            if (meta.status === "frozen") {
-                return error(`case ${cid} is already frozen`);
-            }
-            // Require quality check to pass before freezing.
-            const problems = caseCheckProblems(baseDir, bid, cid);
-            if (problems.length > 0) {
-                return error(`case ${cid} has ${problems.length} quality problem${problems.length > 1 ? "s" : ""}:\n${problems.map((p) => `  - ${p}`).join("\n")}\nFix these before freezing.`);
-            }
-            transitionCaseStatus(baseDir, bid, cid, "frozen");
-            return success(`case ${cid} frozen as formal baseline (immutable)`);
-        }
-        case "meta": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const cid = stripAngleBrackets(args[1] ?? "");
-            const field = args[2] ?? "";
-            const value = args.slice(3).join(" ");
-            if (!bid || !cid || !field || !value) {
-                return error(`benchmark meta needs <bid> <cid> <field> <value> (fields: capability, distinguisher, shortcuts).\n${BENCHMARK_USAGE}`);
-            }
-            const meta = loadCaseMeta(baseDir, bid, cid);
-            if (!meta) {
-                return error(`case ${cid} not found in ${bid}`);
-            }
-            if (meta.status === "frozen") {
-                return error(`case ${cid} is frozen and cannot be modified`);
-            }
-            if (field !== "capability" && field !== "distinguisher" && field !== "shortcuts") {
-                return error(`unknown meta field "${field}" — valid fields: capability, distinguisher, shortcuts`);
-            }
-            meta[field] = value;
-            saveCaseMeta(baseDir, bid, cid, meta);
-            return success(`case ${cid} ${field} updated`);
-        }
-        default:
-            return error(`unknown benchmark subcommand: ${sub}\n${BENCHMARK_USAGE}`);
-    }
-}
-//# sourceMappingURL=benchmark-command.js.map

package/lib/benchmark.d.ts

@@ -1,151 +0,0 @@
-import type { EvolutionEngine } from "./service.js";
-export interface BenchmarkCase {
-    id: string;
-    title: string;
-    statement: string;
-    rubric: string;
-    /**
-     * Case lifecycle state (gap A5): draft → calibrating → frozen.
-     * - draft: newly added, not yet quality-checked or calibrated
-     * - calibrating: pilot run in progress (case may be edited)
-     * - frozen: calibrated, locked as a formal baseline (immutable)
-     */
-    status?: "draft" | "calibrating" | "frozen";
-}
-/**
- * Persistent per-case metadata (stored in `cases/<cid>/meta.json`).
- * Carries quality-gate annotations and calibration history.
- */
-export interface CaseMeta {
-    status: "draft" | "calibrating" | "frozen";
-    /** What this case tests — the capability contract. */
-    capability: string;
-    /** What distinguishes a pass from a fail. */
-    distinguisher: string;
-    /** Known shortcuts the agent might use to game the rubric. */
-    shortcuts: string;
-    /** Calibration run history (appended on each pilot run). */
-    calibrationHistory: CalibrationRecord[];
-}
-/** One pilot-run record in the calibration history. */
-export interface CalibrationRecord {
-    runAt: string;
-    score: number;
-    passed: boolean;
-    notes: string;
-    /** Whether the case was modified after this run. */
-    modified: boolean;
-}
-export interface BenchmarkDefinition {
-    id: string;
-    title: string;
-    description: string;
-    runs: number;
-    passThreshold: number;
-    createdAt: string;
-}
-export interface CellScore {
-    caseId: string;
-    run: number;
-    /**
-     * Failure-cell protocol (gap A2): "ok" = a real score; "failed" = the
-     * unit could not produce one (rubric decrypt error, child crash, protocol
-     * error). A failed cell is NOT a zero — aggregation excludes it and
-     * counts it, and the acceptance rule rejects a round with more failures
-     * than the threshold instead of silently averaging a 0 into the mean.
-     */
-    status: "ok" | "failed";
-    score: number;
-    passed: boolean;
-    notes: string;
-    /**
-     * Trace evidence pointer (gap A4): the executor child's session id whose
-     * transcript produced this cell's evidence — the score can be drilled
-     * back to the exact session steps that earned it.
-     */
-    sessionId?: string;
-    /**
-     * Runtime evidence verification (gap A3): the actual provider and model
-     * used by the evaluation units — written from the host (not the model),
-     * so it reflects reality. Combined with `caseHash`, these make material
-     * and route drift between reference and candidate runs detectable:
-     * `score.flagMaterialDrift` re-marks a candidate cell failed when its
-     * case hash no longer matches the reference (version_changed semantics).
-     */
-    provider?: string;
-    model?: string;
-    /**
-     * Material hash: SHA-256 prefix of the case statement + rubric envelope,
-     * so a material change between reference and candidate runs is detectable.
-     * absence means pre-A3 cell (backward compatible).
-     */
-    caseHash?: string;
-    /**
-     * Gap C3: wall-clock duration of this cell's evaluation (both executor +
-     * reviewer subagents) in milliseconds. Absent means pre-C3 cell.
-     */
-    durationMs?: number;
-}
-export interface EvaluationEntry {
-    label: string;
-    refinementId?: string;
-    createdAt: string;
-    cells: CellScore[];
-    /** Code-owned aggregates (model never writes these). */
-    aggregate: Record<string, number | null>;
-    overall: number | null;
-}
-export interface Scoreboard {
-    reference?: EvaluationEntry;
-    candidates: EvaluationEntry[];
-    decisions: {
-        candidateLabel: string;
-        refinementId?: string;
-        accepted: boolean;
-        reasons: string[];
-        createdAt: string;
-    }[];
-}
-export interface AutoRollbackOutcome {
-    rolledBack: boolean;
-    message: string;
-}
-/**
- * Close the acceptance loop: when the code-owned decision rejects a
- * candidate refinement, revert it deterministically. The rollback is the
- * same engine path as `/evolve rollback` (inverse edits rebuilt from the
- * applied result — no LLM re-guessing), so it snapshots, versions, and
- * audits like any other mutation. Failures (e.g. the refinement belongs to
- * another session's history) are reported, never thrown: the command shows
- * the manual fallback instead.
- */
-export declare function rollbackRejectedCandidate(engine: EvolutionEngine, sessionId: string | undefined, candidateId: string): AutoRollbackOutcome;
-export declare function benchmarkDir(baseDir: string, bid: string): string;
-export declare function sanitizeId(raw: string): string;
-export declare function createBenchmark(baseDir: string, opts: {
-    title: string;
-    description?: string;
-    runs?: number;
-    passThreshold?: number;
-}): BenchmarkDefinition;
-export declare function listBenchmarks(baseDir: string): BenchmarkDefinition[];
-export declare function loadBenchmark(baseDir: string, bid: string): BenchmarkDefinition | undefined;
-export declare function addCase(baseDir: string, bid: string, title: string, statement: string, rubric: string, rubricKey?: Buffer): BenchmarkCase;
-export declare function listCases(baseDir: string, bid: string): BenchmarkCase[];
-export declare function loadScoreboard(baseDir: string, bid: string): Scoreboard;
-export declare function saveScoreboard(baseDir: string, bid: string, board: Scoreboard): void;
-export declare function loadCaseMeta(baseDir: string, bid: string, cid: string): CaseMeta | undefined;
-export declare function saveCaseMeta(baseDir: string, bid: string, cid: string, meta: CaseMeta): void;
-/**
- * Transition a case's lifecycle state. Throws on illegal transitions.
- *   draft → calibrating (start pilot)
- *   calibrating → draft (abandon calibration)
- *   calibrating → frozen (lock baseline)
- */
-export declare function transitionCaseStatus(baseDir: string, bid: string, cid: string, to: "draft" | "calibrating" | "frozen"): CaseMeta;
-/**
- * Quality-check a case: mechanical validation without LLM calls.
- * Returns human-readable problems; empty array means the case passes.
- */
-export declare function caseCheckProblems(baseDir: string, bid: string, cid: string): string[];
-//# sourceMappingURL=benchmark.d.ts.map