dsh-continual-evolve 0.2.0 → 0.3.0

package/lib/command.js

@@ -4,16 +4,15 @@ import { planWithLlm } from "./planner.js";
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { requireGlobalApproval } from "./approval.js";
-import { assessLocalEntries, candidateKey, filterPromotable, listLocalCandidates, splitArchiveGuards, splitPromoteBlocked, splitPromoteProposals, wholePromoteProposals } from "./wrapup.js";
 import { saveHarnessState } from "./state.js";
-import { loadLedger, mountSkill, unmountSkill } from "./mount.js";
-import { blockEvolutionGoal, completeEvolutionGoal, goalServiceOf, goalStatusText, upsertEvolutionGoal } from "./goal.js";
 import { appendResult, storePaths } from "./store.js";
-import { addCase, createBenchmark, listBenchmarks, listCases, loadBenchmark, loadScoreboard, rollbackRejectedCandidate, saveScoreboard } from "./benchmark.js";
-import { decide, decisionReport, entryFromCells } from "./score.js";
-import { evaluateState } from "./evaluate.js";
 import { entrySourceOf } from "./source.js";
 import { filterLogBySession, formatLogLine, pluginLogFilePath } from "./logfile.js";
+import { readBenchmarkFailures, readReviewFailures, summarizeFailures, formatFailureSummary } from "./failures.js";
+import { executeGoalCommand } from "./goal-command.js";
+import { executeMountCommand, executeUnmountCommand } from "./mount-command.js";
+import { executeBenchmarkCommand } from "./benchmark-command.js";
+import { executeWrapupCommand } from "./wrapup-command.js";
 const USAGE = `Usage:
   /evolve                  show this help and the current local store
   /evolve list [global]    list entries (add "global" for the cross-session store)
@@ -25,6 +24,7 @@ const USAGE = `Usage:
   /evolve archive <id> [global]   hide an entry from injection (data kept, restorable)
   /evolve unarchive <id> [global] restore an archived entry
   /evolve log [tail N]            show the recent plugin log (default 50 lines)
+  /evolve failures               aggregated failure counts (gate + benchmark, by class)
   /evolve export [global] <path>  backup a store to a JSON file
   /evolve import [global] <path>  restore a store from an export file
   /evolve mount <skillId>    hot-mount a skill entry as a live cordis plugin
@@ -167,6 +167,22 @@ async function executeEvolveCommand(ctx, engine, invocation, opts, runtime) {
                 }, { scope });
                 return success(renderResult(result));
             }
+            case "failures": {
+                // /evolve failures — failure-signature aggregation (D1 observation):
+                // failed review-gate records + failed benchmark cells, counted by class.
+                const failed = [...readReviewFailures(engine.baseDir), ...readBenchmarkFailures(engine.baseDir)];
+                const summary = summarizeFailures(failed);
+                const parts = formatFailureSummary(summary).split("\n");
+                const recent = failed
+                    .sort((a, b) => (b.timestamp ?? "").localeCompare(a.timestamp ?? ""))
+                    .slice(0, 10)
+                    .map((f) => `  [${f.timestamp ?? "(benchmark)"}] ${f.kind} · ${f.source}: ${f.message.slice(0, 140)}`);
+                if (recent.length > 0) {
+                    parts.push("recent 10:");
+                    parts.push(...recent);
+                }
+                return success(parts.join("\n"));
+            }
             case "log": {
                 // /evolve log [tail N] [session <sessionId>]
                 let tail = 50;
@@ -279,18 +295,13 @@ async function executeEvolveCommand(ctx, engine, invocation, opts, runtime) {
                 return executeGoalCommand(ctx, invocation, rest);
             }
             case "mount": {
-                return executeMountCommand(ctx, engine, invocation, rest);
+                return await executeMountCommand(ctx, engine, invocation, rest);
             }
             case "unmount": {
-                const id = stripAngleBrackets(rest[0] ?? "");
-                if (!id) {
-                    return error(`unmount requires a mount id (see /evolve mount list).`);
-                }
-                const record = await unmountSkill(ctx, engine.baseDir, id);
-                return record ? success(`unmounted ${record.id} (${record.entryId})`) : error(`no mount found for ${id}`);
+                return await executeUnmountCommand(ctx, engine, rest);
             }
             case "benchmark": {
-                return executeBenchmarkCommand(ctx, engine, invocation, rest, runtime);
+                return await executeBenchmarkCommand(ctx, engine, invocation, rest, runtime);
             }
             default:
                 return error(`unknown subcommand: ${sub}\n${USAGE}`);
@@ -300,434 +311,6 @@ async function executeEvolveCommand(ctx, engine, invocation, opts, runtime) {
         return error(cause instanceof Error ? cause.message : String(cause));
     }
 }
-function executeGoalCommand(ctx, invocation, rest) {
-    const agent = invocation.agent;
-    const goals = goalServiceOf(ctx);
-    if (!goals) {
-        return error(`/evolve goal requires the goals service (load @deepseek-ai/dsh-goal)`);
-    }
-    const sub = rest[0] ?? "";
-    try {
-        if (sub === "done") {
-            const view = completeEvolutionGoal(ctx, agent);
-            return view ? success(`evolution goal completed: ${goalStatusText(view)}`) : success("(no goal to complete)");
-        }
-        if (sub === "block") {
-            const reason = rest.slice(1).join(" ") || "user requested block";
-            const view = blockEvolutionGoal(ctx, agent, reason);
-            return view ? success(`evolution goal blocked: ${goalStatusText(view)}`) : success("(no active goal to block)");
-        }
-        if (sub.length === 0) {
-            const current = goals.get(agent);
-            return current ? success(goalStatusText(current)) : success("(no evolution goal — /evolve goal <objective> to create one)");
-        }
-        const objective = rest.join(" ");
-        const view = upsertEvolutionGoal(ctx, agent, objective);
-        return success(`evolution goal ready: ${goalStatusText(view)}\n(active goal drives the review gate every round)`);
-    }
-    catch (cause) {
-        return error(cause instanceof Error ? cause.message : String(cause));
-    }
-}
-async function executeWrapupCommand(ctx, engine, invocation) {
-    const sessionId = invocation.agent.id;
-    const localState = engine.load("local", sessionId);
-    const globalState = engine.load("global", undefined);
-    const candidates = listLocalCandidates(localState, globalState);
-    if (candidates.length === 0) {
-        return success(`(nothing to wrap up: ${sessionId}'s local store has no active, un-promoted entries — use /evolve list to inspect it)`);
-    }
-    // 1. Classify: the model judges each audited candidate's fate.
-    const assessment = await assessLocalEntries(ctx, invocation.agent, candidates, { signal: invocation.signal });
-    const byKey = new Map(candidates.map((candidate) => [candidateKey(candidate.kind, candidate.id), candidate]));
-    // 2. Partition by action. Deterministic guards re-check the LIVE global
-    //    store right before anything lands (state may have changed mid-call).
-    const { promotable, skipped } = filterPromotable(assessment.items, globalState, candidates);
-    const promoteItems = promotable.filter((item) => item.verdict === "promote");
-    const archiveItems = assessment.items.filter((item) => item.verdict === "archive");
-    // Split promotion (A-form): archive a mixed entry but promote ONLY the
-    // cleaned durable part the model extracted. Guarded the same way as whole
-    // promotes — a split that would duplicate a globally covered topic is
-    // dropped and the entry archives plain.
-    const splitItems = [];
-    const splitSkipped = [];
-    for (const item of archiveItems) {
-        if (!item.promote)
-            continue;
-        const candidate = byKey.get(item.key);
-        if (!candidate) {
-            splitSkipped.push({ key: item.key, reason: "not in the audited candidate list" });
-            continue;
-        }
-        const blocked = splitPromoteBlocked(item, globalState, candidate.kind);
-        if (blocked) {
-            splitSkipped.push({ key: item.key, reason: blocked });
-            continue;
-        }
-        splitItems.push({ item, candidate });
-    }
-    // Plain archives (no split payload): the symmetric guard — an archive that
-    // is NOT globally covered AND was distilled from real user messages must
-    // not proceed silently.
-    const plainArchives = archiveItems.filter((item) => !item.promote);
-    const { silent: silentArchives, review: reviewArchives } = splitArchiveGuards(plainArchives, candidates);
-    const keepItems = assessment.items.filter((item) => item.verdict === "keep");
-    // 3. Report the assessment before touching anything.
-    const lines = [
-        `wrapup assessment (${sessionId}): ${candidates.length} candidates${candidates.some((c) => c.coveredGlobally) ? `, ${candidates.filter((c) => c.coveredGlobally).length} covered globally` : ""}`,
-        `${assessment.rationale}`,
-    ];
-    for (const [heading, items] of [
-        ["PROMOTE (to global)", promoteItems],
-        ["SPLIT (archive + promote durable part)", splitItems.map((split) => split.item)],
-        ["ARCHIVE", silentArchives],
-        ["ARCHIVE (needs review)", reviewArchives],
-        ["KEEP", keepItems],
-    ]) {
-        lines.push(`${heading}: ${items.length}`);
-        for (const item of items) {
-            const candidate = byKey.get(item.key);
-            const title = candidate ? candidate.title : item.key;
-            const splitNote = item.promote ? ` → 拆出提升「${item.promote.title}」` : "";
-            lines.push(`- ${item.key} "${title}"${splitNote} — ${item.reason}`);
-        }
-    }
-    for (const skip of skipped) {
-        lines.push(`- promote skipped: ${skip.key} — ${skip.reason}`);
-    }
-    for (const skip of splitSkipped) {
-        lines.push(`- split skipped: ${skip.key} — ${skip.reason}`);
-    }
-    lines.push("");
-    const applied = [];
-    // 4. Global writes: governed resource — ONE human approval gate covers
-    //    every create (whole promotes AND split promotions). On approval:
-    //    - whole promote → create global copy + stamp local promotedTo+archivedAt;
-    //    - split → create the cleaned durable part + archive the original with
-    //      promotedTo. On rejection: whole promotes are not written, and each
-    //      split's original STILL archives plain (its snapshot half deserves
-    //      the archive; the durable half is reported for manual handling).
-    const wholeCreates = promoteItems.map((item) => ({ item, candidate: byKey.get(item.key) }));
-    const splitCreates = splitItems;
-    const allCreates = new Set([...wholeCreates.map((c) => c.item.key), ...splitCreates.map((c) => c.item.key)]);
-    if (allCreates.size > 0) {
-        const what = `wrapup 将写入跨会话 global store（共 ${allCreates.size} 条：${promoteItems.length} 条整条提升 + ${splitItems.length} 条拆解提升）：\n${[
-            ...promoteItems.map((item) => `- 整条提升 ${item.key} "${byKey.get(item.key)?.title ?? item.key}"`),
-            ...splitItems.map((split) => `- 拆解提升 ${split.item.key} → 清洗「${split.item.promote?.title}」（原条目随之归档）`),
-        ].join("\n")}`;
-        let promoteAllowed = true;
-        try {
-            await requireGlobalApproval(ctx, invocation.agent, invocation.signal, what);
-        }
-        catch (cause) {
-            promoteAllowed = false;
-            const message = `global 写入未批准 — 整条提升与拆解提升均未写入 (${cause instanceof Error ? cause.message : String(cause)})`;
-            applied.push(message);
-            lines.push(message);
-        }
-        if (promoteAllowed) {
-            // Whole promotes: create global entry, retire the local copy.
-            // Shared proposal builders keep the wrap-up command and the gate's
-            // local-fate dimension writing IDENTICAL edits.
-            for (const { item, candidate } of wholeCreates) {
-                if (!candidate)
-                    continue;
-                const proposals = wholePromoteProposals(item, candidate, sessionId);
-                const globalResult = engine.apply("global", undefined, proposals.global, { scope: "global" });
-                const createdId = globalResult.appliedEdits.find((edit) => edit.applied)?.id ?? candidate.id;
-                const localResult = engine.apply("local", sessionId, proposals.localStamp(createdId), {
-                    scope: "local",
-                    baselineState: localState,
-                });
-                applied.push(`promoted ${item.key} → global:${createdId} (${globalResult.id}; local stamped ${localResult.id})`);
-            }
-            // Split promotions: create the cleaned durable part, retire the
-            // original local entry (its snapshot half is archived along).
-            for (const { item, candidate } of splitCreates) {
-                if (!item.promote)
-                    continue;
-                const proposals = splitPromoteProposals(item, candidate, sessionId);
-                const globalResult = engine.apply("global", undefined, proposals.global, { scope: "global" });
-                const createdId = globalResult.appliedEdits.find((edit) => edit.applied)?.id ?? candidate.id;
-                const localResult = engine.apply("local", sessionId, proposals.localStamp(createdId), {
-                    scope: "local",
-                    baselineState: localState,
-                });
-                applied.push(`split ${item.key}: promoted cleaned part → global:${createdId} (${globalResult.id}); original archived (${localResult.id})`);
-            }
-        }
-        else {
-            // Rejected: whole promotes stay un-written; each split's original
-            // still archives plain (reported, data restorable).
-            for (const { item, candidate } of splitCreates) {
-                if (!candidate)
-                    continue;
-                const result = engine.apply("local", sessionId, {
-                    summary: `wrapup: split promotion not approved — archive original ${item.key} plain`,
-                    rationale: item.reason,
-                    expectedOutcome: `The original leaves injection; the cleaned part was NOT written (reported for manual handling).`,
-                    edits: [{ action: "archive", kind: candidate.kind, id: candidate.id }],
-                }, { scope: "local", baselineState: localState });
-                applied.push(`split ${item.key}: promotion not approved — original archived plain (${result.id})`);
-            }
-        }
-    }
-    // 5. Silent archives: deterministic local action (hidden from injection,
-    //    data kept restorable) — covered topics and operational entries need no
-    //    confirmation, matching the original behavior.
-    for (const item of silentArchives) {
-        const candidate = byKey.get(item.key);
-        if (!candidate)
-            continue;
-        const result = engine.apply("local", sessionId, {
-            summary: `wrapup: archive local ${item.key} — ${item.reason}`,
-            rationale: item.reason,
-            expectedOutcome: `The entry stops being injected but stays restorable.`,
-            edits: [{ action: "archive", kind: candidate.kind, id: candidate.id }],
-        }, { scope: "local", baselineState: localState });
-        applied.push(`archived ${item.key} (${result.id})`);
-    }
-    // 6. Review archives (symmetric guard): not covered globally + distilled
-    //    from real user messages — the user decides before this content is
-    //    hidden from future sessions. No question service → conservative keep.
-    const userQuestions = ctx.userQuestions;
-    for (const item of reviewArchives) {
-        const candidate = byKey.get(item.key);
-        if (!candidate)
-            continue;
-        if (!userQuestions) {
-            applied.push(`kept ${item.key} — archive pending user confirmation (no question service)`);
-            continue;
-        }
-        const questionId = "evolve-wrapup-archive-review";
-        let archiveConfirmed = false;
-        try {
-            const answer = await userQuestions.ask({
-                questions: [
-                    {
-                        id: questionId,
-                        question: `wrapup：条目「${candidate.title}」未被全局覆盖且源自真实对话，直接归档会隐藏它（数据保留、可恢复）。确认归档？`,
-                        options: [{ label: "归档" }, { label: "保留" }],
-                    },
-                ],
-                agent: invocation.agent,
-                signal: invocation.signal,
-            });
-            archiveConfirmed = answer.answers?.find((entry) => entry.id === questionId)?.selected?.includes("归档") ?? false;
-        }
-        catch {
-            archiveConfirmed = false;
-        }
-        if (archiveConfirmed) {
-            const result = engine.apply("local", sessionId, {
-                summary: `wrapup: archive local ${item.key} (user-confirmed) — ${item.reason}`,
-                rationale: item.reason,
-                expectedOutcome: `The entry stops being injected but stays restorable.`,
-                edits: [{ action: "archive", kind: candidate.kind, id: candidate.id }],
-            }, { scope: "local", baselineState: localState });
-            applied.push(`archived ${item.key} (user-confirmed, ${result.id})`);
-        }
-        else {
-            applied.push(`kept ${item.key} — user declined the archive`);
-        }
-    }
-    lines.push(...(applied.length > 0 ? applied : ["(no changes applied — all entries kept)"]));
-    return success(lines.join("\n"));
-}
-async function executeMountCommand(ctx, engine, invocation, rest) {
-    const sub = rest[0] ?? "";
-    if (sub === "list") {
-        const ledger = loadLedger(engine.baseDir);
-        if (ledger.mounted.length === 0) {
-            return success("(no hot-mounted plugins — /evolve mount <skillId>)");
-        }
-        return success(ledger.mounted.map((m) => `- ${m.id} (${m.entryId}, v${m.version}, ${m.mountedAt})`).join("\n"));
-    }
-    const skillId = stripAngleBrackets(sub);
-    if (!skillId) {
-        return error(`mount requires a skill entry id.\nUsage: /evolve mount <skillId> | /evolve mount list`);
-    }
-    const sessionId = invocation.agent.id;
-    const local = engine.load("local", sessionId);
-    const globalState = engine.load("global", undefined);
-    const entry = local.entries.skill[skillId] ??
-        globalState.entries.skill[skillId] ??
-        Object.values(local.entries.skill).find((e) => e.id === skillId) ??
-        Object.values(globalState.entries.skill).find((e) => e.id === skillId);
-    if (!entry) {
-        return error(`skill entry ${skillId} not found in local or global store`);
-    }
-    try {
-        const record = await mountSkill(ctx, engine.baseDir, entry);
-        return success(`mounted ${record.id} as ${record.entryId} (v${record.version}) — tool: skill_${record.id.replace(/_/g, "-")}`);
-    }
-    catch (cause) {
-        return error(cause instanceof Error ? cause.message : String(cause));
-    }
-}
-/** " (N failed)" suffix when an evaluation entry carries failed cells. */
-function failedTextOf(entry) {
-    const failed = entry.cells.filter((cell) => cell.status === "failed").length;
-    return failed > 0 ? ` (${failed} failed)` : "";
-}
-async function executeBenchmarkCommand(ctx, engine, invocation, rest, runtime) {
-    const sub = rest[0] ?? "";
-    const args = rest.slice(1);
-    const sessionId = invocation.agent.id;
-    const baseDir = engine.baseDir;
-    switch (sub) {
-        case "":
-        case "help":
-            return success(BENCHMARK_USAGE);
-        case "new": {
-            const title = args[0] ?? "";
-            if (!title) {
-                return error(`benchmark new requires a title.\n${BENCHMARK_USAGE}`);
-            }
-            const runs = args[1] !== undefined ? parsePositiveInt(args[1], "runs") : undefined;
-            const definition = createBenchmark(baseDir, { title, ...(runs !== undefined ? { runs } : {}) });
-            return success(`benchmark ${definition.id} created (runs=${definition.runs}, passThreshold=${definition.passThreshold})\nAdd cases with: /evolve benchmark add-case ${definition.id} "<title>" "<statement>" "<rubric>"`);
-        }
-        case "list": {
-            const benchmarks = listBenchmarks(baseDir);
-            if (benchmarks.length === 0) {
-                return success("(no benchmarks yet — use /evolve benchmark new <title>)");
-            }
-            const lines = benchmarks.map((b) => {
-                const cases = listCases(baseDir, b.id);
-                const board = loadScoreboard(baseDir, b.id);
-                const ref = board.reference ? ` ref=${board.reference.overall ?? "?"}` : " no-reference";
-                return `- ${b.id} (${cases.length} cases, runs=${b.runs})${ref}`;
-            });
-            return success(lines.join("\n"));
-        }
-        case "add-case": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const title = args[1] ?? "";
-            const statement = args[2] ?? "";
-            const rubric = args[3] ?? "";
-            if (!bid || !title || !statement || !rubric) {
-                return error(`benchmark add-case needs <bid> <title> <statement> <rubric>.\n${BENCHMARK_USAGE}`);
-            }
-            const caseItem = addCase(baseDir, bid, title, statement, rubric, runtime.rubricKey);
-            return success(`case ${caseItem.id} added to ${bid}`);
-        }
-        case "reset": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            if (!bid) {
-                return error(`benchmark reset needs a <bid>.\n${BENCHMARK_USAGE}`);
-            }
-            if (!loadBenchmark(baseDir, bid)) {
-                return error(`benchmark ${bid} not found`);
-            }
-            saveScoreboard(baseDir, bid, { candidates: [], decisions: [] });
-            return success(`scoreboard reset for ${bid} — run /evolve benchmark run ${bid} to record a fresh reference`);
-        }
-        case "status": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const board = loadScoreboard(baseDir, bid);
-            const lines = [];
-            if (board.reference) {
-                lines.push(`reference "${board.reference.label}": overall=${board.reference.overall ?? "?"} cells=${board.reference.cells.length}${failedTextOf(board.reference)}`);
-            }
-            else {
-                lines.push("(no reference evaluation yet)");
-            }
-            for (const c of board.candidates) {
-                lines.push(`candidate "${c.label}": overall=${c.overall ?? "?"} cells=${c.cells.length}${failedTextOf(c)}${c.refinementId ? ` (${c.refinementId})` : ""}`);
-            }
-            for (const d of board.decisions) {
-                lines.push(`decision: ${d.accepted ? "ACCEPTED" : "rejected"} ${d.candidateLabel} — ${d.reasons.join("; ") || "ok"}`);
-            }
-            return success(lines.join("\n") || "(empty scoreboard)");
-        }
-        case "run": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const candidateId = args.includes("candidate") ? stripAngleBrackets(args[args.indexOf("candidate") + 1] ?? "") : undefined;
-            const definition = loadBenchmark(baseDir, bid);
-            if (!definition) {
-                return error(`benchmark ${bid} not found`);
-            }
-            const cases = listCases(baseDir, bid);
-            if (cases.length === 0) {
-                return error(`benchmark ${bid} has no cases — use /evolve benchmark add-case`);
-            }
-            const board = loadScoreboard(baseDir, bid);
-            const label = candidateId ? `candidate:${candidateId}` : "reference";
-            if (!candidateId && board.reference) {
-                return error(`reference already evaluated (${board.reference.overall ?? "?"}); evaluate a candidate instead: /evolve benchmark run ${bid} candidate <refinementId>`);
-            }
-            const overview = formatHarnessStateForPrompt(engine.load("local", sessionId));
-            const outcome = await evaluateState(ctx, invocation.agent, {
-                cases,
-                rubricKey: runtime.rubricKey,
-                runs: definition.runs,
-                passThreshold: definition.passThreshold,
-                harnessOverview: overview,
-                label,
-                signal: invocation.signal,
-            });
-            const entry = entryFromCells(label, outcome.cells, candidateId);
-            const failedCells = outcome.cells.filter((cell) => cell.status === "failed").length;
-            const lines = [
-                `evaluation "${label}": ${outcome.cells.length} cells${failedCells > 0 ? `, ${failedCells} failed` : ""}, overall=${entry.overall ?? "?"}`,
-                ...Object.entries(entry.aggregate)
-                    .filter(([key]) => key !== "overall" && key !== "failed" && key !== "total")
-                    .map(([key, value]) => `  ${key}: ${value ?? "?"}`),
-            ];
-            if (failedCells > 0) {
-                for (const cell of outcome.cells.filter((cell) => cell.status === "failed")) {
-                    lines.push(`  [failed] ${cell.caseId} r${cell.run}: ${cell.notes}`);
-                }
-            }
-            if (candidateId) {
-                if (!board.reference) {
-                    lines.push("(no reference yet — this run only recorded the candidate)");
-                    board.candidates.push(entry);
-                }
-                else {
-                    const decision = decide(board.reference, entry, {
-                        passThreshold: definition.passThreshold,
-                        regressionTolerance: 0,
-                        maxFailedCells: 0,
-                    });
-                    board.candidates.push(entry);
-                    board.decisions.push({
-                        candidateLabel: label,
-                        refinementId: candidateId,
-                        accepted: decision.accepted,
-                        reasons: decision.reasons,
-                        createdAt: new Date().toISOString(),
-                    });
-                    lines.push(...decisionReport(board.reference, entry, decision));
-                    if (!decision.accepted) {
-                        lines.push(`Consider rolling back the candidate: /evolve rollback <${candidateId}>`);
-                        if (runtime.autoRollbackOnReject) {
-                            const outcome = rollbackRejectedCandidate(engine, sessionId, candidateId);
-                            lines.push(outcome.message);
-                        }
-                    }
-                }
-            }
-            else {
-                board.reference = entry;
-                lines.push("reference evaluation recorded as the baseline");
-            }
-            saveScoreboard(baseDir, bid, board);
-            return success(lines.join("\n"));
-        }
-        default:
-            return error(`unknown benchmark subcommand: ${sub}\n${BENCHMARK_USAGE}`);
-    }
-}
-const BENCHMARK_USAGE = `Usage:
-  /evolve benchmark new <title>                          create a benchmark (runs=1)
-  /evolve benchmark add-case <bid> <title> <statement> <rubric>
-  /evolve benchmark list                                 list benchmarks + reference status
-  /evolve benchmark status <bid>                         show scoreboard + decisions
-  /evolve benchmark reset <bid>                          clear the scoreboard (fresh reference)
-  /evolve benchmark run <bid>                            evaluate current state as the reference
-  /evolve benchmark run <bid> candidate <refinementId>   evaluate the post-refinement state and decide`;
 function renderResult(result) {
     const applied = result.appliedEdits.filter((e) => e.applied);
     const failed = result.appliedEdits.filter((e) => !e.applied);

package/lib/evaluate.d.ts

@@ -60,6 +60,13 @@ export interface ExecutorResult {
     run: number;
     evidence: string;
 }
+/**
+ * Deterministic hash of a case's statement + rubric envelope (gap A3):
+ * a 16-char SHA-256 prefix, hex-encoded. Used to detect material changes
+ * between reference and candidate evaluation runs (see
+ * `score.flagMaterialDrift`).
+ */
+export declare function caseHash(caseItem: BenchmarkCase): string;
 /** Validate a provider-validated executor result; returns undefined when malformed. */
 export declare function normalizeExecutor(value: unknown, caseId: string, run: number): ExecutorResult | undefined;
 /** Validate a provider-validated structured cell; returns undefined when malformed. */

package/lib/evaluate.js

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { mapPool } from "./pool.js";
 import { decryptRubric, deriveKey, DEV_RUBRIC_KEY } from "./rubric.js";
 /** Key used when the caller did not resolve one: mirrors resolveRubricKey's last-resort dev fallback. */
@@ -78,7 +79,19 @@ export async function evaluateState(ctx, agent, options) {
 function failedCell(caseId, run, message) {
     return { caseId, run, status: "failed", score: 0, passed: false, notes: message };
 }
+/**
+ * Deterministic hash of a case's statement + rubric envelope (gap A3):
+ * a 16-char SHA-256 prefix, hex-encoded. Used to detect material changes
+ * between reference and candidate evaluation runs (see
+ * `score.flagMaterialDrift`).
+ */
+export function caseHash(caseItem) {
+    const material = `${caseItem.statement}\n${caseItem.rubric}`;
+    return createHash("sha256").update(material).digest("hex").slice(0, 16);
+}
 async function runUnit(subagents, agent, options, c, run) {
+    // Gap C3: track wall-clock duration of the entire cell evaluation.
+    const cellStart = Date.now();
     // The ONLY rubric decryption point: the envelope is opened here, in the
     // host, and the plaintext goes ONLY into the reviewer prompt — the
     // executor branch never touches it (gap A1).
@@ -87,8 +100,13 @@ async function runUnit(subagents, agent, options, c, run) {
         rubric = decryptRubric(c.rubric, options.rubricKey ?? devRubricKey());
     }
     catch (cause) {
-        return failedCell(c.id, run, `rubric decrypt failed: ${cause instanceof Error ? cause.message : String(cause)}`);
+        return { ...failedCell(c.id, run, `rubric decrypt failed: ${cause instanceof Error ? cause.message : String(cause)}`), durationMs: Date.now() - cellStart };
     }
+    // Runtime evidence (gap A3): record actual provider/model from the host,
+    // and compute a material hash of the case for change detection.
+    const actualProvider = agent.options.provider ?? "unknown";
+    const actualModel = agent.options.model ?? "unknown";
+    const hash = caseHash(c);
     // Stage 1: executor — task + evidence, NO rubric.
     let evidence;
     let sessionId;
@@ -130,7 +148,7 @@ async function runUnit(subagents, agent, options, c, run) {
         }
     }
     catch (cause) {
-        return failedCell(c.id, run, `executor failed: ${cause instanceof Error ? cause.message : String(cause)}`);
+        return { ...failedCell(c.id, run, `executor failed: ${cause instanceof Error ? cause.message : String(cause)}`), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
     }
     // Stage 2: independent reviewer — rubric + evidence, NO task execution.
     try {
@@ -166,17 +184,14 @@ async function runUnit(subagents, agent, options, c, run) {
             if (!cell) {
                 throw new Error("reviewer returned neither a structured value nor usable text");
             }
-            if (sessionId !== undefined) {
-                return { ...cell, sessionId };
-            }
-            return cell;
+            return { ...cell, ...(sessionId !== undefined ? { sessionId } : {}), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
         }
         finally {
             reviewerRun.dispose();
         }
     }
     catch (cause) {
-        return failedCell(c.id, run, `reviewer failed: ${cause instanceof Error ? cause.message : String(cause)}`);
+        return { ...failedCell(c.id, run, `reviewer failed: ${cause instanceof Error ? cause.message : String(cause)}`), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
     }
 }
 /** Cap the evidence handed to the reviewer so the grading call stays bounded. */

package/lib/evolve-event.d.ts

@@ -0,0 +1,38 @@
+import type { RefinementResult } from "./types.js";
+/** The structured event payload emitted after every successful refinement. */
+export interface EvolveCompleteEvent {
+    /** Event discriminator for consumers. */
+    type: "evolve_complete";
+    /** The refinement id (same as the result). */
+    refinementId: string;
+    /** One-line summary of the refinement. */
+    summary: string;
+    /** Number of edits that were actually applied. */
+    appliedEdits: number;
+    /** Number of edits that failed to apply. */
+    failedEdits: number;
+    /** Scope of the refinement ("local" or "global"). */
+    scope: string;
+    /** What triggered this refinement (e.g. "auto_review", "manual_plan", "manual_tool"). */
+    trigger: string;
+    /** Session id that owns the refinement (auto or manual). */
+    sessionId: string;
+    /** ISO timestamp. */
+    timestamp: string;
+    /** Per-edit summaries (kind + id + action) for consumers that want detail. */
+    edits: {
+        action: string;
+        kind: string;
+        id: string;
+        applied: boolean;
+    }[];
+}
+/** Build a structured evolve-complete event from a refinement result. */
+export declare function buildEvolveCompleteEvent(result: RefinementResult, trigger: string, sessionId: string): EvolveCompleteEvent;
+/**
+ * Emit an evolve_complete event to the reviews.jsonl audit trail. The event
+ * is JSONL-formatted (one line) so consumers can tail and parse it. This is
+ * a best-effort write — failure never blocks the refinement path.
+ */
+export declare function emitEvolveComplete(baseDir: string, event: EvolveCompleteEvent): void;
+//# sourceMappingURL=evolve-event.d.ts.map