dsh-continual-evolve 0.1.1 → 0.3.0

package/lib/command.js

@@ -2,25 +2,29 @@ import { ARCHIVED_AT_KEY } from "./types.js";
 import { formatHarnessStateForPrompt, historyForPrompt } from "./render.js";
 import { planWithLlm } from "./planner.js";
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
 import { requireGlobalApproval } from "./approval.js";
 import { saveHarnessState } from "./state.js";
-import { loadLedger, mountSkill, unmountSkill } from "./mount.js";
-import { blockEvolutionGoal, completeEvolutionGoal, goalServiceOf, goalStatusText, upsertEvolutionGoal } from "./goal.js";
 import { appendResult, storePaths } from "./store.js";
-import { addCase, createBenchmark, listBenchmarks, listCases, loadBenchmark, loadScoreboard, rollbackRejectedCandidate, saveScoreboard } from "./benchmark.js";
-import { decide, decisionReport, entryFromCells } from "./score.js";
-import { evaluateState } from "./evaluate.js";
 import { entrySourceOf } from "./source.js";
 import { filterLogBySession, formatLogLine, pluginLogFilePath } from "./logfile.js";
+import { readBenchmarkFailures, readReviewFailures, summarizeFailures, formatFailureSummary } from "./failures.js";
+import { executeGoalCommand } from "./goal-command.js";
+import { executeMountCommand, executeUnmountCommand } from "./mount-command.js";
+import { executeBenchmarkCommand } from "./benchmark-command.js";
+import { executeWrapupCommand } from "./wrapup-command.js";
 const USAGE = `Usage:
   /evolve                  show this help and the current local store
   /evolve list [global]    list entries (add "global" for the cross-session store)
   /evolve history [global] show applied refinements (rollback ids)
   /evolve rollback <id> [global]  deterministically revert a refinement
   /evolve plan [msg]       run the LLM planner against the current store
+  /evolve wrapup           assess this session's local entries: promote reusable ones
+                           to the global store (approval required), archive one-offs
   /evolve archive <id> [global]   hide an entry from injection (data kept, restorable)
   /evolve unarchive <id> [global] restore an archived entry
   /evolve log [tail N]            show the recent plugin log (default 50 lines)
+  /evolve failures               aggregated failure counts (gate + benchmark, by class)
   /evolve export [global] <path>  backup a store to a JSON file
   /evolve import [global] <path>  restore a store from an export file
   /evolve mount <skillId>    hot-mount a skill entry as a live cordis plugin
@@ -163,6 +167,22 @@ async function executeEvolveCommand(ctx, engine, invocation, opts, runtime) {
                 }, { scope });
                 return success(renderResult(result));
             }
+            case "failures": {
+                // /evolve failures — failure-signature aggregation (D1 observation):
+                // failed review-gate records + failed benchmark cells, counted by class.
+                const failed = [...readReviewFailures(engine.baseDir), ...readBenchmarkFailures(engine.baseDir)];
+                const summary = summarizeFailures(failed);
+                const parts = formatFailureSummary(summary).split("\n");
+                const recent = failed
+                    .sort((a, b) => (b.timestamp ?? "").localeCompare(a.timestamp ?? ""))
+                    .slice(0, 10)
+                    .map((f) => `  [${f.timestamp ?? "(benchmark)"}] ${f.kind} · ${f.source}: ${f.message.slice(0, 140)}`);
+                if (recent.length > 0) {
+                    parts.push("recent 10:");
+                    parts.push(...recent);
+                }
+                return success(parts.join("\n"));
+            }
             case "log": {
                 // /evolve log [tail N] [session <sessionId>]
                 let tail = 50;
@@ -255,6 +275,8 @@ async function executeEvolveCommand(ctx, engine, invocation, opts, runtime) {
                     ...(instructions ? { instructions } : {}),
                     global: scope === "global",
                     signal: invocation.signal,
+                    // skill-creator template facts (fallback: builtin guide).
+                    skillsRoot: join(engine.baseDir, "skills"),
                 });
                 if (scope === "global" && opts.requireGlobalApproval && proposal.edits.length > 0) {
                     await requireGlobalApproval(ctx, invocation.agent, invocation.signal, `/evolve plan global 将应用 ${proposal.edits.length} 条编辑到跨会话 store：${proposal.summary}`);
@@ -266,22 +288,20 @@ async function executeEvolveCommand(ctx, engine, invocation, opts, runtime) {
                 });
                 return success(renderResult(result));
             }
+            case "wrapup": {
+                return await executeWrapupCommand(ctx, engine, invocation);
+            }
             case "goal": {
                 return executeGoalCommand(ctx, invocation, rest);
             }
             case "mount": {
-                return executeMountCommand(ctx, engine, invocation, rest);
+                return await executeMountCommand(ctx, engine, invocation, rest);
             }
             case "unmount": {
-                const id = stripAngleBrackets(rest[0] ?? "");
-                if (!id) {
-                    return error(`unmount requires a mount id (see /evolve mount list).`);
-                }
-                const record = await unmountSkill(ctx, engine.baseDir, id);
-                return record ? success(`unmounted ${record.id} (${record.entryId})`) : error(`no mount found for ${id}`);
+                return await executeUnmountCommand(ctx, engine, rest);
             }
             case "benchmark": {
-                return executeBenchmarkCommand(ctx, engine, invocation, rest, runtime);
+                return await executeBenchmarkCommand(ctx, engine, invocation, rest, runtime);
             }
             default:
                 return error(`unknown subcommand: ${sub}\n${USAGE}`);
@@ -291,214 +311,6 @@ async function executeEvolveCommand(ctx, engine, invocation, opts, runtime) {
         return error(cause instanceof Error ? cause.message : String(cause));
     }
 }
-function executeGoalCommand(ctx, invocation, rest) {
-    const agent = invocation.agent;
-    const goals = goalServiceOf(ctx);
-    if (!goals) {
-        return error(`/evolve goal requires the goals service (load @deepseek-ai/dsh-goal)`);
-    }
-    const sub = rest[0] ?? "";
-    try {
-        if (sub === "done") {
-            const view = completeEvolutionGoal(ctx, agent);
-            return view ? success(`evolution goal completed: ${goalStatusText(view)}`) : success("(no goal to complete)");
-        }
-        if (sub === "block") {
-            const reason = rest.slice(1).join(" ") || "user requested block";
-            const view = blockEvolutionGoal(ctx, agent, reason);
-            return view ? success(`evolution goal blocked: ${goalStatusText(view)}`) : success("(no active goal to block)");
-        }
-        if (sub.length === 0) {
-            const current = goals.get(agent);
-            return current ? success(goalStatusText(current)) : success("(no evolution goal — /evolve goal <objective> to create one)");
-        }
-        const objective = rest.join(" ");
-        const view = upsertEvolutionGoal(ctx, agent, objective);
-        return success(`evolution goal ready: ${goalStatusText(view)}\n(active goal drives the review gate every round)`);
-    }
-    catch (cause) {
-        return error(cause instanceof Error ? cause.message : String(cause));
-    }
-}
-async function executeMountCommand(ctx, engine, invocation, rest) {
-    const sub = rest[0] ?? "";
-    if (sub === "list") {
-        const ledger = loadLedger(engine.baseDir);
-        if (ledger.mounted.length === 0) {
-            return success("(no hot-mounted plugins — /evolve mount <skillId>)");
-        }
-        return success(ledger.mounted.map((m) => `- ${m.id} (${m.entryId}, v${m.version}, ${m.mountedAt})`).join("\n"));
-    }
-    const skillId = stripAngleBrackets(sub);
-    if (!skillId) {
-        return error(`mount requires a skill entry id.\nUsage: /evolve mount <skillId> | /evolve mount list`);
-    }
-    const sessionId = invocation.agent.id;
-    const local = engine.load("local", sessionId);
-    const globalState = engine.load("global", undefined);
-    const entry = local.entries.skill[skillId] ??
-        globalState.entries.skill[skillId] ??
-        Object.values(local.entries.skill).find((e) => e.id === skillId) ??
-        Object.values(globalState.entries.skill).find((e) => e.id === skillId);
-    if (!entry) {
-        return error(`skill entry ${skillId} not found in local or global store`);
-    }
-    try {
-        const record = await mountSkill(ctx, engine.baseDir, entry);
-        return success(`mounted ${record.id} as ${record.entryId} (v${record.version}) — tool: skill_${record.id.replace(/_/g, "-")}`);
-    }
-    catch (cause) {
-        return error(cause instanceof Error ? cause.message : String(cause));
-    }
-}
-async function executeBenchmarkCommand(ctx, engine, invocation, rest, runtime) {
-    const sub = rest[0] ?? "";
-    const args = rest.slice(1);
-    const sessionId = invocation.agent.id;
-    const baseDir = engine.baseDir;
-    switch (sub) {
-        case "":
-        case "help":
-            return success(BENCHMARK_USAGE);
-        case "new": {
-            const title = args[0] ?? "";
-            if (!title) {
-                return error(`benchmark new requires a title.\n${BENCHMARK_USAGE}`);
-            }
-            const runs = args[1] !== undefined ? parsePositiveInt(args[1], "runs") : undefined;
-            const definition = createBenchmark(baseDir, { title, ...(runs !== undefined ? { runs } : {}) });
-            return success(`benchmark ${definition.id} created (runs=${definition.runs}, passThreshold=${definition.passThreshold})\nAdd cases with: /evolve benchmark add-case ${definition.id} "<title>" "<statement>" "<rubric>"`);
-        }
-        case "list": {
-            const benchmarks = listBenchmarks(baseDir);
-            if (benchmarks.length === 0) {
-                return success("(no benchmarks yet — use /evolve benchmark new <title>)");
-            }
-            const lines = benchmarks.map((b) => {
-                const cases = listCases(baseDir, b.id);
-                const board = loadScoreboard(baseDir, b.id);
-                const ref = board.reference ? ` ref=${board.reference.overall ?? "?"}` : " no-reference";
-                return `- ${b.id} (${cases.length} cases, runs=${b.runs})${ref}`;
-            });
-            return success(lines.join("\n"));
-        }
-        case "add-case": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const title = args[1] ?? "";
-            const statement = args[2] ?? "";
-            const rubric = args[3] ?? "";
-            if (!bid || !title || !statement || !rubric) {
-                return error(`benchmark add-case needs <bid> <title> <statement> <rubric>.\n${BENCHMARK_USAGE}`);
-            }
-            const caseItem = addCase(baseDir, bid, title, statement, rubric, runtime.rubricKey);
-            return success(`case ${caseItem.id} added to ${bid}`);
-        }
-        case "reset": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            if (!bid) {
-                return error(`benchmark reset needs a <bid>.\n${BENCHMARK_USAGE}`);
-            }
-            if (!loadBenchmark(baseDir, bid)) {
-                return error(`benchmark ${bid} not found`);
-            }
-            saveScoreboard(baseDir, bid, { candidates: [], decisions: [] });
-            return success(`scoreboard reset for ${bid} — run /evolve benchmark run ${bid} to record a fresh reference`);
-        }
-        case "status": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const board = loadScoreboard(baseDir, bid);
-            const lines = [];
-            if (board.reference) {
-                lines.push(`reference "${board.reference.label}": overall=${board.reference.overall ?? "?"} cells=${board.reference.cells.length}`);
-            }
-            else {
-                lines.push("(no reference evaluation yet)");
-            }
-            for (const c of board.candidates) {
-                lines.push(`candidate "${c.label}": overall=${c.overall ?? "?"} cells=${c.cells.length}${c.refinementId ? ` (${c.refinementId})` : ""}`);
-            }
-            for (const d of board.decisions) {
-                lines.push(`decision: ${d.accepted ? "ACCEPTED" : "rejected"} ${d.candidateLabel} — ${d.reasons.join("; ") || "ok"}`);
-            }
-            return success(lines.join("\n") || "(empty scoreboard)");
-        }
-        case "run": {
-            const bid = stripAngleBrackets(args[0] ?? "");
-            const candidateId = args.includes("candidate") ? stripAngleBrackets(args[args.indexOf("candidate") + 1] ?? "") : undefined;
-            const definition = loadBenchmark(baseDir, bid);
-            if (!definition) {
-                return error(`benchmark ${bid} not found`);
-            }
-            const cases = listCases(baseDir, bid);
-            if (cases.length === 0) {
-                return error(`benchmark ${bid} has no cases — use /evolve benchmark add-case`);
-            }
-            const board = loadScoreboard(baseDir, bid);
-            const label = candidateId ? `candidate:${candidateId}` : "reference";
-            if (!candidateId && board.reference) {
-                return error(`reference already evaluated (${board.reference.overall ?? "?"}); evaluate a candidate instead: /evolve benchmark run ${bid} candidate <refinementId>`);
-            }
-            const overview = formatHarnessStateForPrompt(engine.load("local", sessionId));
-            const outcome = await evaluateState(ctx, invocation.agent, {
-                cases,
-                rubricKey: runtime.rubricKey,
-                runs: definition.runs,
-                passThreshold: definition.passThreshold,
-                harnessOverview: overview,
-                label,
-                signal: invocation.signal,
-            });
-            const entry = entryFromCells(label, outcome.cells, candidateId);
-            const lines = [
-                `evaluation "${label}": ${outcome.cells.length} cells, overall=${entry.overall ?? "?"}`,
-                ...Object.entries(entry.aggregate)
-                    .filter(([key]) => key !== "overall")
-                    .map(([key, value]) => `  ${key}: ${value ?? "?"}`),
-            ];
-            if (candidateId) {
-                if (!board.reference) {
-                    lines.push("(no reference yet — this run only recorded the candidate)");
-                    board.candidates.push(entry);
-                }
-                else {
-                    const decision = decide(board.reference, entry, { passThreshold: definition.passThreshold, regressionTolerance: 0 });
-                    board.candidates.push(entry);
-                    board.decisions.push({
-                        candidateLabel: label,
-                        refinementId: candidateId,
-                        accepted: decision.accepted,
-                        reasons: decision.reasons,
-                        createdAt: new Date().toISOString(),
-                    });
-                    lines.push(...decisionReport(board.reference, entry, decision));
-                    if (!decision.accepted) {
-                        lines.push(`Consider rolling back the candidate: /evolve rollback <${candidateId}>`);
-                        if (runtime.autoRollbackOnReject) {
-                            const outcome = rollbackRejectedCandidate(engine, sessionId, candidateId);
-                            lines.push(outcome.message);
-                        }
-                    }
-                }
-            }
-            else {
-                board.reference = entry;
-                lines.push("reference evaluation recorded as the baseline");
-            }
-            saveScoreboard(baseDir, bid, board);
-            return success(lines.join("\n"));
-        }
-        default:
-            return error(`unknown benchmark subcommand: ${sub}\n${BENCHMARK_USAGE}`);
-    }
-}
-const BENCHMARK_USAGE = `Usage:
-  /evolve benchmark new <title>                          create a benchmark (runs=1)
-  /evolve benchmark add-case <bid> <title> <statement> <rubric>
-  /evolve benchmark list                                 list benchmarks + reference status
-  /evolve benchmark status <bid>                         show scoreboard + decisions
-  /evolve benchmark reset <bid>                          clear the scoreboard (fresh reference)
-  /evolve benchmark run <bid>                            evaluate current state as the reference
-  /evolve benchmark run <bid> candidate <refinementId>   evaluate the post-refinement state and decide`;
 function renderResult(result) {
     const applied = result.appliedEdits.filter((e) => e.applied);
     const failed = result.appliedEdits.filter((e) => !e.applied);

package/lib/evaluate.d.ts

@@ -1,14 +1,34 @@
 /**
- * The evaluation matrix runner: executes every case × run as a fresh
- * structured-output subagent, with the provider/model frozen to the calling
- * agent's own route. Raw per-cell scores come back to the host; aggregation
- * and acceptance happen in code (`src/score.ts`).
+ * The evaluation matrix runner: executes every case × run as a TWO-STAGE
+ * structured-output subagent pair, with the provider/model frozen to the
+ * calling agent's own route.
+ *
+ * Stage 1 — executor: a fresh child performs the case task with its tools
+ * and records CONCRETE EVIDENCE of what it did and found. It NEVER sees the
+ * rubric (gap A1, evaluator/scorer separation): the agent under test cannot
+ * optimize its behavior toward the grading criteria, and cannot grade its
+ * own execution.
+ *
+ * Stage 2 — reviewer: an INDEPENDENT child grades the executor's evidence
+ * strictly against the rubric. The rubric plaintext is decrypted in the host
+ * and flows ONLY into this reviewer branch; the executor branch never
+ * touches it. A separate model instance grading someone else's evidence
+ * removes the self-serving bias of self-scoring.
+ *
+ * Failure-cell protocol (gap A2): a unit that cannot produce a score
+ * (decrypt error, executor crash, reviewer crash, protocol error) returns a
+ * cell with `status: "failed"` — NEVER a zero — so aggregation can exclude
+ * it and the acceptance rule can reject rounds with too many failures instead
+ * of silently averaging a 0 into the mean.
+ *
+ * Each cell carries the executor child's session id (gap A4): the score can
+ * be drilled back to the exact session transcript that produced the evidence.
  *
  * Uses the host-plane `subagents` service (available in every profile) with
  * the native `outputSchema` structured-output seam: the provider validates
- * the child's reply against the cell schema, so the host never parses
- * model text for evaluations. (The workflow engine was rejected because the
- * web profile keeps it in a per-agent isolated realm a host plugin cannot
+ * each child's reply against its cell schema, so the host never parses model
+ * text for evaluations. (The workflow engine was rejected because the web
+ * profile keeps it in a per-agent isolated realm a host plugin cannot
  * resolve.)
  */
 import type { Context } from "@deepseek-ai/cordis";
@@ -32,7 +52,23 @@ export interface EvaluationOutcome {
 }
 /** How many evaluation units may run concurrently (bounded subagent fan-out). */
 export declare const DEFAULT_EVALUATION_CONCURRENCY = 4;
+/** Evidence handed to the reviewer is capped so the grading call stays bounded. */
+export declare const MAX_EVIDENCE_CHARS = 8000;
 export declare function evaluateState(ctx: Context, agent: Agent, options: EvaluateOptions): Promise<EvaluationOutcome>;
+export interface ExecutorResult {
+    caseId: string;
+    run: number;
+    evidence: string;
+}
+/**
+ * Deterministic hash of a case's statement + rubric envelope (gap A3):
+ * a 16-char SHA-256 prefix, hex-encoded. Used to detect material changes
+ * between reference and candidate evaluation runs (see
+ * `score.flagMaterialDrift`).
+ */
+export declare function caseHash(caseItem: BenchmarkCase): string;
+/** Validate a provider-validated executor result; returns undefined when malformed. */
+export declare function normalizeExecutor(value: unknown, caseId: string, run: number): ExecutorResult | undefined;
 /** Validate a provider-validated structured cell; returns undefined when malformed. */
 export declare function normalizeCell(value: unknown, caseId: string, run: number, passThreshold: number): CellScore | undefined;
 //# sourceMappingURL=evaluate.d.ts.map

package/lib/evaluate.js

@@ -1,18 +1,47 @@
+import { createHash } from "node:crypto";
 import { mapPool } from "./pool.js";
 import { decryptRubric, deriveKey, DEV_RUBRIC_KEY } from "./rubric.js";
 /** Key used when the caller did not resolve one: mirrors resolveRubricKey's last-resort dev fallback. */
 function devRubricKey() {
     return deriveKey(DEV_RUBRIC_KEY);
 }
-const EVAL_SYSTEM_PROMPT = `You are one evaluation unit in a benchmark matrix.
+/** Stage 1: the agent under test — sees the task, never the rubric. */
+const EXECUTOR_SYSTEM_PROMPT = `You are one evaluation unit in a benchmark matrix.
 
-You are the agent under evaluation. Perform the case task using your tools,
-then score your own execution strictly against the rubric. The harness
-guidance attached to the state under test is included in the task.
+You are the agent under evaluation. Perform the case task using your tools.
+You are NOT asked to grade yourself: a separate evaluator will judge your
+work against criteria you do not see. Instead, record CONCRETE, VERIFIABLE
+EVIDENCE of what you did and found — the actual commands/reads you performed,
+what the harness state contains, the exact text you produced. Evidence is
+what your work will be scored from; vague self-assessment earns nothing.
 
-Your reply is structured (see the requested output schema): caseId, run,
-score (0-100), passed (true iff score >= the stated threshold), and notes
-(concrete evidence for the score).`;
+The harness guidance attached to the state under test is included in the task.
+Your reply is structured (see the requested output schema): caseId, run, and
+evidence (the concrete artifact/report of your execution).`;
+/** Stage 2: the independent grader — sees the rubric, never executes the task. */
+const REVIEWER_SYSTEM_PROMPT = `You are an independent evaluator in a benchmark matrix.
+
+You did NOT perform the task and you cannot interact with the runtime: grade
+strictly from the EVIDENCE produced by the agent under evaluation, against
+the rubric. The harness guidance under test is included for context so you
+can judge whether the evidence genuinely reflects the state under test
+(e.g. whether the agent actually inspected the harness store).
+
+Score YOUR JUDGMENT of the evidence against the rubric criteria: a
+0-100 score, and passed (true iff score >= the stated threshold). Cite in
+notes the concrete rubric criteria and the evidence that supports the score.
+Do not inflate: grade the evidence as presented, not what the agent might
+have meant.`;
+const EXECUTOR_SCHEMA = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        caseId: { type: "string" },
+        run: { type: "number" },
+        evidence: { type: "string" },
+    },
+    required: ["caseId", "run", "evidence"],
+};
 const CELL_SCHEMA = {
     type: "object",
     additionalProperties: false,
@@ -27,6 +56,8 @@ const CELL_SCHEMA = {
 };
 /** How many evaluation units may run concurrently (bounded subagent fan-out). */
 export const DEFAULT_EVALUATION_CONCURRENCY = 4;
+/** Evidence handed to the reviewer is capped so the grading call stays bounded. */
+export const MAX_EVIDENCE_CHARS = 8000;
 export async function evaluateState(ctx, agent, options) {
     if (!agent.options.provider || !agent.options.model) {
         throw new Error("evolve: benchmark evaluation requires a provider/model route");
@@ -44,65 +75,162 @@ export async function evaluateState(ctx, agent, options) {
     const cells = await mapPool(units, DEFAULT_EVALUATION_CONCURRENCY, (unit) => runUnit(subagents, agent, options, unit.case, unit.run));
     return { label: options.label, cells, stopReason: "completed" };
 }
+/** A cell that could not be produced: never a zero, excluded by aggregation. */
+function failedCell(caseId, run, message) {
+    return { caseId, run, status: "failed", score: 0, passed: false, notes: message };
+}
+/**
+ * Deterministic hash of a case's statement + rubric envelope (gap A3):
+ * a 16-char SHA-256 prefix, hex-encoded. Used to detect material changes
+ * between reference and candidate evaluation runs (see
+ * `score.flagMaterialDrift`).
+ */
+export function caseHash(caseItem) {
+    const material = `${caseItem.statement}\n${caseItem.rubric}`;
+    return createHash("sha256").update(material).digest("hex").slice(0, 16);
+}
 async function runUnit(subagents, agent, options, c, run) {
+    // Gap C3: track wall-clock duration of the entire cell evaluation.
+    const cellStart = Date.now();
     // The ONLY rubric decryption point: the envelope is opened here, in the
-    // host, and the plaintext goes straight into the child prompt. The
-    // optimizer never reaches this path.
+    // host, and the plaintext goes ONLY into the reviewer prompt — the
+    // executor branch never touches it (gap A1).
     let rubric;
     try {
         rubric = decryptRubric(c.rubric, options.rubricKey ?? devRubricKey());
     }
     catch (cause) {
-        return {
-            caseId: c.id,
-            run,
-            score: 0,
-            passed: false,
-            notes: `rubric decrypt failed: ${cause instanceof Error ? cause.message : String(cause)}`,
-        };
+        return { ...failedCell(c.id, run, `rubric decrypt failed: ${cause instanceof Error ? cause.message : String(cause)}`), durationMs: Date.now() - cellStart };
     }
-    const prompt = [
-        EVAL_SYSTEM_PROMPT,
-        "---",
-        "Your harness guidance (state under test):",
-        `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
-        `Case ${c.id} — task (statement):\n${c.statement}`,
-        `Rubric — score yourself strictly against these criteria:\n${rubric}`,
-        `Run ${run} of ${options.runs}. passThreshold = ${options.passThreshold}.`,
-        "Execute the task with your tools, then produce the structured evaluation.",
-    ].join("\n\n");
+    // Runtime evidence (gap A3): record actual provider/model from the host,
+    // and compute a material hash of the case for change detection.
+    const actualProvider = agent.options.provider ?? "unknown";
+    const actualModel = agent.options.model ?? "unknown";
+    const hash = caseHash(c);
+    // Stage 1: executor — task + evidence, NO rubric.
+    let evidence;
+    let sessionId;
     try {
-        const runObj = await subagents.start("spawn", {
-            label: `${c.id} r${run}`,
-            prompt: [{ type: "text", text: prompt }],
+        const executorRun = await subagents.start("spawn", {
+            label: `${c.id} r${run} execute`,
+            prompt: [
+                {
+                    type: "text",
+                    text: [
+                        EXECUTOR_SYSTEM_PROMPT,
+                        "---",
+                        "Your harness guidance (state under test):",
+                        `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
+                        `Case ${c.id} — task (statement):\n${c.statement}`,
+                        `Run ${run} of ${options.runs}.`,
+                        "Execute the task with your tools, then produce the structured evidence.",
+                    ].join("\n\n"),
+                },
+            ],
+            parent: agent,
+            signal: options.signal ?? new AbortController().signal,
+            outputSchema: EXECUTOR_SCHEMA,
+        });
+        try {
+            const settled = await executorRun.result;
+            if (settled.stopReason !== "completed") {
+                throw new Error(`executor stopped: ${settled.stopReason ?? "unknown"}`);
+            }
+            const parsed = normalizeExecutor(settled.structured, c.id, run) ?? fromEvidenceText(settled.output, c.id, run);
+            if (!parsed) {
+                throw new Error("executor returned neither a structured value nor usable text");
+            }
+            evidence = parsed;
+            sessionId = executorRun.id || undefined;
+        }
+        finally {
+            executorRun.dispose();
+        }
+    }
+    catch (cause) {
+        return { ...failedCell(c.id, run, `executor failed: ${cause instanceof Error ? cause.message : String(cause)}`), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
+    }
+    // Stage 2: independent reviewer — rubric + evidence, NO task execution.
+    try {
+        const reviewerRun = await subagents.start("spawn", {
+            label: `${c.id} r${run} grade`,
+            prompt: [
+                {
+                    type: "text",
+                    text: [
+                        REVIEWER_SYSTEM_PROMPT,
+                        "---",
+                        "Your harness guidance (state under test):",
+                        `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
+                        `Case ${c.id} — task (statement):\n${c.statement}`,
+                        `Rubric — grade the evidence strictly against these criteria:\n${rubric}`,
+                        `Evidence produced by the agent under evaluation:\n<evidence>\n${trimEvidence(evidence.evidence)}\n</evidence>`,
+                        `Run ${run} of ${options.runs}. passThreshold = ${options.passThreshold}.`,
+                        "Produce the structured score.",
+                    ].join("\n\n"),
+                },
+            ],
             parent: agent,
             signal: options.signal ?? new AbortController().signal,
             outputSchema: CELL_SCHEMA,
         });
         try {
-            const settled = await runObj.result;
+            const settled = await reviewerRun.result;
             if (settled.stopReason !== "completed") {
-                throw new Error(`child stopped: ${settled.stopReason ?? "unknown"}`);
+                throw new Error(`reviewer stopped: ${settled.stopReason ?? "unknown"}`);
             }
-            const parsed = normalizeCell(settled.structured, c.id, run, options.passThreshold) ??
+            const cell = normalizeCell(settled.structured, c.id, run, options.passThreshold) ??
                 fromOutputText(settled.output, c.id, run, options.passThreshold);
-            if (!parsed) {
-                throw new Error("child returned neither a structured value nor usable text");
+            if (!cell) {
+                throw new Error("reviewer returned neither a structured value nor usable text");
             }
-            return parsed;
+            return { ...cell, ...(sessionId !== undefined ? { sessionId } : {}), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
         }
         finally {
-            runObj.dispose();
+            reviewerRun.dispose();
         }
     }
     catch (cause) {
-        return {
-            caseId: c.id,
-            run,
-            score: 0,
-            passed: false,
-            notes: `unit failed: ${cause instanceof Error ? cause.message : String(cause)}`,
-        };
+        return { ...failedCell(c.id, run, `reviewer failed: ${cause instanceof Error ? cause.message : String(cause)}`), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
+    }
+}
+/** Cap the evidence handed to the reviewer so the grading call stays bounded. */
+function trimEvidence(text) {
+    if (text.length <= MAX_EVIDENCE_CHARS)
+        return text;
+    return `${text.slice(0, MAX_EVIDENCE_CHARS)}\n…[evidence truncated at ${MAX_EVIDENCE_CHARS} chars]`;
+}
+/** Validate a provider-validated executor result; returns undefined when malformed. */
+export function normalizeExecutor(value, caseId, run) {
+    if (typeof value !== "object" || value === null || Array.isArray(value))
+        return undefined;
+    const record = value;
+    const evidence = typeof record["evidence"] === "string" ? record["evidence"] : "";
+    if (evidence.trim().length === 0)
+        return undefined;
+    return {
+        caseId: typeof record["caseId"] === "string" && record["caseId"].length > 0 ? record["caseId"] : caseId,
+        run: typeof record["run"] === "number" && Number.isFinite(record["run"]) ? Math.trunc(record["run"]) : run,
+        evidence,
+    };
+}
+/** Fallback: recover the executor result from its text blocks when no structured value arrived. */
+function fromEvidenceText(blocks, caseId, run) {
+    if (!Array.isArray(blocks))
+        return undefined;
+    const text = blocks
+        .filter((block) => block.type === "text")
+        .map((block) => block.text ?? "")
+        .join("\n");
+    const trimmed = text.trim();
+    if (trimmed.length === 0)
+        return undefined;
+    try {
+        return normalizeExecutor(JSON.parse(trimmed), caseId, run);
+    }
+    catch {
+        // Not JSON — keep the raw text as the evidence.
+        return { caseId, run, evidence: trimmed };
     }
 }
 /** Validate a provider-validated structured cell; returns undefined when malformed. */
@@ -116,6 +244,7 @@ export function normalizeCell(value, caseId, run, passThreshold) {
     return {
         caseId: typeof record["caseId"] === "string" && record["caseId"].length > 0 ? record["caseId"] : caseId,
         run: typeof record["run"] === "number" && Number.isFinite(record["run"]) ? Math.trunc(record["run"]) : run,
+        status: "ok",
         score: Math.min(100, Math.max(0, score)),
         passed: record["passed"] === true || score >= passThreshold,
         notes: typeof record["notes"] === "string" ? record["notes"] : "",