dsh-continual-evolve 0.1.0 → 0.2.0

package/lib/evaluate.js

@@ -4,15 +4,43 @@ import { decryptRubric, deriveKey, DEV_RUBRIC_KEY } from "./rubric.js";
 function devRubricKey() {
     return deriveKey(DEV_RUBRIC_KEY);
 }
-const EVAL_SYSTEM_PROMPT = `You are one evaluation unit in a benchmark matrix.
+/** Stage 1: the agent under test — sees the task, never the rubric. */
+const EXECUTOR_SYSTEM_PROMPT = `You are one evaluation unit in a benchmark matrix.
 
-You are the agent under evaluation. Perform the case task using your tools,
-then score your own execution strictly against the rubric. The harness
-guidance attached to the state under test is included in the task.
+You are the agent under evaluation. Perform the case task using your tools.
+You are NOT asked to grade yourself: a separate evaluator will judge your
+work against criteria you do not see. Instead, record CONCRETE, VERIFIABLE
+EVIDENCE of what you did and found — the actual commands/reads you performed,
+what the harness state contains, the exact text you produced. Evidence is
+what your work will be scored from; vague self-assessment earns nothing.
 
-Your reply is structured (see the requested output schema): caseId, run,
-score (0-100), passed (true iff score >= the stated threshold), and notes
-(concrete evidence for the score).`;
+The harness guidance attached to the state under test is included in the task.
+Your reply is structured (see the requested output schema): caseId, run, and
+evidence (the concrete artifact/report of your execution).`;
+/** Stage 2: the independent grader — sees the rubric, never executes the task. */
+const REVIEWER_SYSTEM_PROMPT = `You are an independent evaluator in a benchmark matrix.
+
+You did NOT perform the task and you cannot interact with the runtime: grade
+strictly from the EVIDENCE produced by the agent under evaluation, against
+the rubric. The harness guidance under test is included for context so you
+can judge whether the evidence genuinely reflects the state under test
+(e.g. whether the agent actually inspected the harness store).
+
+Score YOUR JUDGMENT of the evidence against the rubric criteria: a
+0-100 score, and passed (true iff score >= the stated threshold). Cite in
+notes the concrete rubric criteria and the evidence that supports the score.
+Do not inflate: grade the evidence as presented, not what the agent might
+have meant.`;
+const EXECUTOR_SCHEMA = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        caseId: { type: "string" },
+        run: { type: "number" },
+        evidence: { type: "string" },
+    },
+    required: ["caseId", "run", "evidence"],
+};
 const CELL_SCHEMA = {
     type: "object",
     additionalProperties: false,
@@ -27,6 +55,8 @@ const CELL_SCHEMA = {
 };
 /** How many evaluation units may run concurrently (bounded subagent fan-out). */
 export const DEFAULT_EVALUATION_CONCURRENCY = 4;
+/** Evidence handed to the reviewer is capped so the grading call stays bounded. */
+export const MAX_EVIDENCE_CHARS = 8000;
 export async function evaluateState(ctx, agent, options) {
     if (!agent.options.provider || !agent.options.model) {
         throw new Error("evolve: benchmark evaluation requires a provider/model route");
@@ -44,65 +74,148 @@ export async function evaluateState(ctx, agent, options) {
     const cells = await mapPool(units, DEFAULT_EVALUATION_CONCURRENCY, (unit) => runUnit(subagents, agent, options, unit.case, unit.run));
     return { label: options.label, cells, stopReason: "completed" };
 }
+/** A cell that could not be produced: never a zero, excluded by aggregation. */
+function failedCell(caseId, run, message) {
+    return { caseId, run, status: "failed", score: 0, passed: false, notes: message };
+}
 async function runUnit(subagents, agent, options, c, run) {
     // The ONLY rubric decryption point: the envelope is opened here, in the
-    // host, and the plaintext goes straight into the child prompt. The
-    // optimizer never reaches this path.
+    // host, and the plaintext goes ONLY into the reviewer prompt — the
+    // executor branch never touches it (gap A1).
     let rubric;
     try {
         rubric = decryptRubric(c.rubric, options.rubricKey ?? devRubricKey());
     }
     catch (cause) {
-        return {
-            caseId: c.id,
-            run,
-            score: 0,
-            passed: false,
-            notes: `rubric decrypt failed: ${cause instanceof Error ? cause.message : String(cause)}`,
-        };
+        return failedCell(c.id, run, `rubric decrypt failed: ${cause instanceof Error ? cause.message : String(cause)}`);
+    }
+    // Stage 1: executor — task + evidence, NO rubric.
+    let evidence;
+    let sessionId;
+    try {
+        const executorRun = await subagents.start("spawn", {
+            label: `${c.id} r${run} execute`,
+            prompt: [
+                {
+                    type: "text",
+                    text: [
+                        EXECUTOR_SYSTEM_PROMPT,
+                        "---",
+                        "Your harness guidance (state under test):",
+                        `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
+                        `Case ${c.id} — task (statement):\n${c.statement}`,
+                        `Run ${run} of ${options.runs}.`,
+                        "Execute the task with your tools, then produce the structured evidence.",
+                    ].join("\n\n"),
+                },
+            ],
+            parent: agent,
+            signal: options.signal ?? new AbortController().signal,
+            outputSchema: EXECUTOR_SCHEMA,
+        });
+        try {
+            const settled = await executorRun.result;
+            if (settled.stopReason !== "completed") {
+                throw new Error(`executor stopped: ${settled.stopReason ?? "unknown"}`);
+            }
+            const parsed = normalizeExecutor(settled.structured, c.id, run) ?? fromEvidenceText(settled.output, c.id, run);
+            if (!parsed) {
+                throw new Error("executor returned neither a structured value nor usable text");
+            }
+            evidence = parsed;
+            sessionId = executorRun.id || undefined;
+        }
+        finally {
+            executorRun.dispose();
+        }
+    }
+    catch (cause) {
+        return failedCell(c.id, run, `executor failed: ${cause instanceof Error ? cause.message : String(cause)}`);
     }
-    const prompt = [
-        EVAL_SYSTEM_PROMPT,
-        "---",
-        "Your harness guidance (state under test):",
-        `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
-        `Case ${c.id} — task (statement):\n${c.statement}`,
-        `Rubric — score yourself strictly against these criteria:\n${rubric}`,
-        `Run ${run} of ${options.runs}. passThreshold = ${options.passThreshold}.`,
-        "Execute the task with your tools, then produce the structured evaluation.",
-    ].join("\n\n");
+    // Stage 2: independent reviewer — rubric + evidence, NO task execution.
     try {
-        const runObj = await subagents.start("spawn", {
-            label: `${c.id} r${run}`,
-            prompt: [{ type: "text", text: prompt }],
+        const reviewerRun = await subagents.start("spawn", {
+            label: `${c.id} r${run} grade`,
+            prompt: [
+                {
+                    type: "text",
+                    text: [
+                        REVIEWER_SYSTEM_PROMPT,
+                        "---",
+                        "Your harness guidance (state under test):",
+                        `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
+                        `Case ${c.id} — task (statement):\n${c.statement}`,
+                        `Rubric — grade the evidence strictly against these criteria:\n${rubric}`,
+                        `Evidence produced by the agent under evaluation:\n<evidence>\n${trimEvidence(evidence.evidence)}\n</evidence>`,
+                        `Run ${run} of ${options.runs}. passThreshold = ${options.passThreshold}.`,
+                        "Produce the structured score.",
+                    ].join("\n\n"),
+                },
+            ],
             parent: agent,
             signal: options.signal ?? new AbortController().signal,
             outputSchema: CELL_SCHEMA,
         });
         try {
-            const settled = await runObj.result;
+            const settled = await reviewerRun.result;
             if (settled.stopReason !== "completed") {
-                throw new Error(`child stopped: ${settled.stopReason ?? "unknown"}`);
+                throw new Error(`reviewer stopped: ${settled.stopReason ?? "unknown"}`);
             }
-            const parsed = normalizeCell(settled.structured, c.id, run, options.passThreshold) ??
+            const cell = normalizeCell(settled.structured, c.id, run, options.passThreshold) ??
                 fromOutputText(settled.output, c.id, run, options.passThreshold);
-            if (!parsed) {
-                throw new Error("child returned neither a structured value nor usable text");
+            if (!cell) {
+                throw new Error("reviewer returned neither a structured value nor usable text");
+            }
+            if (sessionId !== undefined) {
+                return { ...cell, sessionId };
             }
-            return parsed;
+            return cell;
         }
         finally {
-            runObj.dispose();
+            reviewerRun.dispose();
         }
     }
     catch (cause) {
-        return {
-            caseId: c.id,
-            run,
-            score: 0,
-            passed: false,
-            notes: `unit failed: ${cause instanceof Error ? cause.message : String(cause)}`,
-        };
+        return failedCell(c.id, run, `reviewer failed: ${cause instanceof Error ? cause.message : String(cause)}`);
+    }
+}
+/** Cap the evidence handed to the reviewer so the grading call stays bounded. */
+function trimEvidence(text) {
+    if (text.length <= MAX_EVIDENCE_CHARS)
+        return text;
+    return `${text.slice(0, MAX_EVIDENCE_CHARS)}\n…[evidence truncated at ${MAX_EVIDENCE_CHARS} chars]`;
+}
+/** Validate a provider-validated executor result; returns undefined when malformed. */
+export function normalizeExecutor(value, caseId, run) {
+    if (typeof value !== "object" || value === null || Array.isArray(value))
+        return undefined;
+    const record = value;
+    const evidence = typeof record["evidence"] === "string" ? record["evidence"] : "";
+    if (evidence.trim().length === 0)
+        return undefined;
+    return {
+        caseId: typeof record["caseId"] === "string" && record["caseId"].length > 0 ? record["caseId"] : caseId,
+        run: typeof record["run"] === "number" && Number.isFinite(record["run"]) ? Math.trunc(record["run"]) : run,
+        evidence,
+    };
+}
+/** Fallback: recover the executor result from its text blocks when no structured value arrived. */
+function fromEvidenceText(blocks, caseId, run) {
+    if (!Array.isArray(blocks))
+        return undefined;
+    const text = blocks
+        .filter((block) => block.type === "text")
+        .map((block) => block.text ?? "")
+        .join("\n");
+    const trimmed = text.trim();
+    if (trimmed.length === 0)
+        return undefined;
+    try {
+        return normalizeExecutor(JSON.parse(trimmed), caseId, run);
+    }
+    catch {
+        // Not JSON — keep the raw text as the evidence.
+        return { caseId, run, evidence: trimmed };
     }
 }
 /** Validate a provider-validated structured cell; returns undefined when malformed. */
@@ -116,6 +229,7 @@ export function normalizeCell(value, caseId, run, passThreshold) {
     return {
         caseId: typeof record["caseId"] === "string" && record["caseId"].length > 0 ? record["caseId"] : caseId,
         run: typeof record["run"] === "number" && Number.isFinite(record["run"]) ? Math.trunc(record["run"]) : run,
+        status: "ok",
         score: Math.min(100, Math.max(0, score)),
         passed: record["passed"] === true || score >= passThreshold,
         notes: typeof record["notes"] === "string" ? record["notes"] : "",

package/lib/fate.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Gate local-fate dimension (#11 P2): the automatic review gate also decides
+ * what happens to a session's local entries while the session is STILL
+ * running — not only at wrap-up.
+ *
+ * Local entries default to orphans: a later session (not on the parentSession
+ * chain) never sees them, and nothing promotes or archives them — the
+ * exploration results effectively "die" with the session. This module gives
+ * the gate its own fate cadence on top of the existing review:
+ *
+ * - `promote` → the entry moves into the global store. The global store is a
+ *   governed resource: the user is consulted FIRST (the consultSkillEdits
+ *   pattern — never written silently), with a cooldown after a decline.
+ * - `archive` → hidden from injection (data stays restorable). Archives that
+ *   would bury possibly-reusable content (not covered globally AND distilled
+ *   from real user messages) ask the user first; covered/operational entries
+ *   archive silently, exactly like the wrap-up command does.
+ * - `keep` → nothing.
+ *
+ * At compaction the gate NEVER opens a dialog (the agent is mid-compaction):
+ * only deterministic silent archives apply, everything governed is deferred
+ * with an audit record pointing at `/evolve wrapup`.
+ *
+ * Division of labor is the same as wrap-up: the mechanical audit proposes
+ * (listLocalCandidates + coverage guards), the LLM classifies
+ * (assessLocalEntries), the user approves, the code applies deterministically
+ * (wholePromoteProposals / splitPromoteProposals — the SAME edits the wrap-up
+ * command applies). Every decision lands in reviews.jsonl via the gate's
+ * record callback.
+ */
+import type { Context } from "@deepseek-ai/cordis";
+import type { Agent } from "@deepseek-ai/dsh-agent";
+import type { HarnessState, RefinementResult } from "./types.js";
+import type { EvolutionEngine } from "./service.js";
+import type { AutoRefineReason } from "./review.js";
+import type { AutoReviewConfig, GateState, ReviewRecord } from "./auto.js";
+import { type WrapupCandidate, type WrapupItem } from "./wrapup.js";
+/** Turns a declined local-fate proposal stays silent before being offered again. */
+export declare const FATE_CONSULT_COOLDOWN_TURNS = 10;
+/** What the gate decided to do with the session's local entries. */
+export interface FatePlan {
+    candidates: readonly WrapupCandidate[];
+    /** Whole promotions that passed the deterministic global-coverage guard. */
+    promotable: WrapupItem[];
+    /** Split promotions (archive + cleaned promote payload) that passed the guard. */
+    splits: {
+        item: WrapupItem;
+        candidate: WrapupCandidate;
+    }[];
+    /** Archives that may proceed silently (covered globally / no real distillation source). */
+    silentArchives: WrapupItem[];
+    /** Archives that must ask the user first (uncovered + real source). */
+    reviewArchives: WrapupItem[];
+    /** Promotes blocked by the deterministic guard, with why. */
+    skipped: {
+        key: string;
+        reason: string;
+    }[];
+    /** Split promotions blocked by the deterministic guard, with why. */
+    splitSkipped: {
+        key: string;
+        reason: string;
+    }[];
+}
+/**
+ * Partition an assessed wrap-up classification into concrete fate actions,
+ * re-running the deterministic guards against the LIVE global store (state
+ * may have changed while the LLM call was in flight). Pure and unit-tested;
+ * mirrors the partition step of the wrap-up command.
+ */
+export declare function planLocalFates(items: readonly WrapupItem[], candidates: readonly WrapupCandidate[], globalState: HarnessState): FatePlan;
+/**
+ * The cooldown key of a candidate set: the sorted `kind:id` list. The set is
+ * the unit of consultation — a declined proposal is not offered again within
+ * the cooldown window, and a changed set (new entries appeared) starts a
+ * fresh consultation.
+ */
+export declare function fateSetKey(candidates: readonly WrapupCandidate[]): string;
+/**
+ * Whether the local-fate dimension is due for this gate run. Turn-interval
+ * gates respect the fate cadence (an independent counter — goal-driven
+ * sessions run the review EVERY round, the fate assessment must not);
+ * compaction is unconditional: experiences about to be summarized away get
+ * their fate check regardless.
+ */
+export declare function fateCadenceDue(state: GateState, reason: AutoRefineReason, intervalTurns: number): boolean;
+export interface FateConsultResult {
+    approved: boolean;
+    asked: boolean;
+    reason: "nothing-to-ask" | "consented" | "declined" | "cooldown" | "unavailable" | "error";
+}
+/**
+ * Ask the user whether to execute the gate's local-fate proposal. ONE dialog
+ * covers every governed action (promotes, split promotions, review-required
+ * archives) — the gate never spams questions. Conservative on every edge:
+ * no question service → not approved; the question call fails → not approved;
+ * the same candidate set was declined within the cooldown → not asked again.
+ * A decline records the cooldown (the consultSkillEdits pattern).
+ */
+export declare function consultLocalFates(ctx: Context, agent: Agent, plan: FatePlan, gate: GateState): Promise<FateConsultResult>;
+export type FateApplyMode = "full" | "silent-only";
+export interface FateApplyResult {
+    /** Human-readable lines of what was applied (for the notice and the audit record). */
+    applied: string[];
+    /** Every refinement result produced, for rollback discovery. */
+    results: RefinementResult[];
+}
+/**
+ * Deterministically apply the fate plan. `"full"` applies everything
+ * (promotes, splits, silent + review archives); `"silent-only"` applies only
+ * the deterministic silent archives (the compaction path — nothing governed).
+ * Promotes go through the SAME proposals as the wrap-up command
+ * (wholePromoteProposals / splitPromoteProposals), so both paths write
+ * identical global entries and local retirement stamps.
+ */
+export declare function applyLocalFates(engine: EvolutionEngine, sessionId: string, plan: FatePlan, localState: HarnessState, mode: FateApplyMode): FateApplyResult;
+/**
+ * The gate's local-fate phase. Runs after the review phase on every gate
+ * trigger (turn_interval / compact), subject to cadence and cooldown. All
+ * failures are contained and recorded — a broken fate dimension never
+ * disturbs the agent loop.
+ */
+export declare function runLocalFatePhase(ctx: Context, engine: EvolutionEngine, agent: Agent, config: AutoReviewConfig, state: GateState, reason: AutoRefineReason, record: (entry: Omit<ReviewRecord, "timestamp">) => void): Promise<void>;
+/** The user-visible notice after the gate applied local-fate actions. */
+export declare function buildFateNotice(applied: readonly string[]): string;
+//# sourceMappingURL=fate.d.ts.map