dsh-continual-evolve 0.11.0 → 0.12.0

package/lib/evaluate.js

@@ -1,271 +0,0 @@
-import { createHash } from "node:crypto";
-import { mapPool } from "./pool.js";
-import { decryptRubric, deriveKey, DEV_RUBRIC_KEY } from "./rubric.js";
-/** Key used when the caller did not resolve one: mirrors resolveRubricKey's last-resort dev fallback. */
-function devRubricKey() {
-    return deriveKey(DEV_RUBRIC_KEY);
-}
-/** Stage 1: the agent under test — sees the task, never the rubric. */
-const EXECUTOR_SYSTEM_PROMPT = `You are one evaluation unit in a benchmark matrix.
-
-You are the agent under evaluation. Perform the case task using your tools.
-You are NOT asked to grade yourself: a separate evaluator will judge your
-work against criteria you do not see. Instead, record CONCRETE, VERIFIABLE
-EVIDENCE of what you did and found — the actual commands/reads you performed,
-what the harness state contains, the exact text you produced. Evidence is
-what your work will be scored from; vague self-assessment earns nothing.
-
-The harness guidance attached to the state under test is included in the task.
-Your reply is structured (see the requested output schema): caseId, run, and
-evidence (the concrete artifact/report of your execution).`;
-/** Stage 2: the independent grader — sees the rubric, never executes the task. */
-const REVIEWER_SYSTEM_PROMPT = `You are an independent evaluator in a benchmark matrix.
-
-You did NOT perform the task and you cannot interact with the runtime: grade
-strictly from the EVIDENCE produced by the agent under evaluation, against
-the rubric. The harness guidance under test is included for context so you
-can judge whether the evidence genuinely reflects the state under test
-(e.g. whether the agent actually inspected the harness store).
-
-Score YOUR JUDGMENT of the evidence against the rubric criteria: a
-0-100 score, and passed (true iff score >= the stated threshold). Cite in
-notes the concrete rubric criteria and the evidence that supports the score.
-Do not inflate: grade the evidence as presented, not what the agent might
-have meant.`;
-const EXECUTOR_SCHEMA = {
-    type: "object",
-    additionalProperties: false,
-    properties: {
-        caseId: { type: "string" },
-        run: { type: "number" },
-        evidence: { type: "string" },
-    },
-    required: ["caseId", "run", "evidence"],
-};
-const CELL_SCHEMA = {
-    type: "object",
-    additionalProperties: false,
-    properties: {
-        caseId: { type: "string" },
-        run: { type: "number" },
-        score: { type: "number" },
-        passed: { type: "boolean" },
-        notes: { type: "string" },
-    },
-    required: ["caseId", "run", "score", "passed", "notes"],
-};
-/** How many evaluation units may run concurrently (bounded subagent fan-out). */
-export const DEFAULT_EVALUATION_CONCURRENCY = 4;
-/** Evidence handed to the reviewer is capped so the grading call stays bounded. */
-export const MAX_EVIDENCE_CHARS = 8000;
-export async function evaluateState(ctx, agent, options) {
-    if (!agent.options.provider || !agent.options.model) {
-        throw new Error("evolve: benchmark evaluation requires a provider/model route");
-    }
-    const subagents = ctx.subagents;
-    if (!subagents) {
-        throw new Error("evolve: benchmark evaluation requires the subagents service");
-    }
-    const units = [];
-    for (const c of options.cases) {
-        for (let run = 1; run <= options.runs; run += 1) {
-            units.push({ case: c, run });
-        }
-    }
-    const cells = await mapPool(units, DEFAULT_EVALUATION_CONCURRENCY, (unit) => runUnit(subagents, agent, options, unit.case, unit.run));
-    return { label: options.label, cells, stopReason: "completed" };
-}
-/** A cell that could not be produced: never a zero, excluded by aggregation. */
-function failedCell(caseId, run, message) {
-    return { caseId, run, status: "failed", score: 0, passed: false, notes: message };
-}
-/**
- * Deterministic hash of a case's statement + rubric envelope (gap A3):
- * a 16-char SHA-256 prefix, hex-encoded. Used to detect material changes
- * between reference and candidate evaluation runs (see
- * `score.flagMaterialDrift`).
- */
-export function caseHash(caseItem) {
-    const material = `${caseItem.statement}\n${caseItem.rubric}`;
-    return createHash("sha256").update(material).digest("hex").slice(0, 16);
-}
-async function runUnit(subagents, agent, options, c, run) {
-    // Gap C3: track wall-clock duration of the entire cell evaluation.
-    const cellStart = Date.now();
-    // The ONLY rubric decryption point: the envelope is opened here, in the
-    // host, and the plaintext goes ONLY into the reviewer prompt — the
-    // executor branch never touches it (gap A1).
-    let rubric;
-    try {
-        rubric = decryptRubric(c.rubric, options.rubricKey ?? devRubricKey());
-    }
-    catch (cause) {
-        return { ...failedCell(c.id, run, `rubric decrypt failed: ${cause instanceof Error ? cause.message : String(cause)}`), durationMs: Date.now() - cellStart };
-    }
-    // Runtime evidence (gap A3): record actual provider/model from the host,
-    // and compute a material hash of the case for change detection.
-    const actualProvider = agent.options.provider ?? "unknown";
-    const actualModel = agent.options.model ?? "unknown";
-    const hash = caseHash(c);
-    // Stage 1: executor — task + evidence, NO rubric.
-    let evidence;
-    let sessionId;
-    try {
-        const executorRun = await subagents.start("spawn", {
-            label: `${c.id} r${run} execute`,
-            prompt: [
-                {
-                    type: "text",
-                    text: [
-                        EXECUTOR_SYSTEM_PROMPT,
-                        "---",
-                        "Your harness guidance (state under test):",
-                        `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
-                        `Case ${c.id} — task (statement):\n${c.statement}`,
-                        `Run ${run} of ${options.runs}.`,
-                        "Execute the task with your tools, then produce the structured evidence.",
-                    ].join("\n\n"),
-                },
-            ],
-            parent: agent,
-            signal: options.signal ?? new AbortController().signal,
-            outputSchema: EXECUTOR_SCHEMA,
-        });
-        try {
-            const settled = await executorRun.result;
-            if (settled.stopReason !== "completed") {
-                throw new Error(`executor stopped: ${settled.stopReason ?? "unknown"}`);
-            }
-            const parsed = normalizeExecutor(settled.structured, c.id, run) ?? fromEvidenceText(settled.output, c.id, run);
-            if (!parsed) {
-                throw new Error("executor returned neither a structured value nor usable text");
-            }
-            evidence = parsed;
-            sessionId = executorRun.id || undefined;
-        }
-        finally {
-            executorRun.dispose();
-        }
-    }
-    catch (cause) {
-        return { ...failedCell(c.id, run, `executor failed: ${cause instanceof Error ? cause.message : String(cause)}`), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
-    }
-    // Stage 2: independent reviewer — rubric + evidence, NO task execution.
-    try {
-        const reviewerRun = await subagents.start("spawn", {
-            label: `${c.id} r${run} grade`,
-            prompt: [
-                {
-                    type: "text",
-                    text: [
-                        REVIEWER_SYSTEM_PROMPT,
-                        "---",
-                        "Your harness guidance (state under test):",
-                        `<harness_overview>\n${options.harnessOverview}\n</harness_overview>`,
-                        `Case ${c.id} — task (statement):\n${c.statement}`,
-                        `Rubric — grade the evidence strictly against these criteria:\n${rubric}`,
-                        `Evidence produced by the agent under evaluation:\n<evidence>\n${trimEvidence(evidence.evidence)}\n</evidence>`,
-                        `Run ${run} of ${options.runs}. passThreshold = ${options.passThreshold}.`,
-                        "Produce the structured score.",
-                    ].join("\n\n"),
-                },
-            ],
-            parent: agent,
-            signal: options.signal ?? new AbortController().signal,
-            outputSchema: CELL_SCHEMA,
-        });
-        try {
-            const settled = await reviewerRun.result;
-            if (settled.stopReason !== "completed") {
-                throw new Error(`reviewer stopped: ${settled.stopReason ?? "unknown"}`);
-            }
-            const cell = normalizeCell(settled.structured, c.id, run, options.passThreshold) ??
-                fromOutputText(settled.output, c.id, run, options.passThreshold);
-            if (!cell) {
-                throw new Error("reviewer returned neither a structured value nor usable text");
-            }
-            return { ...cell, ...(sessionId !== undefined ? { sessionId } : {}), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
-        }
-        finally {
-            reviewerRun.dispose();
-        }
-    }
-    catch (cause) {
-        return { ...failedCell(c.id, run, `reviewer failed: ${cause instanceof Error ? cause.message : String(cause)}`), provider: actualProvider, model: actualModel, caseHash: hash, durationMs: Date.now() - cellStart };
-    }
-}
-/** Cap the evidence handed to the reviewer so the grading call stays bounded. */
-function trimEvidence(text) {
-    if (text.length <= MAX_EVIDENCE_CHARS)
-        return text;
-    return `${text.slice(0, MAX_EVIDENCE_CHARS)}\n…[evidence truncated at ${MAX_EVIDENCE_CHARS} chars]`;
-}
-/** Validate a provider-validated executor result; returns undefined when malformed. */
-export function normalizeExecutor(value, caseId, run) {
-    if (typeof value !== "object" || value === null || Array.isArray(value))
-        return undefined;
-    const record = value;
-    const evidence = typeof record["evidence"] === "string" ? record["evidence"] : "";
-    if (evidence.trim().length === 0)
-        return undefined;
-    return {
-        caseId: typeof record["caseId"] === "string" && record["caseId"].length > 0 ? record["caseId"] : caseId,
-        run: typeof record["run"] === "number" && Number.isFinite(record["run"]) ? Math.trunc(record["run"]) : run,
-        evidence,
-    };
-}
-/** Fallback: recover the executor result from its text blocks when no structured value arrived. */
-function fromEvidenceText(blocks, caseId, run) {
-    if (!Array.isArray(blocks))
-        return undefined;
-    const text = blocks
-        .filter((block) => block.type === "text")
-        .map((block) => block.text ?? "")
-        .join("\n");
-    const trimmed = text.trim();
-    if (trimmed.length === 0)
-        return undefined;
-    try {
-        return normalizeExecutor(JSON.parse(trimmed), caseId, run);
-    }
-    catch {
-        // Not JSON — keep the raw text as the evidence.
-        return { caseId, run, evidence: trimmed };
-    }
-}
-/** Validate a provider-validated structured cell; returns undefined when malformed. */
-export function normalizeCell(value, caseId, run, passThreshold) {
-    if (typeof value !== "object" || value === null || Array.isArray(value))
-        return undefined;
-    const record = value;
-    const score = Number(record["score"]);
-    if (!Number.isFinite(score))
-        return undefined;
-    return {
-        caseId: typeof record["caseId"] === "string" && record["caseId"].length > 0 ? record["caseId"] : caseId,
-        run: typeof record["run"] === "number" && Number.isFinite(record["run"]) ? Math.trunc(record["run"]) : run,
-        status: "ok",
-        score: Math.min(100, Math.max(0, score)),
-        passed: record["passed"] === true || score >= passThreshold,
-        notes: typeof record["notes"] === "string" ? record["notes"] : "",
-    };
-}
-/** Fallback: recover a cell from the child's text blocks when no structured value arrived. */
-function fromOutputText(blocks, caseId, run, passThreshold) {
-    if (!Array.isArray(blocks))
-        return undefined;
-    const text = blocks
-        .filter((block) => block.type === "text")
-        .map((block) => block.text ?? "")
-        .join("\n");
-    if (text.length === 0)
-        return undefined;
-    try {
-        const value = JSON.parse(text);
-        return normalizeCell(value, caseId, run, passThreshold);
-    }
-    catch {
-        return undefined;
-    }
-}
-//# sourceMappingURL=evaluate.js.map

package/lib/evolve-event.d.ts

@@ -1,40 +0,0 @@
-import type { RefinementResult } from "./types.js";
-/** The structured event payload emitted after every successful refinement. */
-export interface EvolveCompleteEvent {
-    /** Event discriminator for consumers. */
-    type: "evolve_complete";
-    /** The refinement id (same as the result). */
-    refinementId: string;
-    /** One-line summary of the refinement. */
-    summary: string;
-    /** Number of edits that were actually applied. */
-    appliedEdits: number;
-    /** Number of edits that failed to apply. */
-    failedEdits: number;
-    /** Scope of the refinement ("local" or "global"). */
-    scope: string;
-    /** What triggered this refinement (e.g. "auto_review", "manual_plan", "manual_tool"). */
-    trigger: string;
-    /** Session id that owns the refinement (auto or manual). */
-    sessionId: string;
-    /** ISO timestamp. */
-    timestamp: string;
-    /** Per-edit summaries (kind + id + action) for consumers that want detail. */
-    edits: {
-        action: string;
-        kind: string;
-        id: string;
-        applied: boolean;
-    }[];
-}
-/** Build a structured evolve-complete event from a refinement result. */
-export declare function buildEvolveCompleteEvent(result: RefinementResult, trigger: string, sessionId: string): EvolveCompleteEvent;
-/**
- * Emit an evolve_complete event to the reviews.jsonl audit trail. The event
- * is JSONL-formatted (one line) so consumers can tail and parse it. This is
- * a best-effort write — failure never blocks the refinement path. Storage
- * hygiene (#20): the trail is truncated to its tail after the append so the
- * audit file cannot grow without bound; readers only need a recent window.
- */
-export declare function emitEvolveComplete(baseDir: string, event: EvolveCompleteEvent, reviewsRetain?: number): void;
-//# sourceMappingURL=evolve-event.d.ts.map

package/lib/evolve-event.js

@@ -1,60 +0,0 @@
-/**
- * Structured "evolve complete" events (gap C4): a durable, machine-readable
- * record emitted after every successful refinement application — whether
- * auto-gated or manual. Third-party consumers can observe these events via
- * the plugin log (JSONL) and the reviews.jsonl audit trail.
- *
- * Design: prime-agent `/refine` emits `refine_complete{id,summary,
- * appliedEdits,scope}` extension events. We follow the same pattern with
- * added provenance (trigger, source) so consumers know WHY the refinement
- * happened and WHO initiated it.
- */
-import { appendFileSync, mkdirSync } from "node:fs";
-import { join } from "node:path";
-import { DEFAULT_REVIEWS_RETAIN } from "./types.js";
-import { pruneJsonlFile, reviewsPath } from "./store.js";
-/** Build a structured evolve-complete event from a refinement result. */
-export function buildEvolveCompleteEvent(result, trigger, sessionId) {
-    return {
-        type: "evolve_complete",
-        refinementId: result.id,
-        summary: result.summary,
-        appliedEdits: result.appliedEdits.filter((e) => e.applied).length,
-        failedEdits: result.appliedEdits.filter((e) => !e.applied).length,
-        scope: result.scope ?? "local",
-        trigger,
-        sessionId,
-        timestamp: new Date().toISOString(),
-        edits: result.appliedEdits.map((e) => ({
-            action: e.action,
-            kind: e.kind,
-            id: e.id,
-            applied: e.applied,
-        })),
-    };
-}
-/**
- * Emit an evolve_complete event to the reviews.jsonl audit trail. The event
- * is JSONL-formatted (one line) so consumers can tail and parse it. This is
- * a best-effort write — failure never blocks the refinement path. Storage
- * hygiene (#20): the trail is truncated to its tail after the append so the
- * audit file cannot grow without bound; readers only need a recent window.
- */
-export function emitEvolveComplete(baseDir, event, reviewsRetain = DEFAULT_REVIEWS_RETAIN) {
-    try {
-        const dir = join(baseDir, "evolve");
-        mkdirSync(dir, { recursive: true });
-        appendFileSync(join(dir, "reviews.jsonl"), `${JSON.stringify(event)}\n`, "utf8");
-    }
-    catch {
-        // Event emission is diagnostic; never interrupt the refinement path.
-        return;
-    }
-    try {
-        pruneJsonlFile(reviewsPath(baseDir), reviewsRetain);
-    }
-    catch {
-        // ignored — the next emission retries
-    }
-}
-//# sourceMappingURL=evolve-event.js.map

package/lib/extraction-phase.d.ts

@@ -1,132 +0,0 @@
-/**
- * The dedicated memory extraction phase: one bounded ZCode-style loop over a
- * scheduler snapshot. Owns its request loop, frozen manifest, scope approval
- * bookkeeping, and the phase-specific memory checkpoint that advances after a
- * successful/no-op outcome — memory failures leave the boundary retryable.
- *
- * Triggered only by the low-frequency moments wired in `listener.ts`;
- * per-turn extraction was removed with the 2026-10-03 B verdict.
- */
-import type { Context } from "@deepseek-ai/cordis";
-import type { Agent } from "@deepseek-ai/dsh-agent";
-import type { EvolutionEngine } from "./service.js";
-import { type TurnSnapshot } from "./turn-snapshot.js";
-import type { HarnessState, ScopeApprovalDecision } from "./types.js";
-import type { PlannerPrefixCacheMode } from "./types.js";
-import { type RecordLanguagePreference } from "./record-language.js";
-export interface GateState {
-    turns: number;
-    /** Latest successful turn boundary seen by the listener. */
-    completedTurn: number;
-    /** Latest successful/no-op memory boundary. */
-    memoryCheckpoint?: string;
-    /** Per-boundary/scope approval decisions retained until the memory phase settles. */
-    memoryDecisions: Record<string, ScopeApprovalDecision>;
-    lastReviewAt: number;
-    /**
-     * Consecutive idle probes that observed the goal phase "blocked".
-     * Reset to 0 by any non-blocked probe and after a triggered moment.
-     */
-    goalBlockStreak: number;
-}
-export interface AutoReviewConfig {
-    /** Initial runtime state when no runtime.json exists. */
-    enabledByDefault?: boolean;
-    /** ZCode-style minimum lexical words in one direct user text part. */
-    memoryMinUserWords?: number;
-    /** Trajectory slice handed to the extractor, in characters. */
-    maxInputChars: number;
-    /** Output budget for the extraction loop. */
-    budgetTokens: number;
-    /** Queue a visible receipt follow-up after an applied goal-blocked moment. */
-    notifyOnAutoReview: boolean;
-    /**
-     * Optional model override for the extractor (cheaper model).
-     * Format: "provider/model" or just "model" (same provider as the agent).
-     * When absent, the extractor uses the agent's own provider/model.
-     */
-    reviewModel?: string;
-    /** Dedicated memory-agent writes to project/global require the normal human approval boundary. */
-    requireGlobalApproval?: boolean;
-    /**
-     * Authoring language for this run's records. Absent/`auto` resolves per
-     * call through the record-language chain (durable client preference,
-     * then trajectory detection, then `en`); `zh`/`en` pin it.
-     */
-    recordLanguage?: RecordLanguagePreference;
-    /**
-     * Prefix-cache routing for the extraction input. Absent mode → auto-detect;
-     * absent budget → the default prefix budget.
-     */
-    prefixCacheMode?: PlannerPrefixCacheMode;
-    prefixMaxChars?: number;
-    /**
-     * Goal-blocked trigger: after this many CONSECUTIVE idle probes that
-     * observe the session goal in phase "blocked", run one memory extraction
-     * so the blocked encounter is distilled. 0 disables. The streak resets on
-     * any non-blocked probe and after each triggered moment.
-     */
-    goalBlockedWrapupTurns: number;
-    /**
-     * Storage hygiene (#20): tail lines kept in the shared reviews.jsonl
-     * audit trail. Absent → the store default (readers need a recent
-     * window, not the full past).
-     */
-    reviewsRetain?: number;
-    /**
-     * Session-close bounded drain: how long a disposed session's in-flight
-     * extraction may settle before its owner signal aborts. Absent → 15s;
-     * 0 restores the legacy immediate abort.
-     */
-    sessionCloseDrainMs?: number;
-}
-export interface ReviewRecord {
-    timestamp: string;
-    sessionId: string;
-    reason: TurnSnapshot["reason"];
-    turnsSinceLastReview: number;
-    outcome: "approved" | "declined" | "failed" | "assessed" | "deferred" | "skipped" | "armed" | "applied" | "noop";
-    rationale?: string;
-    refinementId?: string;
-    /** P1 extraction stats (§6.7): wall time of the memory phase. */
-    durationMs?: number;
-    /** Extractor loop turns / manifest searches behind this row. */
-    memoryTurns?: number;
-    memorySearches?: number;
-    /** Applied edit count carried by an applied row. */
-    appliedEdits?: number;
-}
-export type ReviewRecorder = (entry: Omit<ReviewRecord, "timestamp">) => void;
-export declare function stateFor(map: Map<string, GateState>, sessionId: string): GateState;
-export declare function advanceMemoryCheckpoint(state: GateState, next: string): void;
-/**
- * The state view the extractor judges: the session's local entries merged
- * with the project and global stores, each entry carrying its real scope.
- * Without the global half the extractor cannot see that a topic is already
- * covered cross-session and happily re-sediments a local duplicate of it.
- *
- * The merged view is read-only context — applying still targets the raw
- * local state (baseline checks compare local entries only).
- */
-export declare function loadGateHarnessView(engine: EvolutionEngine, sessionId: string, opts?: {
-    projectKey?: string;
-}): HarnessState;
-/**
- * Gap C1: parse a "provider/model" or "model" string into its components.
- * Returns undefined when the input is empty (no override). A bare model name
- * falls back to the agent's provider, then "deepseek".
- * Exported for unit testing; production resolves it inside
- * runMemoryExtractionPhase.
- */
-export declare function parseReviewModel(reviewModel: string | undefined, fallbackProvider: string | undefined): {
-    provider: string;
-    model: string;
-} | undefined;
-/**
- * One extraction run over a scheduler snapshot. Records noop / applied /
- * declined audit rows with duration and loop stats, advances the memory
- * checkpoint on settled outcomes, and queues a visible receipt only for an
- * applied goal-blocked moment (compaction and session-close stay audit-only).
- */
-export declare function runMemoryExtractionPhase(ctx: Context, engine: EvolutionEngine, agent: Agent, config: AutoReviewConfig, state: GateState, snapshot: TurnSnapshot, record: ReviewRecorder, signal?: AbortSignal): Promise<void>;
-//# sourceMappingURL=extraction-phase.d.ts.map