dsh-continual-evolve 0.1.1 → 0.3.0

package/lib/score.js

@@ -1,8 +1,22 @@
-export const DEFAULT_AGGREGATE = { passThreshold: 60, regressionTolerance: 0 };
-/** Aggregate raw cells into code-owned per-case means + overall mean. */
+export const DEFAULT_AGGREGATE = {
+    passThreshold: 60,
+    regressionTolerance: 0,
+    maxFailedCells: 0,
+};
+/**
+ * Aggregate raw cells into code-owned per-case means + overall mean.
+ * Failure-cell protocol (gap A2): failed cells are EXCLUDED from means and
+ * counted separately — a crashed unit can never silently drag the mean down
+ * like a zero. A case whose cells all failed reports null (no mean).
+ */
 export function aggregate(cells) {
     const byCase = new Map();
+    let failed = 0;
     for (const cell of cells) {
+        if (cell.status === "failed") {
+            failed += 1;
+            continue;
+        }
         const list = byCase.get(cell.caseId) ?? [];
         list.push(clampScore(cell.score));
         byCase.set(cell.caseId, list);
@@ -12,7 +26,19 @@ export function aggregate(cells) {
         perCase[caseId] = mean(scores);
     }
     const all = [...byCase.values()].flat();
-    return { ...perCase, overall: all.length > 0 ? mean(all) : null };
+    let totalDurationMs = 0;
+    for (const cell of cells) {
+        if (cell.durationMs !== undefined && cell.durationMs >= 0) {
+            totalDurationMs += cell.durationMs;
+        }
+    }
+    return {
+        ...perCase,
+        overall: all.length > 0 ? mean(all) : null,
+        failed,
+        total: cells.length,
+        totalDurationMs,
+    };
 }
 export function entryFromCells(label, cells, refinementId) {
     const aggr = aggregate(cells);
@@ -28,33 +54,111 @@ export function entryFromCells(label, cells, refinementId) {
 /** Human-readable decision report with per-case before → after deltas. */
 export function decisionReport(reference, candidate, decision) {
     const lines = [`overall: ${reference.overall ?? "?"} → ${candidate.overall ?? "?"}`];
-    for (const [caseId, refScore] of Object.entries(reference.aggregate)) {
-        if (caseId === "overall" || refScore === null)
+    // Per-case deltas over the actual evaluated cases only (aggregate() mixes
+    // case means with metadata keys like totalDurationMs — never render those
+    // as cases).
+    for (const caseId of new Set(reference.cells.map((cell) => cell.caseId))) {
+        const refScore = reference.aggregate[caseId];
+        if (refScore === null || refScore === undefined)
             continue;
         const candScore = candidate.aggregate[caseId];
-        lines.push(`  ${caseId}: ${refScore} → ${candScore ?? "?"}`);
+        const failedMark = isCaseFailed(reference, caseId) || isCaseFailed(candidate, caseId) ? " (failed)" : "";
+        lines.push(`  ${caseId}: ${refScore} → ${candScore ?? "?"}${failedMark}`);
+    }
+    const refFailed = reference.aggregate.failed ?? 0;
+    const candFailed = candidate.aggregate.failed ?? 0;
+    if (refFailed > 0 || candFailed > 0) {
+        lines.push(`failed cells: reference ${refFailed}/${reference.aggregate.total ?? 0} · candidate ${candFailed}/${candidate.aggregate.total ?? 0}`);
+    }
+    // Gap C3: show duration summary when available.
+    const refDuration = reference.aggregate.totalDurationMs ?? 0;
+    const candDuration = candidate.aggregate.totalDurationMs ?? 0;
+    if (refDuration > 0 || candDuration > 0) {
+        lines.push(`duration: ${formatDuration(refDuration)} → ${formatDuration(candDuration)}`);
     }
     lines.push(decision.accepted
         ? "DECISION: ACCEPTED — overall improved, no regression"
         : `DECISION: REJECTED — ${decision.reasons.join("; ")}`);
     return lines;
 }
+/**
+ * Gap A3 (version_changed semantics): detect cells whose case material
+ * changed between the reference and candidate evaluation runs. A candidate
+ * cell whose `caseHash` differs from the reference cell of the SAME case
+ * means the statement/rubric was edited between the two runs — its score is
+ * not comparable to the baseline and must not count toward the decision.
+ *
+ * The check is conservative: cells without a hash on either side (pre-A3
+ * data) and cells already failed are left untouched. Mismatched cells are
+ * returned re-marked as `failed` with a reason in notes, so aggregation
+ * excludes them and the acceptance rule can reject the round.
+ */
+export function flagMaterialDrift(reference, candidateCells) {
+    const referenceHashes = new Map();
+    for (const cell of reference.cells) {
+        if (cell.status !== "failed" && cell.caseHash !== undefined && !referenceHashes.has(cell.caseId)) {
+            referenceHashes.set(cell.caseId, cell.caseHash);
+        }
+    }
+    if (referenceHashes.size === 0) {
+        return [...candidateCells];
+    }
+    return candidateCells.map((cell) => {
+        if (cell.status === "failed" || cell.caseHash === undefined) {
+            return cell;
+        }
+        const refHash = referenceHashes.get(cell.caseId);
+        if (refHash !== undefined && refHash !== cell.caseHash) {
+            return {
+                ...cell,
+                status: "failed",
+                passed: false,
+                notes: `materials changed: case ${cell.caseId} hash ${cell.caseHash} ≠ reference ${refHash} (re-run the reference or fix the material)`,
+            };
+        }
+        return cell;
+    });
+}
+function isCaseFailed(entry, caseId) {
+    return entry.cells.some((cell) => cell.caseId === caseId && cell.status === "failed");
+}
 /**
  * Non-regressive acceptance rule (Self-Harness style):
  * the candidate is accepted iff its overall mean is STRICTLY higher than the
- * reference AND no case regresses by more than `regressionTolerance` points.
+ * reference, no case regresses by more than `regressionTolerance` points,
+ * and neither side has more failed cells than `maxFailedCells` (failure-cell
+ * protocol, gap A2 — a partial/invalid round is never accepted).
  */
 export function decide(reference, candidate, opts) {
     const reasons = [];
     if (reference.overall === null || candidate.overall === null) {
         return { accepted: false, reasons: ["reference or candidate evaluation is incomplete"] };
     }
+    const refFailed = reference.aggregate.failed ?? 0;
+    const candFailed = candidate.aggregate.failed ?? 0;
+    if (refFailed > opts.maxFailedCells) {
+        reasons.push(`reference has ${refFailed} failed cells (max ${opts.maxFailedCells})`);
+    }
+    if (candFailed > opts.maxFailedCells) {
+        reasons.push(`candidate has ${candFailed} failed cells (max ${opts.maxFailedCells})`);
+    }
+    if (reasons.length > 0) {
+        return { accepted: false, reasons };
+    }
     if (candidate.overall <= reference.overall) {
         reasons.push(`overall not improved: ${candidate.overall} <= ${reference.overall}`);
     }
-    for (const [caseId, refScore] of Object.entries(reference.aggregate)) {
-        if (caseId === "overall" || refScore === null)
-            continue;
+    // Per-case regression is judged ONLY over the cases that actually exist in
+    // the evaluation — aggregate() mixes per-case means with metadata keys
+    // (failed/total/totalDurationMs), and iterating raw keys would (and did)
+    // treat totalDurationMs as a case score, rejecting a candidate whose run
+    // merely took longer. Derived from reference.cells, never from the
+    // aggregate key set.
+    for (const caseId of new Set(reference.cells.map((cell) => cell.caseId))) {
+        const refScore = reference.aggregate[caseId];
+        if (refScore === null || refScore === undefined) {
+            continue; // case had no comparable mean (all cells failed) — nothing to regress
+        }
         const candScore = candidate.aggregate[caseId];
         if (candScore === null || candScore === undefined) {
             reasons.push(`candidate missing case ${caseId}`);
@@ -78,4 +182,10 @@ function clampScore(score) {
 function round2(value) {
     return Math.round(value * 100) / 100;
 }
+/** Gap C3: human-readable duration (ms → "1.2s" or "340ms"). */
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${Math.round(ms)}ms`;
+    return `${(ms / 1000).toFixed(1)}s`;
+}
 //# sourceMappingURL=score.js.map

package/lib/service.d.ts

@@ -5,7 +5,6 @@
  */
 import type { EntrySource, HarnessScope, RefinementProposal, RefinementResult } from "./types.js";
 import { applyRefinementProposal } from "./apply.js";
-import { storePaths } from "./store.js";
 export interface ApplyContext {
     scope: HarnessScope;
     sessionId?: string;
@@ -13,6 +12,8 @@ export interface ApplyContext {
     baselineState?: Parameters<typeof applyRefinementProposal>[0];
     /** Trajectory citation stamped into newly created entries (see apply.ts). */
     source?: EntrySource | undefined;
+    /** Marks the resulting refinement as the deterministic rollback of another (audit chain). */
+    rollbackOf?: string | undefined;
 }
 export interface EvolutionHooks {
     /** Called after every applied refinement (side-effect boundary: skills sync, etc.). */
@@ -26,5 +27,4 @@ export declare function createEvolutionEngine(baseDir: string, hooks?: Evolution
     baseDir: string;
 };
 export type EvolutionEngine = ReturnType<typeof createEvolutionEngine>;
-export { storePaths };
 //# sourceMappingURL=service.d.ts.map

package/lib/service.js

@@ -17,6 +17,7 @@ export function createEvolutionEngine(baseDir, hooks = {}) {
             scope,
             ...(context?.source ? { source: context.source } : {}),
             ...(context?.baselineState ? { baselineState: context.baselineState } : {}),
+            ...(context?.rollbackOf ? { rollbackOf: context.rollbackOf } : {}),
         });
         saveHarnessState(paths.stateDir, state);
         appendResult(paths, result);
@@ -31,12 +32,14 @@ export function createEvolutionEngine(baseDir, hooks = {}) {
             throw new Error(`Refinement ${refinementId} not found in ${scope} history`);
         }
         const proposal = rollbackProposal(target);
-        return apply(scope, sessionId, proposal);
+        // The rollback refinement carries rollbackOf so the audit chain links
+        // the inverse operation back to its origin (previously the rollback
+        // record only echoed "Rollback refinement <id>" in its summary text).
+        return apply(scope, sessionId, proposal, { scope, rollbackOf: refinementId });
     }
     function history(scope, sessionId) {
         return loadResults(storePaths(baseDir, scope, sessionId));
     }
     return { load, apply, rollback, history, baseDir };
 }
-export { storePaths };
 //# sourceMappingURL=service.js.map

package/lib/skill-render.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Skill rendering: pure functions that convert harness entries into
+ * SKILL.md documents. Extracted from skill.ts to break the circular
+ * dependency between skill.ts ↔ skillquality.ts.
+ *
+ * Both skill.ts (materializer) and skillquality.ts (validator) need
+ * these rendering functions; importing from this shared leaf module
+ * keeps the dependency graph acyclic.
+ */
+import type { HarnessEntry } from "./types.js";
+/** Convert a harness entry id (underscore slug) to a kebab-case skill name. */
+export declare function skillNameOf(id: string): string;
+/** Render a harness skill entry as a discoverable SKILL.md document. */
+export declare function renderSkillMarkdown(entry: HarnessEntry): string;
+//# sourceMappingURL=skill-render.d.ts.map

package/lib/skill-render.js

@@ -0,0 +1,30 @@
+/** Convert a harness entry id (underscore slug) to a kebab-case skill name. */
+export function skillNameOf(id) {
+    return id.toLowerCase().replace(/_/g, "-");
+}
+/** Render a harness skill entry as a discoverable SKILL.md document. */
+export function renderSkillMarkdown(entry) {
+    const lines = [
+        "---",
+        `name: ${skillNameOf(entry.id)}`,
+        `description: ${oneLine(entry.title)}`,
+        "---",
+        "",
+        entry.content.trim(),
+    ];
+    const reference = entry.reference;
+    if (reference && typeof reference === "object" && Object.keys(reference).length > 0) {
+        lines.push("", "## Invocation");
+        for (const [key, value] of Object.entries(reference)) {
+            lines.push(`- ${key}: ${JSON.stringify(value)}`);
+        }
+    }
+    if (Object.keys(entry.arguments).length > 0) {
+        lines.push("", "## Arguments", "```json", JSON.stringify(entry.arguments, null, 2), "```");
+    }
+    return `${lines.join("\n").trimEnd()}\n`;
+}
+function oneLine(text) {
+    return text.replace(/\s+/g, " ").trim();
+}
+//# sourceMappingURL=skill-render.js.map

package/lib/skill.d.ts

@@ -1,10 +1,15 @@
-import type { HarnessEntry, RefinementResult } from "./types.js";
-/** Convert a harness entry id (underscore slug) to a kebab-case skill name. */
-export declare function skillNameOf(id: string): string;
+import type { RefinementResult } from "./types.js";
+export { renderSkillMarkdown, skillNameOf } from "./skill-render.js";
 /** Resolve and defend the skill directory for an entry id. */
 export declare function skillDir(skillsRoot: string, id: string): string;
-/** Render a harness skill entry as a discoverable SKILL.md document. */
-export declare function renderSkillMarkdown(entry: HarnessEntry): string;
-/** Apply the skill-kind edits of an applied refinement to the skills root. */
-export declare function syncSkillsFromResult(skillsRoot: string, result: RefinementResult): void;
+/**
+ * Apply the skill-kind edits of an applied refinement to the skills root.
+ * Returns materialization warnings (rendered-SKILL.md mechanical problems
+ * and dangling resource references) — the file is still written, but the
+ * caller should surface them: a rendered file that fails the platform's
+ * frontmatter rules would be IGNORED by the skill loader, and a body
+ * referencing resources the entry does not ship would load with broken
+ * links.
+ */
+export declare function syncSkillsFromResult(skillsRoot: string, result: RefinementResult): string[];
 //# sourceMappingURL=skill.d.ts.map

package/lib/skill.js

@@ -9,10 +9,9 @@
  */
 import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { join, resolve, sep } from "node:path";
-/** Convert a harness entry id (underscore slug) to a kebab-case skill name. */
-export function skillNameOf(id) {
-    return id.toLowerCase().replace(/_/g, "-");
-}
+import { renderSkillMarkdown, skillNameOf } from "./skill-render.js";
+import { skillResourceRefs, validateRenderedSkill } from "./skillquality.js";
+export { renderSkillMarkdown, skillNameOf } from "./skill-render.js";
 /** Resolve and defend the skill directory for an entry id. */
 export function skillDir(skillsRoot, id) {
     const root = resolve(skillsRoot);
@@ -22,30 +21,17 @@ export function skillDir(skillsRoot, id) {
     }
     return dir;
 }
-/** Render a harness skill entry as a discoverable SKILL.md document. */
-export function renderSkillMarkdown(entry) {
-    const lines = [
-        "---",
-        `name: ${skillNameOf(entry.id)}`,
-        `description: ${oneLine(entry.title)}`,
-        "---",
-        "",
-        entry.content.trim(),
-    ];
-    const reference = entry.reference;
-    if (reference && typeof reference === "object" && Object.keys(reference).length > 0) {
-        lines.push("", "## Invocation");
-        for (const [key, value] of Object.entries(reference)) {
-            lines.push(`- ${key}: ${JSON.stringify(value)}`);
-        }
-    }
-    if (Object.keys(entry.arguments).length > 0) {
-        lines.push("", "## Arguments", "```json", JSON.stringify(entry.arguments, null, 2), "```");
-    }
-    return `${lines.join("\n").trimEnd()}\n`;
-}
-/** Apply the skill-kind edits of an applied refinement to the skills root. */
+/**
+ * Apply the skill-kind edits of an applied refinement to the skills root.
+ * Returns materialization warnings (rendered-SKILL.md mechanical problems
+ * and dangling resource references) — the file is still written, but the
+ * caller should surface them: a rendered file that fails the platform's
+ * frontmatter rules would be IGNORED by the skill loader, and a body
+ * referencing resources the entry does not ship would load with broken
+ * links.
+ */
 export function syncSkillsFromResult(skillsRoot, result) {
+    const warnings = [];
     for (const edit of result.appliedEdits) {
         if (edit.kind !== "skill" || !edit.applied)
             continue;
@@ -53,15 +39,37 @@ export function syncSkillsFromResult(skillsRoot, result) {
             removeSkill(skillsRoot, edit.id);
             continue;
         }
-        writeSkill(skillsRoot, edit.after);
+        warnings.push(...writeSkill(skillsRoot, edit.after));
     }
+    return warnings;
 }
+/** Write one skill entry as a SKILL.md; returns materialization warnings. */
 function writeSkill(skillsRoot, entry) {
     const dir = skillDir(skillsRoot, entry.id);
     mkdirSync(dir, { recursive: true });
     const temp = join(dir, `SKILL.md.${process.pid}.tmp`);
     writeFileSync(temp, renderSkillMarkdown(entry), "utf8");
     renameSync(temp, join(dir, "SKILL.md"));
+    return materializationWarnings(dir, entry);
+}
+/** Post-write checks on the exact file that landed on disk. */
+function materializationWarnings(dir, entry) {
+    const warnings = [];
+    for (const problem of validateRenderedSkill(entry)) {
+        warnings.push(`skill ${entry.id}: rendered SKILL.md would be ignored by the platform: ${problem}`);
+    }
+    const root = resolve(dir);
+    for (const ref of skillResourceRefs(entry.content)) {
+        const target = resolve(root, ref);
+        if (target !== root && !target.startsWith(`${root}${sep}`)) {
+            warnings.push(`skill ${entry.id}: body resource reference escapes the skill directory: ${ref}`);
+            continue;
+        }
+        if (!existsSync(target)) {
+            warnings.push(`skill ${entry.id}: body references missing resource ${ref} (expected at ${target})`);
+        }
+    }
+    return warnings;
 }
 function removeSkill(skillsRoot, id) {
     const dir = skillDir(skillsRoot, id);
@@ -69,7 +77,4 @@ function removeSkill(skillsRoot, id) {
         rmSync(dir, { recursive: true, force: true });
     }
 }
-function oneLine(text) {
-    return text.replace(/\s+/g, " ").trim();
-}
 //# sourceMappingURL=skill.js.map

package/lib/skillquality.d.ts

@@ -0,0 +1,80 @@
+import type { HarnessEntry } from "./types.js";
+/** Relative location of the skill-creator template facts. */
+export declare const SKILL_CREATOR_TEMPLATE_REL: string;
+/**
+ * Read the skill-creator template facts
+ * (`<skillsRoot>/skill-creator/references/template.md`; facts distilled
+ * from the official deepseek-harness skills). Returns null when the skills
+ * are not installed — callers fall back to the builtin distilled guide.
+ * Reading is a runtime reference, never a copy: template updates in the
+ * skill are picked up automatically.
+ */
+export declare function readSkillCreatorTemplate(skillsRoot: string): string | null;
+/**
+ * Builtin distilled skill-quality guide (fallback when the skill-creator
+ * template is not installed). Condenses the template facts — frontmatter
+ * schema, the 7 structural features, paragraph skeleton, and the
+ * no-duplication / real-trigger rules — so a planner still authors skills
+ * to the standard on installs without the skill-creator / skill-audit
+ * skills.
+ */
+export declare const BUILTIN_SKILL_QUALITY_GUIDE = "DSH skill quality standard (distilled by the author from the official deepseek-harness 11 skills; the full facts live in <skillsRoot>/skill-creator/references/template.md when installed):\n\nFrontmatter schema (platform-enforced; violations make the platform IGNORE the whole file):\n- name: required, kebab-case only (^[a-z0-9]+(?:-[a-z0-9]+)*$)\n- description: required, non-empty; write \"use when / do not use when\" routing so the model can select it correctly\n- invocation booleans accept true/false/yes/no/on/off/1/0; legacy camelCase keys (disableModelInvocation / modelInvocable / userInvocable) are rejected\n- whenToUse (optional): non-empty string; metadata (optional): object\n\nThe 7 structural features of the official deepseek-harness skills:\n1. Frontmatter is routing metadata, not a summary (description = when to use / when not to use)\n2. Opens with a boundary declaration (guidance, not a script; mechanical flow skills may omit the disclaimer)\n3. Prerequisites + exclusions: explicit required input, stop when missing (report the required input and stop), excluded scenarios\n4. Layered information: Sources of truth (link only, do not re-summarize) -> numbered blocking requirements -> manual checks -> verification commands -> report format; all executable, no slogans\n5. Skill interlinks: reference a single source of truth instead of duplicating it\n6. Verifiable completion criteria: explicit verification commands and report format\n7. Real use + iteration: a real trigger scenario must exist; calibration conclusions distill into references/\n\nParagraph skeleton (writing order): frontmatter -> H1 + boundary declaration -> Sources of truth -> numbered requirements / workflow (full commands) -> exclusions / stop conditions -> verification and report.\n\nCreation rules: only create a skill for a REAL trigger scenario (who, in what real task, what signal) grounded in the trajectory \u2014 never invent one to pad the store; do not duplicate the official 11 skills or existing entries; skill bodies should be a SKILL.md document (this is what materializes under <skillsRoot>/<kebab-name>/SKILL.md).";
+export interface SkillQualityGuide {
+    /** Where the guide text came from: the on-disk template or the builtin guide. */
+    source: "template" | "builtin";
+    text: string;
+}
+/**
+ * The quality guide handed to the planner: the skill-creator template facts
+ * when the skills are installed, otherwise the builtin distilled guide.
+ * Never throws — a missing/unreadable template degrades to the builtin.
+ */
+export declare function skillQualityGuide(skillsRoot: string | undefined): SkillQualityGuide;
+/** Split frontmatter out of a raw SKILL.md. Returns { yaml, body } or null when delimiters are missing. */
+export declare function splitFrontmatter(raw: string): {
+    yaml: string;
+    body: string;
+} | null;
+/**
+ * Mechanical frontmatter validation of a rendered SKILL.md, mirroring
+ * `skill-creator/scripts/validate-frontmatter.mjs` (and the platform's
+ * skill-filesystem rules): delimiter structure, name kebab-case, non-empty
+ * description, invocation-boolean spellings, legacy camelCase key rejection,
+ * whenToUse/metadata types. Returns human-readable problems; an empty array
+ * means the file would load.
+ */
+export declare function validateRenderedSkillMarkdown(markdown: string): string[];
+/**
+ * Mechanical validation of a skill entry's raw `content` (the SKILL.md body
+ * that materializes under the generated frontmatter). Code-enforced at
+ * apply time so a bad entry never reaches the store:
+ * - empty content is rejected;
+ * - content must not open with a `---` block: the materializer generates
+ *   its own frontmatter, and a second frontmatter in the body would be
+ *   parsed instead of the generated one (the platform reads the FIRST
+ *   closing `---`), so the file could be ignored or routed wrongly;
+ * - resource references (`references/…`, `scripts/…`) must be skill-local
+ *   relative paths — parent-relative (`../`) or absolute targets escape the
+ *   skill directory and are rejected.
+ * Returns human-readable problems; an empty array means the content is
+ * mechanically acceptable.
+ */
+export declare function validateSkillEntryContent(content: string): string[];
+/**
+ * Validate the FULL rendered SKILL.md of an entry (generated frontmatter +
+ * body) — the exact bytes that materialize on disk. Used as the final
+ * code-enforced line after materialization; problems here mean the platform
+ * would refuse to load the file.
+ */
+export declare function validateRenderedSkill(entry: HarnessEntry): string[];
+/**
+ * Resource references (`references/…`, `scripts/…`) found in a skill body —
+ * the same scanning policy as validate-frontmatter.mjs: markdown link
+ * targets starting with the category, plus backticked/prose paths carrying
+ * a filename extension. Used after materialization to warn about dangling
+ * references (a body referencing a resource the entry never ships).
+ */
+export declare function skillResourceRefs(content: string): string[];
+/** Kebab-case name under which the entry materializes (exported for diagnostics). */
+export { skillNameOf } from "./skill-render.js";
+//# sourceMappingURL=skillquality.d.ts.map