@hmharness/evolution 0.14.1 → 0.14.5

package/dist/bench.d.ts

@@ -24,6 +24,12 @@ export interface BenchResult {
     detail: string;
 }
 export declare function matchExpect(output: string, expect: string[]): boolean;
+/** Fence convention: when the reply is a fenced block, assertions test the
+ *  fence INNER content (day-55 finding: models preserve literals verbatim
+ *  inside code fences while normalizing them in prose - the fence is the
+ *  reply convention that makes exactness attainable for habit-prone models).
+ *  Tolerates a trailing space after the opening fence markers. */
+export declare function fenceInner(output: string): string;
 /** Full structured assertion: every declared mode must hold. */
 export declare function matchCase(output: string, c: Pick<BenchCase, 'expect' | 'expectExact' | 'expectRegex' | 'expectNone' | 'expectAny'>): {
     pass: boolean;

package/dist/bench.js

@@ -24,27 +24,37 @@ export function matchExpect(output, expect) {
     const lower = output.toLowerCase();
     return expect.every((e) => lower.includes(e.toLowerCase()));
 }
+/** Fence convention: when the reply is a fenced block, assertions test the
+ *  fence INNER content (day-55 finding: models preserve literals verbatim
+ *  inside code fences while normalizing them in prose - the fence is the
+ *  reply convention that makes exactness attainable for habit-prone models).
+ *  Tolerates a trailing space after the opening fence markers. */
+export function fenceInner(output) {
+    const m = output.match(/```[a-zA-Z]*[ \t]*\r?\n([\s\S]*?)(?:\r?\n)?```/);
+    return m ? m[1] : output;
+}
 /** Full structured assertion: every declared mode must hold. */
 export function matchCase(output, c) {
-    if (c.expectExact !== undefined && output.trim() !== c.expectExact.trim()) {
-        return { pass: false, detail: `exact mismatch: got "${output.trim().slice(0, 80)}"` };
+    const body = fenceInner(output);
+    if (c.expectExact !== undefined && body.trim() !== c.expectExact.trim()) {
+        return { pass: false, detail: `exact mismatch: got "${body.trim().slice(0, 80)}"` };
     }
     if (c.expectRegex !== undefined) {
         try {
-            if (!new RegExp(c.expectRegex).test(output))
+            if (!new RegExp(c.expectRegex).test(body))
                 return { pass: false, detail: `regex mismatch: /${c.expectRegex.slice(0, 60)}/` };
         }
         catch {
             return { pass: false, detail: `invalid regex in case: ${c.expectRegex.slice(0, 40)}` };
         }
     }
-    if (c.expectNone && c.expectNone.some((e) => output.toLowerCase().includes(e.toLowerCase()))) {
+    if (c.expectNone && c.expectNone.some((e) => body.toLowerCase().includes(e.toLowerCase()))) {
         return { pass: false, detail: `forbidden marker present: ${c.expectNone.join(' && ')}` };
     }
-    if (c.expectAny && !c.expectAny.some((e) => output.toLowerCase().includes(e.toLowerCase()))) {
+    if (c.expectAny && !c.expectAny.some((e) => body.toLowerCase().includes(e.toLowerCase()))) {
         return { pass: false, detail: `none of the allowed markers found: ${c.expectAny.join(' || ')}` };
     }
-    if (!matchExpect(output, c.expect)) {
+    if (!matchExpect(body, c.expect)) {
         return { pass: false, detail: `missing "${c.expect.join('" && "')}" in output` };
     }
     return { pass: true, detail: 'ok' };

package/dist/candidates.d.ts

@@ -46,6 +46,12 @@ export interface ExperimentArm {
     n: number;
     passRate: number;
     tokens: number;
+    /** per-case rows (day-56): which case flipped is the point of a diff */
+    results?: Array<{
+        name: string;
+        pass: boolean;
+        tokens: number;
+    }>;
 }
 export interface ExperimentReport {
     candidateId: string;

package/dist/candidates.js

@@ -105,23 +105,29 @@ export async function runCandidateExperiment(home, candidateId, opts) {
     // holdout cases are excluded from promotion gates (bench.ts anti-memorization)
     const gate = opts.cases.filter((c) => !c.holdout).slice(0, Math.max(1, opts.maxCases ?? 12));
     let cPass = 0, cTok = 0, tPass = 0, tTok = 0;
+    // per-case rows (day-56 observability: which case flipped is the whole
+    // point of a diff - aggregates alone cannot answer it)
+    const ctlRows = [];
+    const trtRows = [];
     for (const c of gate) {
         const ctl = await opts.runCase(c, 'control');
         if (ctl.pass)
             cPass++;
         cTok += ctl.tokens;
+        ctlRows.push({ name: c.name, pass: ctl.pass, tokens: ctl.tokens });
         const trt = await opts.runCase(c, 'treatment');
         if (trt.pass)
             tPass++;
         tTok += trt.tokens;
+        trtRows.push({ name: c.name, pass: trt.pass, tokens: trt.tokens });
     }
     const v = verdictFor({ pass: cPass, n: gate.length }, { pass: tPass, n: gate.length });
     const report = {
         candidateId,
         ranAt: new Date().toISOString(),
         cases: gate.length,
-        control: { pass: cPass, n: gate.length, passRate: gate.length ? cPass / gate.length : 0, tokens: cTok },
-        treatment: { pass: tPass, n: gate.length, passRate: gate.length ? tPass / gate.length : 0, tokens: tTok },
+        control: { pass: cPass, n: gate.length, passRate: gate.length ? cPass / gate.length : 0, tokens: cTok, results: ctlRows },
+        treatment: { pass: tPass, n: gate.length, passRate: gate.length ? tPass / gate.length : 0, tokens: tTok, results: trtRows },
         ...v,
     };
     const dir = join(home, 'evolution', 'experiments', candidateId);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evolution",
-  "version": "0.14.1",
+  "version": "0.14.5",
   "description": "hmharness evolution subsystem: persistent memory, insight capture, skill library, and the bench that gives evolution its fitness signal. First-class kernel citizen, not a plugin.",
   "type": "module",
   "main": "dist/index.js",