@hmharness/evolution 0.14.11 → 0.14.13

package/dist/reward-model.d.ts

@@ -64,3 +64,28 @@ export declare function exportDpoPairs(labels: HumanLabel[], insights: InsightLi
     crossBucketPrefix?: number;
     sources?: Map<string, 'human' | 'judge'>;
 }): DpoPair[];
+/** Deterministic ~80/20 train/eval split by pair identity. Pure. */
+export declare function splitDpoPairs(pairs: DpoPair[], salt?: string): {
+    train: DpoPair[];
+    eval: DpoPair[];
+};
+/** Leakage/quality audit before training consumes the pairs: empty sides,
+ *  exact duplicates, and eval pairs whose prompt ALSO appears in train
+ *  (prompt leakage inflates offline metrics). Pure. */
+export declare function auditDpoPairs(train: DpoPair[], evalP: DpoPair[]): {
+    emptyPrompt: number;
+    emptyAnswer: number;
+    duplicatePairs: number;
+    evalPromptsLeakedIntoTrain: number;
+};
+/** The versioned manifest written beside the exported pairs file - the
+ *  dataset's birth certificate: counts by source, split sizes, audit. */
+export interface DpoManifest {
+    version: string;
+    pairsTotal: number;
+    bySource: Record<string, number>;
+    train: number;
+    eval: number;
+    audit: ReturnType<typeof auditDpoPairs>;
+    exportedAt: string;
+}

package/dist/reward-model.js

@@ -146,7 +146,13 @@ export function exportDpoPairs(labels, insights, taskOf, answerOf, opts = {}) {
         for (const b of rows) {
             if (a.session >= b.session)
                 continue;
-            const gap = a.y - b.y;
+            // round: 3/5 - 1/5 = 0.39999999999999997 in floats, and the epsilon
+            // silently ate every 2-step in-bucket pair (★1 vs ★3) - the judge's
+            // whole variance band
+            // abs: rows iterate in label-file order - the lower score can come first
+            // (judge labels do), which made a.y - b.y negative and silently dropped
+            // the pair; rounding kills the 3/5-1/5 float epsilon
+            const gap = Math.round(Math.abs(a.y - b.y) * 1000) / 1000;
             if (gap < minGap)
                 continue;
             const hi = gap > 0 ? a : b;
@@ -185,3 +191,48 @@ export function exportDpoPairs(labels, insights, taskOf, answerOf, opts = {}) {
     }
     return pairs;
 }
+/* ---------------- DPO dataset form (J1 follow-up) ---------------- */
+/** Deterministic zero-dep FNV-1a - the split must not reshuffle between
+ *  runs: a pair that was in eval yesterday must stay in eval today. */
+function fnv1a(str) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < str.length; i++) {
+        h ^= str.charCodeAt(i);
+        h = (h + (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
+    }
+    return h >>> 0;
+}
+/** Deterministic ~80/20 train/eval split by pair identity. Pure. */
+export function splitDpoPairs(pairs, salt = 'dpo') {
+    const train = [];
+    const evalP = [];
+    for (const p of pairs) {
+        (fnv1a(salt + '|' + p.prompt + '|' + p.chosenSession + '|' + p.rejectedSession) % 5 === 0 ? evalP : train).push(p);
+    }
+    return { train, eval: evalP };
+}
+/** Leakage/quality audit before training consumes the pairs: empty sides,
+ *  exact duplicates, and eval pairs whose prompt ALSO appears in train
+ *  (prompt leakage inflates offline metrics). Pure. */
+export function auditDpoPairs(train, evalP) {
+    let emptyPrompt = 0, emptyAnswer = 0;
+    const seen = new Set();
+    let duplicatePairs = 0;
+    for (const p of [...train, ...evalP]) {
+        if (!p.prompt.trim())
+            emptyPrompt++;
+        if (!p.chosen.trim() || !p.rejected.trim())
+            emptyAnswer++;
+        const key = p.prompt + '\u0000' + p.chosen + '\u0000' + p.rejected;
+        if (seen.has(key))
+            duplicatePairs++;
+        else
+            seen.add(key);
+    }
+    const trainPrompts = new Set(train.map((p) => p.prompt));
+    let evalPromptsLeakedIntoTrain = 0;
+    for (const p of evalP)
+        if (trainPrompts.has(p.prompt))
+            evalPromptsLeakedIntoTrain++;
+    return { emptyPrompt, emptyAnswer, duplicatePairs, evalPromptsLeakedIntoTrain };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evolution",
-  "version": "0.14.11",
+  "version": "0.14.13",
   "description": "hmharness evolution subsystem: persistent memory, insight capture, skill library, and the bench that gives evolution its fitness signal. First-class kernel citizen, not a plugin.",
   "type": "module",
   "main": "dist/index.js",