@hmharness/evolution 0.14.9 → 0.14.10

package/dist/index.d.ts

@@ -5,6 +5,7 @@ export * from './skillpayload.ts';
 export * from './dataset.ts';
 export * from './readiness.ts';
 export * from './labels.ts';
+export * from './reward-model.ts';
 export * from './insights.ts';
 export * from './skills.ts';
 export * from './bench.ts';

package/dist/index.js

@@ -5,6 +5,7 @@ export * from "./skillpayload.js";
 export * from "./dataset.js";
 export * from "./readiness.js";
 export * from "./labels.js";
+export * from "./reward-model.js";
 export * from "./insights.js";
 export * from "./skills.js";
 export * from "./bench.js";

package/dist/reward-model.d.ts

@@ -0,0 +1,62 @@
+import type { HumanLabel } from './labels.ts';
+export type { HumanLabel };
+export interface RewardFeatures {
+    /** outcome bucket: 2=ok, 1=turn-budget, 0=error */
+    outcome: number;
+    /** failed tool calls / total tool calls, 0..1 */
+    toolFailRate: number;
+    /** agent turns used, raw count */
+    turns: number;
+    /** tool calls issued, raw count */
+    toolUses: number;
+}
+export interface RewardWeights {
+    /** logistic weights over [outcome, toolFailRate, log1p(turns), log1p(toolUses), bias] */
+    w: [number, number, number, number, number];
+    trainedAt: string;
+    nSamples: number;
+    trainRmse: number;
+    /** holdout RMSE when n >= 20 (80/20 split, deterministic) */
+    holdoutRmse?: number;
+}
+/** Human label record: re-used from labels.ts (single source). */
+/** Insight record fields the model joins on. */
+export interface InsightLike {
+    session: string;
+    outcome: string;
+    turns?: number;
+    toolUses?: number;
+}
+/** Join labels with insights and map to training rows. Pure - testable. */
+export declare function trainingRows(labels: HumanLabel[], insights: InsightLike[]): Array<{
+    f: RewardFeatures;
+    y: number;
+    session: string;
+}>;
+/** Fit the reward model on joined human labels. Deterministic. */
+export declare function fitRewardModel(labels: HumanLabel[], insights: InsightLike[]): RewardWeights;
+/** Score a run's features with fitted weights -> reward in [0,1]. */
+export declare function scoreFeatures(w: RewardWeights['w'], f: RewardFeatures): number;
+/** Persistence: HMH_HOME/evolution/reward-model.json. */
+export declare function saveRewardModel(home: string, m: RewardWeights): Promise<void>;
+export declare function loadRewardModel(home: string): Promise<RewardWeights | null>;
+/** DPO preference pair for external fine-tuning. */
+export interface DpoPair {
+    prompt: string;
+    chosen: string;
+    rejected: string;
+    /** provenance for auditability */
+    chosenSession: string;
+    rejectedSession: string;
+    gap: number;
+}
+/** Build DPO pairs from joined labels + insights. Two classes:
+ *  (a) IN-BUCKET (gold): same outcome, >=2 star gap - human preference the
+ *      outcome reward cannot see (4-vs-5 star distinctions);
+ *  (b) CROSS-BUCKET (classic): ok-vs-degraded completions of the SAME task
+ *      template (task prefix match) - the textbook chosen/rejected pair.
+ *  Pure - testable. */
+export declare function exportDpoPairs(labels: HumanLabel[], insights: InsightLike[], taskOf: (session: string) => string, answerOf: (session: string) => string, opts?: {
+    minInBucketGap?: number;
+    crossBucketPrefix?: number;
+}): DpoPair[];

package/dist/reward-model.js

@@ -0,0 +1,184 @@
+/**
+ * @hmharness/evolution - reward-model (V3 RL phase, ADR-0010)
+ * A LEARNED reward model over run features, fit on the human star labels -
+ * the blueprint's "Reward -> RL" step after the M11 gate opened.
+ *
+ * Design (grounded in the Agent Lightning lesson: the app defines the task
+ * and the reward; the harness already produces rollouts):
+ *  - features are what the harness already measures per session: outcome
+ *    bucket, tool-failure rate, turns, tool uses (all z-free, reproducible)
+ *  - the model is a tiny logistic regressor (zero deps, deterministic
+ *    gradient descent) predicting the HUMAN score in [0,1]
+ *  - the calibrated outcome-based reward (Spearman 0.981 vs human) is the
+ *    prior; the learned model adds the fine, in-bucket distinctions the
+ *    outcome signal cannot see (4-star vs 5-star runs)
+ *  - weights persist to HMH_HOME/evolution/reward-model.json; refit as
+ *    labels accumulate
+ *
+ * What this is NOT: a fine-tuned LLM. The blueprint explicitly defers
+ * model fine-tuning; DPO pairs for an external trainer are exported
+ * separately (exportDpoPairs).
+ */
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+const FEATURES = [
+    (f) => f.outcome / 2, // 0..1
+    (f) => f.toolFailRate, // 0..1
+    (f) => Math.log1p(Math.min(f.turns, 200)) / Math.log(201), // 0..1
+    (f) => Math.log1p(Math.min(f.toolUses, 200)) / Math.log(201), // 0..1
+];
+function x(f) {
+    return [...FEATURES.map((fn) => fn(f)), 1];
+}
+function sigmoid(z) {
+    return 1 / (1 + Math.exp(-z));
+}
+/** Deterministic 80/20 split by index (no RNG - reproducible fits). */
+function split(rows) {
+    const train = [];
+    const hold = [];
+    rows.forEach((r, i) => (i % 5 === 4 ? hold : train).push(r));
+    return { train, hold };
+}
+function fitLinear(rows, epochs = 4000, lr = 0.05, l2 = 0.01) {
+    const d = 5;
+    let w = new Array(d).fill(0);
+    const xs = rows.map((r) => x(r.f));
+    if (xs.length === 0)
+        return w;
+    for (let e = 0; e < epochs; e++) {
+        const grad = new Array(d).fill(0);
+        for (let i = 0; i < rows.length; i++) {
+            let z = 0;
+            for (let k = 0; k < d; k++)
+                z += w[k] * xs[i][k];
+            const err = sigmoid(z) - rows[i].y; // logistic loss gradient
+            for (let k = 0; k < d; k++)
+                grad[k] += err * xs[i][k];
+        }
+        for (let k = 0; k < d; k++)
+            w[k] -= lr * (grad[k] / rows.length + l2 * w[k]);
+    }
+    return w;
+}
+function rmse(w, rows) {
+    if (rows.length === 0)
+        return 0;
+    let s = 0;
+    for (const r of rows) {
+        let z = 0;
+        const xv = x(r.f);
+        for (let k = 0; k < w.length; k++)
+            z += w[k] * xv[k];
+        s += (sigmoid(z) - r.y) ** 2;
+    }
+    return Math.sqrt(s / rows.length);
+}
+/** Join labels with insights and map to training rows. Pure - testable. */
+export function trainingRows(labels, insights) {
+    const bySession = new Map();
+    for (const i of insights)
+        if (!bySession.has(i.session))
+            bySession.set(i.session, i);
+    const rows = [];
+    for (const l of labels) {
+        const ins = bySession.get(l.session);
+        if (!ins)
+            continue;
+        const uses = Number(ins.toolUses ?? 0);
+        const outcome = ins.outcome === 'ok' ? 2 : ins.outcome === 'turn-budget' ? 1 : 0;
+        rows.push({
+            f: { outcome, toolFailRate: 0, turns: Number(ins.turns ?? 0), toolUses: uses },
+            y: l.score / 5,
+            session: l.session,
+        });
+    }
+    return rows;
+}
+/** Fit the reward model on joined human labels. Deterministic. */
+export function fitRewardModel(labels, insights) {
+    const rows = trainingRows(labels, insights);
+    const { train, hold } = split(rows);
+    const w = fitLinear(train);
+    return {
+        w: w,
+        trainedAt: new Date().toISOString(),
+        nSamples: rows.length,
+        trainRmse: Number(rmse(w, train).toFixed(4)),
+        ...(hold.length >= 4 ? { holdoutRmse: Number(rmse(w, hold).toFixed(4)) } : {}),
+    };
+}
+/** Score a run's features with fitted weights -> reward in [0,1]. */
+export function scoreFeatures(w, f) {
+    const xv = x(f);
+    let z = 0;
+    for (let k = 0; k < w.length; k++)
+        z += w[k] * xv[k];
+    return sigmoid(z);
+}
+/** Persistence: HMH_HOME/evolution/reward-model.json. */
+export async function saveRewardModel(home, m) {
+    const dir = join(home, 'evolution');
+    await mkdir(dir, { recursive: true });
+    await writeFile(join(dir, 'reward-model.json'), JSON.stringify(m, null, 2) + '\n', 'utf8');
+}
+export async function loadRewardModel(home) {
+    try {
+        return JSON.parse(await readFile(join(home, 'evolution', 'reward-model.json'), 'utf8'));
+    }
+    catch {
+        return null;
+    }
+}
+/** Build DPO pairs from joined labels + insights. Two classes:
+ *  (a) IN-BUCKET (gold): same outcome, >=2 star gap - human preference the
+ *      outcome reward cannot see (4-vs-5 star distinctions);
+ *  (b) CROSS-BUCKET (classic): ok-vs-degraded completions of the SAME task
+ *      template (task prefix match) - the textbook chosen/rejected pair.
+ *  Pure - testable. */
+export function exportDpoPairs(labels, insights, taskOf, answerOf, opts = {}) {
+    const minGap = opts.minInBucketGap ?? 0.4;
+    const prefixLen = opts.crossBucketPrefix ?? 60;
+    const rows = trainingRows(labels, insights);
+    const insOf = new Map(insights.map((i) => [i.session, i]));
+    const pairs = [];
+    for (const a of rows) {
+        for (const b of rows) {
+            if (a.session >= b.session)
+                continue;
+            const gap = a.y - b.y;
+            if (gap < minGap)
+                continue;
+            const hi = gap > 0 ? a : b;
+            const lo = gap > 0 ? b : a;
+            const hiIns = insOf.get(hi.session);
+            const loIns = insOf.get(lo.session);
+            if (!hiIns || !loIns)
+                continue;
+            const sameOutcome = hiIns.outcome === loIns.outcome;
+            let ok = false;
+            if (sameOutcome) {
+                ok = true; // in-bucket gold
+            }
+            else {
+                // cross-bucket: only pair completions of the SAME task template
+                // (bench templates share long prefixes) - otherwise they teach
+                // nothing transferable
+                const ta = taskOf(hi.session);
+                const tb = taskOf(lo.session);
+                ok = ta.length >= prefixLen && ta.slice(0, prefixLen) === tb.slice(0, prefixLen);
+            }
+            if (!ok)
+                continue;
+            pairs.push({
+                prompt: taskOf(hi.session),
+                chosen: answerOf(hi.session),
+                rejected: answerOf(lo.session),
+                chosenSession: hi.session,
+                rejectedSession: lo.session,
+                gap: Number(gap.toFixed(2)),
+            });
+        }
+    }
+    return pairs;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evolution",
-  "version": "0.14.9",
+  "version": "0.14.10",
   "description": "hmharness evolution subsystem: persistent memory, insight capture, skill library, and the bench that gives evolution its fitness signal. First-class kernel citizen, not a plugin.",
   "type": "module",
   "main": "dist/index.js",

package/dist/bench.d.ts.v2bak.v2bak

@@ -1,44 +0,0 @@
-export interface BenchCase {
-    name: string;
-    prompt: string;
-    /** All substrings (split on &&) must appear in the output, case-insensitive. */
-    expect: string[];
-    /** Output must equal this string exactly (both trimmed). */
-    expectExact?: string;
-    /** Output must match this regex (whole output, case-sensitive). */
-    expectRegex?: string;
-    /** Substrings that must NOT appear (failure-marker exclusion). */
-    expectNone?: string[];
-    /** At least ONE of these substrings must appear. */
-    expectAny?: string[];
-    /** When true the case runs through the full agent loop with tools. */
-    tools: boolean;
-    /** Holdout cases are excluded from the promotion gate and re-verify after promotion (anti-memorization, GDPevo style). */
-    holdout: boolean;
-    /** Candidate token-cost ceiling as a multiple of the baseline run. */
-    costCap?: number;
-}
-export interface BenchResult {
-    name: string;
-    pass: boolean;
-    detail: string;
-}
-export declare function matchExpect(output: string, expect: string[]): boolean;
-/** Fence convention: when the reply is a fenced block, assertions test the
- *  fence INNER content (day-55 finding: models preserve literals verbatim
- *  inside code fences while normalizing them in prose - the fence is the
- *  reply convention that makes exactness attainable for habit-prone models).
- *  Tolerates a trailing space after the opening fence markers. */
-export declare function fenceInner(output: string): string;
-/** Full structured assertion: every declared mode must hold. */
-export declare function matchCase(output: string, c: Pick<BenchCase, 'expect' | 'expectExact' | 'expectRegex' | 'expectNone' | 'expectAny'>): {
-    pass: boolean;
-    detail: string;
-};
-export declare function listCases(home: string): Promise<BenchCase[]>;
-export declare function runBench(home: string, run: (c: BenchCase) => Promise<string>): Promise<{
-    results: BenchResult[];
-    passRate: number;
-}>;
-/** Starter cases so the evolve gate has a signal from day one. */
-export declare function seedCases(home: string): Promise<string[]>;

package/dist/bench.js.v2bak.v2bak

@@ -1,156 +0,0 @@
-/**
- * @hmharness/evolution - bench
- * The fitness signal for self-evolution. bench/cases/*.task files are:
- *   line 1        the prompt
- *   expect: a && b   substrings that must ALL appear in the final output
- *   expect-exact: "..."   the output equals this string (trimmed)
- *   expect-regex: ...     a regex the output must match
- *   expect-none: a && b   substrings that must NOT appear
- *   expect-any: a || b    one of these substrings must appear
- *   tools: loop   (optional) run through the full agent loop with tools
- *   cost-cap: 1.3  (optional) candidate cost multiplier ceiling vs baseline
- * Skill or prompt changes must keep the bench green before promotion - the
- * evolve loop enforces this A/B (baseline vs candidate).
- *
- * Assertion upgrade (gate methodology): plain substring matching lets
- * verbose-but-wrong outputs pass; the structured modes pin exact shapes,
- * forbid failure markers, and (with cost-cap) stop a candidate that only
- * passes by burning 3x tokens. Old files keep working - `expect:` keeps
- * its ALL-substrings semantics.
- */
-import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
-import { join } from 'node:path';
-export function matchExpect(output, expect) {
-    const lower = output.toLowerCase();
-    return expect.every((e) => lower.includes(e.toLowerCase()));
-}
-/** Fence convention: when the reply is a fenced block, assertions test the
- *  fence INNER content (day-55 finding: models preserve literals verbatim
- *  inside code fences while normalizing them in prose - the fence is the
- *  reply convention that makes exactness attainable for habit-prone models).
- *  Tolerates a trailing space after the opening fence markers. */
-export function fenceInner(output) {
-    const m = output.match(/```[a-zA-Z]*[ \t]*\r?\n([\s\S]*?)(?:\r?\n)?```/);
-    return m ? m[1] : output;
-}
-/** Full structured assertion: every declared mode must hold. */
-export function matchCase(output, c) {
-    const body = fenceInner(output);
-    if (c.expectExact !== undefined && body.trim() !== c.expectExact.trim()) {
-        return { pass: false, detail: `exact mismatch: got "${body.trim().slice(0, 80)}"` };
-    }
-    if (c.expectRegex !== undefined) {
-        try {
-            if (!new RegExp(c.expectRegex).test(body))
-                return { pass: false, detail: `regex mismatch: /${c.expectRegex.slice(0, 60)}/` };
-        }
-        catch {
-            return { pass: false, detail: `invalid regex in case: ${c.expectRegex.slice(0, 40)}` };
-        }
-    }
-    if (c.expectNone && c.expectNone.some((e) => body.toLowerCase().includes(e.toLowerCase()))) {
-        return { pass: false, detail: `forbidden marker present: ${c.expectNone.join(' && ')}` };
-    }
-    if (c.expectAny && !c.expectAny.some((e) => body.toLowerCase().includes(e.toLowerCase()))) {
-        return { pass: false, detail: `none of the allowed markers found: ${c.expectAny.join(' || ')}` };
-    }
-    if (!matchExpect(body, c.expect)) {
-        return { pass: false, detail: `missing "${c.expect.join('" && "')}" in output` };
-    }
-    return { pass: true, detail: 'ok' };
-}
-export async function listCases(home) {
-    const dir = join(home, 'bench', 'cases');
-    let files;
-    try {
-        files = (await readdir(dir)).filter((f) => f.endsWith('.task'));
-    }
-    catch {
-        return [];
-    }
-    const cases = [];
-    for (const f of files.sort()) {
-        const raw = (await readFile(join(dir, f), 'utf8')).trim();
-        const lines = raw.split('\n');
-        const prompt = lines[0];
-        const pick = (key) => {
-            const l = lines.find((x) => x.startsWith(key + ':'));
-            return l ? l.slice(key.length + 1).trim() : undefined;
-        };
-        const list = (key) => {
-            const v = pick(key);
-            return v ? v.split(/&&|\|\|/).map((s) => s.trim()).filter(Boolean) : undefined;
-        };
-        const costCap = Number(pick('cost-cap'));
-        cases.push({
-            name: f.replace(/\.task$/, ''),
-            prompt,
-            expect: list('expect') ?? [],
-            expectExact: pick('expect-exact')?.replace(/^"|"$/g, ''),
-            expectRegex: pick('expect-regex'),
-            expectNone: list('expect-none'),
-            expectAny: list('expect-any'),
-            tools: pick('tools') === 'loop',
-            holdout: pick('holdout') === 'true',
-            costCap: Number.isFinite(costCap) && costCap > 0 ? costCap : undefined,
-        });
-    }
-    return cases;
-}
-export async function runBench(home, run) {
-    const cases = await listCases(home);
-    const results = [];
-    for (const c of cases) {
-        try {
-            const out = await run(c);
-            const r = matchCase(out, c);
-            results.push({ name: c.name, pass: r.pass, detail: r.detail });
-        }
-        catch (err) {
-            results.push({ name: c.name, pass: false, detail: String(err).slice(0, 120) });
-        }
-    }
-    const passed = results.filter((r) => r.pass).length;
-    // persist a history record: readiness's "eval-regression-stable" condition
-    // reads evolution/benches/*.json - without these records the RL gate's
-    // stability condition can never be met (day-20 audit gap)
-    try {
-        const dir = join(home, 'evolution', 'benches');
-        await mkdir(dir, { recursive: true });
-        const record = {
-            time: new Date().toISOString(),
-            total: cases.length,
-            passed,
-            passRate: cases.length === 0 ? 1 : passed / cases.length,
-            results: results.map((r) => ({ name: r.name, pass: r.pass, detail: r.detail.slice(0, 200) })),
-        };
-        await writeFile(join(dir, `${record.time.replace(/[:.]/g, '-')}.json`), JSON.stringify(record, null, 2) + '\n', 'utf8');
-    }
-    catch { /* history is best-effort; the run result still returns */ }
-    return { results, passRate: cases.length === 0 ? 1 : passed / cases.length };
-}
-/** Starter cases so the evolve gate has a signal from day one. */
-export async function seedCases(home) {
-    const dir = join(home, 'bench', 'cases');
-    const written = [];
-    const seeds = [
-        ['toolchain-report.task', '运行鸿蒙工具链体检,然后逐项说出一共检查了哪三个工具的名称\nexpect: hdc && hvigorw && ohpm\ntools: loop\n'],
-        ['reply-determinism.task', '只回复这六个字符,不要任何其他内容:HMH-OK\nexpect: HMH-OK\n'],
-        ['toolchain-ohpm-status.task', '运行鸿蒙工具链体检,然后只回答 ohpm 的状态是 OK 还是 MISSING\nexpect: OK\ntools: loop\nholdout: true\n'],
-    ];
-    let existing = [];
-    try {
-        existing = await readdir(dir);
-    }
-    catch {
-        /* first seed */
-    }
-    for (const [name, body] of seeds) {
-        if (existing.includes(name))
-            continue;
-        await mkdir(dir, { recursive: true });
-        await writeFile(join(dir, name), body, 'utf8');
-        written.push(name);
-    }
-    return written;
-}

package/dist/candidates.d.ts.v2bak.v2bak

@@ -1,114 +0,0 @@
-import type { BenchCase } from './bench.ts';
-export type CandidateTarget = 'prompt' | 'skill' | 'context' | 'tool_policy' | 'model_router' | 'workflow' | 'harness';
-export declare const CANDIDATE_TARGETS: CandidateTarget[];
-export interface EvolutionCandidate {
-    id: string;
-    target: CandidateTarget;
-    /** the version this candidate is measured against (baseline pointer) */
-    baseVersion: string;
-    candidateVersion: string;
-    hypothesis: string;
-    /** metric the experiment reports on, e.g. 'bench.passRate' */
-    expectedMetric: string;
-    /** target-specific payload: skill markdown, prompt delta, router table… */
-    payload?: string;
-    /** who proposed it - 'agent' proposals on prompt/harness need human sign-off */
-    origin?: string;
-    createdAt: string;
-}
-export declare function registerCandidate(home: string, cand: Omit<EvolutionCandidate, 'id' | 'createdAt'> & {
-    id?: string;
-}): Promise<EvolutionCandidate>;
-export declare function listCandidates(home: string): Promise<EvolutionCandidate[]>;
-export declare function getCandidate(home: string, id: string): Promise<EvolutionCandidate | null>;
-/** Two-proportion two-sided test (treatment vs control pass rates). */
-export declare function twoProportionTest(control: {
-    pass: number;
-    n: number;
-}, treatment: {
-    pass: number;
-    n: number;
-}): {
-    z: number;
-    p: number;
-    diff: number;
-};
-export interface ArmResult {
-    pass: boolean;
-    /** token cost of the arm run (cost-cap honesty, same currency as estTokens) */
-    tokens: number;
-}
-/** Injected by the caller: run one bench case under one arm. The runner owns
- *  the actual injection point (skills block, memory config, router table…). */
-export type ArmRunner = (c: BenchCase, arm: 'control' | 'treatment') => Promise<ArmResult>;
-export interface ExperimentArm {
-    pass: number;
-    n: number;
-    passRate: number;
-    tokens: number;
-    /** per-case rows (day-56): which case flipped is the point of a diff */
-    results?: Array<{
-        name: string;
-        pass: boolean;
-        tokens: number;
-    }>;
-}
-export interface ExperimentReport {
-    candidateId: string;
-    ranAt: string;
-    cases: number;
-    control: ExperimentArm;
-    treatment: ExperimentArm;
-    diff: number;
-    z: number;
-    p: number;
-    verdict: 'promote-eligible' | 'reject' | 'needs-data';
-    reason: string;
-}
-/** Per-arm minimum before any verdict counts (aligned with impact.ts canary
- *  threshold: >=8 sessions, and blueprint "one success is not significance"). */
-export declare const MIN_ARM_N = 8;
-export declare function verdictFor(control: {
-    pass: number;
-    n: number;
-}, treatment: {
-    pass: number;
-    n: number;
-}): {
-    verdict: ExperimentReport['verdict'];
-    reason: string;
-    z: number;
-    p: number;
-    diff: number;
-};
-export declare function runCandidateExperiment(home: string, candidateId: string, opts: {
-    runCase: ArmRunner;
-    cases: BenchCase[];
-    maxCases?: number;
-}): Promise<ExperimentReport>;
-/** Latest experiment report for a candidate (promotion evidence). */
-export declare function latestExperiment(home: string, candidateId: string): Promise<ExperimentReport | null>;
-export interface ActiveVersion {
-    target: CandidateTarget;
-    version: string;
-    candidateId: string;
-    since: string;
-    reportAt?: string;
-}
-export declare function activeVersions(home: string): Promise<Record<string, ActiveVersion>>;
-export interface PromoteResult {
-    ok: boolean;
-    error?: string;
-    active?: ActiveVersion;
-}
-/** Blueprint gates, enforced: latest report must be promote-eligible, a
- *  previous pointer is written BEFORE activation, agent-origin prompt/harness
- *  candidates need approvedByHuman. Skill promotions ride the existing
- *  promoteSkill canary channel. */
-export declare function promoteCandidate(home: string, candidateId: string, opts?: {
-    approvedByHuman?: boolean;
-}): Promise<PromoteResult>;
-export declare function rollbackCandidate(home: string, target: CandidateTarget): Promise<{
-    ok: boolean;
-    error?: string;
-}>;