@hmharness/evolution 0.1.0

package/dist/bench.d.ts

@@ -0,0 +1,38 @@
+export interface BenchCase {
+    name: string;
+    prompt: string;
+    /** All substrings (split on &&) must appear in the output, case-insensitive. */
+    expect: string[];
+    /** Output must equal this string exactly (both trimmed). */
+    expectExact?: string;
+    /** Output must match this regex (whole output, case-sensitive). */
+    expectRegex?: string;
+    /** Substrings that must NOT appear (failure-marker exclusion). */
+    expectNone?: string[];
+    /** At least ONE of these substrings must appear. */
+    expectAny?: string[];
+    /** When true the case runs through the full agent loop with tools. */
+    tools: boolean;
+    /** Holdout cases are excluded from the promotion gate and re-verify after promotion (anti-memorization, GDPevo style). */
+    holdout: boolean;
+    /** Candidate token-cost ceiling as a multiple of the baseline run. */
+    costCap?: number;
+}
+export interface BenchResult {
+    name: string;
+    pass: boolean;
+    detail: string;
+}
+export declare function matchExpect(output: string, expect: string[]): boolean;
+/** Full structured assertion: every declared mode must hold. */
+export declare function matchCase(output: string, c: Pick<BenchCase, 'expect' | 'expectExact' | 'expectRegex' | 'expectNone' | 'expectAny'>): {
+    pass: boolean;
+    detail: string;
+};
+export declare function listCases(home: string): Promise<BenchCase[]>;
+export declare function runBench(home: string, run: (c: BenchCase) => Promise<string>): Promise<{
+    results: BenchResult[];
+    passRate: number;
+}>;
+/** Starter cases so the evolve gate has a signal from day one. */
+export declare function seedCases(home: string): Promise<string[]>;

package/dist/bench.js

@@ -0,0 +1,130 @@
+/**
+ * @hmharness/evolution - bench
+ * The fitness signal for self-evolution. bench/cases/*.task files are:
+ *   line 1        the prompt
+ *   expect: a && b   substrings that must ALL appear in the final output
+ *   expect-exact: "..."   the output equals this string (trimmed)
+ *   expect-regex: ...     a regex the output must match
+ *   expect-none: a && b   substrings that must NOT appear
+ *   expect-any: a || b    one of these substrings must appear
+ *   tools: loop   (optional) run through the full agent loop with tools
+ *   cost-cap: 1.3  (optional) candidate cost multiplier ceiling vs baseline
+ * Skill or prompt changes must keep the bench green before promotion - the
+ * evolve loop enforces this A/B (baseline vs candidate).
+ *
+ * Assertion upgrade (gate methodology): plain substring matching lets
+ * verbose-but-wrong outputs pass; the structured modes pin exact shapes,
+ * forbid failure markers, and (with cost-cap) stop a candidate that only
+ * passes by burning 3x tokens. Old files keep working - `expect:` keeps
+ * its ALL-substrings semantics.
+ */
+import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+export function matchExpect(output, expect) {
+    const lower = output.toLowerCase();
+    return expect.every((e) => lower.includes(e.toLowerCase()));
+}
+/** Full structured assertion: every declared mode must hold. */
+export function matchCase(output, c) {
+    if (c.expectExact !== undefined && output.trim() !== c.expectExact.trim()) {
+        return { pass: false, detail: `exact mismatch: got "${output.trim().slice(0, 80)}"` };
+    }
+    if (c.expectRegex !== undefined) {
+        try {
+            if (!new RegExp(c.expectRegex).test(output))
+                return { pass: false, detail: `regex mismatch: /${c.expectRegex.slice(0, 60)}/` };
+        }
+        catch {
+            return { pass: false, detail: `invalid regex in case: ${c.expectRegex.slice(0, 40)}` };
+        }
+    }
+    if (c.expectNone && c.expectNone.some((e) => output.toLowerCase().includes(e.toLowerCase()))) {
+        return { pass: false, detail: `forbidden marker present: ${c.expectNone.join(' && ')}` };
+    }
+    if (c.expectAny && !c.expectAny.some((e) => output.toLowerCase().includes(e.toLowerCase()))) {
+        return { pass: false, detail: `none of the allowed markers found: ${c.expectAny.join(' || ')}` };
+    }
+    if (!matchExpect(output, c.expect)) {
+        return { pass: false, detail: `missing "${c.expect.join('" && "')}" in output` };
+    }
+    return { pass: true, detail: 'ok' };
+}
+export async function listCases(home) {
+    const dir = join(home, 'bench', 'cases');
+    let files;
+    try {
+        files = (await readdir(dir)).filter((f) => f.endsWith('.task'));
+    }
+    catch {
+        return [];
+    }
+    const cases = [];
+    for (const f of files.sort()) {
+        const raw = (await readFile(join(dir, f), 'utf8')).trim();
+        const lines = raw.split('\n');
+        const prompt = lines[0];
+        const pick = (key) => {
+            const l = lines.find((x) => x.startsWith(key + ':'));
+            return l ? l.slice(key.length + 1).trim() : undefined;
+        };
+        const list = (key) => {
+            const v = pick(key);
+            return v ? v.split(/&&|\|\|/).map((s) => s.trim()).filter(Boolean) : undefined;
+        };
+        const costCap = Number(pick('cost-cap'));
+        cases.push({
+            name: f.replace(/\.task$/, ''),
+            prompt,
+            expect: list('expect') ?? [],
+            expectExact: pick('expect-exact')?.replace(/^"|"$/g, ''),
+            expectRegex: pick('expect-regex'),
+            expectNone: list('expect-none'),
+            expectAny: list('expect-any'),
+            tools: pick('tools') === 'loop',
+            holdout: pick('holdout') === 'true',
+            costCap: Number.isFinite(costCap) && costCap > 0 ? costCap : undefined,
+        });
+    }
+    return cases;
+}
+export async function runBench(home, run) {
+    const cases = await listCases(home);
+    const results = [];
+    for (const c of cases) {
+        try {
+            const out = await run(c);
+            const r = matchCase(out, c);
+            results.push({ name: c.name, pass: r.pass, detail: r.detail });
+        }
+        catch (err) {
+            results.push({ name: c.name, pass: false, detail: String(err).slice(0, 120) });
+        }
+    }
+    const passed = results.filter((r) => r.pass).length;
+    return { results, passRate: cases.length === 0 ? 1 : passed / cases.length };
+}
+/** Starter cases so the evolve gate has a signal from day one. */
+export async function seedCases(home) {
+    const dir = join(home, 'bench', 'cases');
+    const written = [];
+    const seeds = [
+        ['toolchain-report.task', '运行鸿蒙工具链体检,然后逐项说出一共检查了哪三个工具的名称\nexpect: hdc && hvigorw && ohpm\ntools: loop\n'],
+        ['reply-determinism.task', '只回复这六个字符,不要任何其他内容:HMH-OK\nexpect: HMH-OK\n'],
+        ['toolchain-ohpm-status.task', '运行鸿蒙工具链体检,然后只回答 ohpm 的状态是 OK 还是 MISSING\nexpect: OK\ntools: loop\nholdout: true\n'],
+    ];
+    let existing = [];
+    try {
+        existing = await readdir(dir);
+    }
+    catch {
+        /* first seed */
+    }
+    for (const [name, body] of seeds) {
+        if (existing.includes(name))
+            continue;
+        await mkdir(dir, { recursive: true });
+        await writeFile(join(dir, name), body, 'utf8');
+        written.push(name);
+    }
+    return written;
+}

package/dist/evolve.d.ts

@@ -0,0 +1,89 @@
+import { type ProviderConfig } from '@hmharness/kernel';
+import { type BenchCase } from './bench.ts';
+export interface SkillProposal {
+    name: string;
+    description: string;
+    skill_md: string;
+}
+export interface ProposalOutcome {
+    name: string;
+    action: 'promoted' | 'rejected' | 'error';
+    reason: string;
+    baseline?: {
+        passRate: number;
+        cases: string;
+    };
+    candidate?: {
+        passRate: number;
+        cases: string;
+    };
+    holdout?: {
+        baselineRate: number;
+        candidateRate: number;
+    };
+    /** P0 lineage ledger: what fed this decision (provenance for impact
+     *  attribution - which insights produced which skill, with which scores).
+     *  Evolution-artifact genealogy is the GEPA/DGM archive lesson: decisions
+     *  without ancestry cannot be audited for objective hacking. */
+    lineage?: {
+        parentInsights: string[];
+        scores: {
+            train: number;
+            holdout?: number;
+        };
+        metaModel: string;
+        decidedAt: string;
+    };
+}
+export interface EvolveReport {
+    time: string;
+    model: string;
+    seededCases: string[];
+    insightCount: number;
+    noteCount: number;
+    proposals: SkillProposal[];
+    outcomes: ProposalOutcome[];
+    memoryDistilled: string | null;
+    /** code-level evolution: proposed patches (DGM bridge) */
+    codePatches?: Array<{
+        name: string;
+        file: string;
+        reason: string;
+    }>;
+    /** code-level outcomes: merged/reverted/error per patch */
+    patchOutcomes?: Array<{
+        name: string;
+        action: string;
+        reason: string;
+        branch?: string;
+    }>;
+    /** P0 impact attribution: canary A/B comparison applied this cycle */
+    impact?: {
+        rows: Array<{
+            skill: string;
+            exposed: string;
+            control: string;
+            verdict: string;
+        }>;
+        applied: string[];
+    };
+    /** P1 lifecycle: skills moved to dormant this cycle */
+    decayed?: string[];
+    /** budget state at cycle start (observability) */
+    budget?: {
+        cyclesToday: number;
+        maxCyclesPerDay?: number;
+    };
+}
+/** Runs one bench case with the given skills block injected. */
+export type CaseRunner = (c: BenchCase, skillsPrompt: string) => Promise<string>;
+export declare function runEvolution(opts: {
+    home: string;
+    provider: ProviderConfig;
+    runCase: CaseRunner;
+    maxProposals?: number;
+    /** Skip the meta-model call and evaluate these proposals directly (tests / future UIs). */
+    presetProposals?: SkillProposal[];
+    log?: (line: string) => void;
+}): Promise<EvolveReport>;
+export declare function screenForPoison(text: string): string | null;