@hmharness/evolution 0.14.5 → 0.14.9

package/dist/dataset.js.v2bak.v2bak

@@ -0,0 +1,184 @@
+/**
+ * @hmharness/evolution - dataset (V2 M10: Trajectory → Dataset Version)
+ * Pure data engineering, zero model calls. Pipeline per blueprint:
+ *   Trajectory → Filter → Deduplicate → Label → Evidence Attach → Reward
+ *   → Dataset Version → Train/Eval Split
+ * Source of truth: HMH_HOME/runs/<run-id>/ (M1 trajectory recorder's
+ * summary.json + trajectory.jsonl). Output: versioned, redacted, split
+ * under evolution/datasets/<version>/. See ADR-0004.
+ */
+import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { redactSecrets } from "./insights.js";
+const RUNS_DIR = 'runs';
+/** Interpretable reward mapping (ADR-0004): ok runs start at 1.0 and lose
+ *  0.1 per 20% tool-failure rate; non-ok runs cap at 0.3. Evidence rank
+ *  from the M2 ladder nudges confidence but never creates a free perfect. */
+export function rewardFor(outcome, toolFailRate, evidenceRank = 0) {
+    let r;
+    if (outcome === 'ok') {
+        r = 1.0 - Math.min(0.6, Math.round(toolFailRate * 10) / 10);
+    }
+    else {
+        r = Math.max(0, 0.3 - Math.round(toolFailRate * 10) / 10);
+    }
+    // llm-judge ladder ranks (5+) can never mint a perfect sample
+    if (evidenceRank >= 5)
+        r = Math.min(r, 0.7);
+    return Math.max(0, Math.min(1, r));
+}
+/** Stable fingerprint of the filter config - manifest must be reproducible. */
+export function filterFingerprint(opts) {
+    return createHash('sha256').update(JSON.stringify({
+        keepOutcome: opts.keepOutcome ?? ['ok', 'turn-budget', 'error'],
+        minTurns: opts.minTurns ?? 1,
+        redact: true,
+    })).digest('hex').slice(0, 12);
+}
+/** Normalize both the recorder schema (outcome object + metrics block) and
+ *  the flat test/hand-written shape into one view. */
+function normalizeSummary(s) {
+    const outcome = typeof s.outcome === 'string'
+        ? s.outcome
+        : (s.outcome?.success ? 'ok' : String(s.outcome?.reason ?? ''));
+    return {
+        task: String(s.task ?? ''),
+        outcome,
+        turns: Number(s.turns ?? s.metrics?.turns ?? 0),
+        toolUses: Number(s.toolUses ?? s.metrics?.toolUses ?? 0),
+        toolFailures: Number(s.toolFailures ?? s.metrics?.toolFailures ?? 0),
+        toolsUsed: s.toolsUsed ?? [],
+        model: String(s.model ?? ''),
+        rank: Number(s.evidence?.rank ?? 0),
+        detail: String(s.evidence?.detail ?? ''),
+    };
+}
+async function readRunSummaries(home) {
+    const root = join(home, RUNS_DIR);
+    let ids = [];
+    try {
+        ids = (await readdir(root)).filter((d) => !d.startsWith('.'));
+    }
+    catch {
+        return [];
+    }
+    ids.sort();
+    const out = [];
+    for (const id of ids) {
+        try {
+            const s = JSON.parse(await readFile(join(root, id, 'summary.json'), 'utf8'));
+            out.push({ runId: id, summary: s });
+        }
+        catch { /* torn or absent summary - not a dataset candidate */ }
+    }
+    return out;
+}
+/** Deterministic hash-based split (same run always lands in the same half). */
+export function splitFor(runId, seed, evalRatio = 0.2) {
+    const h = createHash('sha256').update(`${seed}:${runId}`).digest();
+    const v = h.readUInt32BE(0) / 0xffffffff;
+    return v < evalRatio ? 'eval' : 'train';
+}
+export async function buildDataset(home, opts = {}) {
+    const keep = opts.keepOutcome ?? ['ok', 'turn-budget', 'error'];
+    const minTurns = opts.minTurns ?? 1;
+    const evalRatio = opts.evalRatio ?? 0.2;
+    const seed = opts.splitSeed ?? 20260913;
+    const runs = await readRunSummaries(home);
+    const fp = filterFingerprint({ keepOutcome: keep, minTurns });
+    const byFingerprint = new Map();
+    let dropped = 0;
+    let duplicates = 0;
+    for (const { runId, summary } of runs) {
+        const n = normalizeSummary(summary);
+        if (!n.outcome || !keep.includes(n.outcome) || n.turns < minTurns) {
+            dropped++;
+            continue;
+        }
+        const fpRun = createHash('sha256').update(`${n.task}|${n.outcome}|${n.turns}|${n.toolsUsed.join(',')}`, 'utf8').digest('hex').slice(0, 16);
+        if (byFingerprint.has(fpRun)) {
+            duplicates++;
+            continue;
+        } // keep first (oldest) - newest is a rerun of the same shape
+        byFingerprint.set(fpRun, { runId, summary });
+    }
+    const samples = [];
+    const histogram = {};
+    for (const { runId, summary } of byFingerprint.values()) {
+        const n = normalizeSummary(summary);
+        const failRate = n.toolUses > 0 ? n.toolFailures / n.toolUses : 0;
+        const reward = rewardFor(n.outcome, failRate, n.rank);
+        const bucket = reward.toFixed(1);
+        histogram[bucket] = (histogram[bucket] ?? 0) + 1;
+        samples.push({
+            runId,
+            task: redactSecrets(n.task),
+            outcome: n.outcome,
+            turns: n.turns,
+            toolUses: n.toolUses,
+            toolFailRate: Math.round(failRate * 100) / 100,
+            toolsUsed: n.toolsUsed,
+            model: n.model,
+            evidenceRank: n.rank,
+            reward,
+            ...(n.detail ? { evidence: redactSecrets(n.detail).slice(0, 400) } : {}),
+            split: splitFor(runId, seed, evalRatio),
+        });
+    }
+    samples.sort((a, b) => (a.runId < b.runId ? -1 : 1));
+    const version = opts.version ?? `v-${new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)}`;
+    const dir = join(home, 'evolution', 'datasets', version);
+    await mkdir(dir, { recursive: true });
+    await writeFile(join(dir, 'samples.jsonl'), samples.map((s) => JSON.stringify(s)).join('\n') + (samples.length ? '\n' : ''), 'utf8');
+    const manifest = {
+        version,
+        createdAt: new Date().toISOString(),
+        runWindow: { first: runs[0]?.runId ?? null, last: runs[runs.length - 1]?.runId ?? null },
+        filterFingerprint: fp,
+        splitSeed: seed,
+        splitRatio: evalRatio,
+        counts: {
+            scanned: runs.length,
+            kept: samples.length,
+            dropped,
+            duplicates,
+            train: samples.filter((s) => s.split === 'train').length,
+            eval: samples.filter((s) => s.split === 'eval').length,
+        },
+        rewardHistogram: histogram,
+    };
+    await writeFile(join(dir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8');
+    return { manifest, dir };
+}
+export async function listDatasets(home) {
+    const root = join(home, 'evolution', 'datasets');
+    let vers = [];
+    try {
+        vers = (await readdir(root)).filter((d) => d.startsWith('v-'));
+    }
+    catch {
+        return [];
+    }
+    vers.sort();
+    const out = [];
+    for (const v of vers) {
+        try {
+            out.push(JSON.parse(await readFile(join(root, v, 'manifest.json'), 'utf8')));
+        }
+        catch { /* skip torn */ }
+    }
+    return out;
+}
+export async function loadDataset(home, version, split) {
+    const file = join(home, 'evolution', 'datasets', version, 'samples.jsonl');
+    let text;
+    try {
+        text = await readFile(file, 'utf8');
+    }
+    catch {
+        return [];
+    }
+    return text.split('\n').filter(Boolean).map((l) => JSON.parse(l))
+        .filter((s) => !split || s.split === split);
+}

package/dist/evolve.d.ts

@@ -95,4 +95,18 @@ export declare function runEvolution(opts: {
     presetProposals?: SkillProposal[];
     log?: (line: string) => void;
 }): Promise<EvolveReport>;
+/** Cost-regression veto (pass-by-rambling detector). Multiplicative cap
+ *  alone is a HAIR TRIGGER on microscopic outputs: a case whose expected
+ *  answer is ~4 tokens (cjk-ex-9: 等待中……) allows ~1 token of slack at
+ *  1.3x, so ANY skill that perturbs output by one character vetoes the
+ *  candidate — ten consecutive candidates died exactly there. An absolute
+ *  slack floor (+8 tokens) keeps the intent (real rambling is 2-10x) while
+ *  letting tiny-output cases tolerate needle-scale perturbation. */
+export declare function isCostRegression(base: number, cand: number, cap?: number): boolean;
+/** Dual-metric veto helper: names of cases where the candidate passed but
+ *  cost more than its baseline cap (rambling). Pure - unit-tested. */
+export declare function costRegressionNames(train: Array<{
+    name: string;
+    costCap?: number;
+}>, baseCost: Record<string, number>, candCost: Record<string, number>): string[];
 export declare function screenForPoison(text: string): string | null;

package/dist/evolve.d.ts.v2bak.v2bak

@@ -0,0 +1,98 @@
+import { type ProviderConfig } from '@hmharness/kernel';
+import { type BenchCase } from './bench.ts';
+export interface SkillProposal {
+    name: string;
+    description: string;
+    skill_md: string;
+}
+export interface ProposalOutcome {
+    name: string;
+    action: 'promoted' | 'rejected' | 'error';
+    reason: string;
+    baseline?: {
+        passRate: number;
+        cases: string;
+    };
+    candidate?: {
+        passRate: number;
+        cases: string;
+    };
+    holdout?: {
+        baselineRate: number;
+        candidateRate: number;
+    };
+    /** P0 lineage ledger: what fed this decision (provenance for impact
+     *  attribution - which insights produced which skill, with which scores).
+     *  Evolution-artifact genealogy is the GEPA/DGM archive lesson: decisions
+     *  without ancestry cannot be audited for objective hacking. */
+    lineage?: {
+        parentInsights: string[];
+        scores: {
+            train: number;
+            holdout?: number;
+        };
+        metaModel: string;
+        decidedAt: string;
+    };
+}
+export interface EvolveReport {
+    time: string;
+    model: string;
+    seededCases: string[];
+    insightCount: number;
+    noteCount: number;
+    proposals: SkillProposal[];
+    outcomes: ProposalOutcome[];
+    memoryDistilled: string | null;
+    /** code-level evolution: proposed patches (DGM bridge) */
+    codePatches?: Array<{
+        name: string;
+        file: string;
+        reason: string;
+    }>;
+    /** code-level outcomes: merged/reverted/error per patch */
+    patchOutcomes?: Array<{
+        name: string;
+        action: string;
+        reason: string;
+        branch?: string;
+    }>;
+    /** P0 impact attribution: canary A/B comparison applied this cycle */
+    impact?: {
+        rows: Array<{
+            skill: string;
+            exposed: string;
+            control: string;
+            verdict: string;
+        }>;
+        applied: string[];
+    };
+    /** P1 lifecycle: skills moved to dormant this cycle */
+    decayed?: string[];
+    /** budget state at cycle start (observability) */
+    budget?: {
+        cyclesToday: number;
+        maxCyclesPerDay?: number;
+    };
+    /** chars/4 estimate of this cycle's meta-call traffic; feeds the daily
+     *  token budget gate (readBudget sums today's entries) */
+    estTokens?: number;
+}
+/** Runs one bench case with the given skills block injected. */
+export type CaseRunner = (c: BenchCase, skillsPrompt: string) => Promise<string>;
+/** Rough token estimate, language-aware: ASCII runs ~4 chars/token, CJK
+ * ~1 char/token. A flat chars/4 undercounted Chinese 2-4x, letting verbose
+ * zh candidates dodge the cost cap. Used on BOTH sides of every comparison
+ * (baseline and candidate), never as the only rejection reason (the
+ * pass-rate gate decides; cost-cap only vetoes pass-by-rambling). */
+export declare function estTokens(text: string): number;
+export declare function runEvolution(opts: {
+    home: string;
+    provider: ProviderConfig;
+    runCase: CaseRunner;
+    maxProposals?: number;
+    /** Skip the meta-model call and evaluate these proposals directly (tests / future UIs). */
+    presetProposals?: SkillProposal[];
+    log?: (line: string) => void;
+}): Promise<EvolveReport>;
+export declare function screenForPoison(text: string): string | null;

package/dist/evolve.js

@@ -167,14 +167,18 @@ export async function runEvolution(opts) {
     }
     else {
         proposals = await proposeSkills(provider, signals, say, ancestor ?? undefined, mergeWith ?? undefined);
-        if (proposals.length === 0) {
-            try {
-                const { workflowProposals } = await import("./workflows.js");
-                proposals = await workflowProposals(provider, home, say);
-            }
-            catch (err) {
-                say(`  awm skipped: ${String(err).slice(0, 80)}`);
-            }
+        // AWM is ADDITIVE, not fallback-only (design intent: "higher abstraction
+        // than per-mistake reflection"). With batch-produced task archetypes
+        // repeating 100+ times/day, the workflow path must get its turn even
+        // when the meta-model also proposes - the -workflow naming + gate still
+        // decides promotion. Guard: only append when clusters are real (>=5).
+        try {
+            const { workflowProposals } = await import("./workflows.js");
+            const wf = await workflowProposals(provider, home, say);
+            proposals = [...proposals, ...wf];
+        }
+        catch (err) {
+            say(`  awm skipped: ${String(err).slice(0, 80)}`);
         }
     }
     report.proposals = proposals;
@@ -227,13 +231,9 @@ export async function runEvolution(opts) {
             const regression = baseResults.some((b) => b.pass && !candResults.find((c) => c.name === b.name)?.pass);
             // dual-metric veto: a passing case that costs > cost-cap x its
             // baseline counts as a cost regression (the candidate passed by
-            // rambling) - only enforced where the case declares a cost-cap
-            const costRegressions = train.filter((c) => {
-                const cap = c.costCap ?? 1.3; // default 1.3x for all cases
-                const base = baseCost[c.name] ?? 0;
-                const cand = candCost[c.name] ?? 0;
-                return base > 0 && cand > base * cap;
-            }).map((c) => c.name);
+            // rambling); isCostRegression carries the absolute-slack floor that
+            // keeps microscopic-output cases from hair-triggering
+            const costRegressions = costRegressionNames(train, baseCost, candCost);
             const summary = (rs) => rs.map((r) => `${r.name}:${r.pass ? 'pass' : 'FAIL'}`).join(' ');
             if (regression || candRate < baseRate) {
                 await deleteDraft(home, p.name);
@@ -455,6 +455,7 @@ async function proposeSkills(provider, signals, say, ancestor, mergeWith) {
         'Your job: read session signals and decide whether any repeatable procedure is worth crystallizing into a skill.',
         'A skill is a markdown how-to document the agent reads on demand. Topics must be limited to: HarmonyOS toolchain usage (hdc/hvigorw/ohpm/DevEco), this framework\'s tools (list_dir/read_file/write_file/run_command/remember/harmony_*), and reusable task workflows observed in the signals.',
         'Rules: name is kebab-case; description is one line; skill_md is at most 60 lines with concrete steps and example commands; do NOT propose skills about security config, approval policy, or anything outside the topics; if nothing is genuinely reusable, return an empty array.',
+        'COST GATE (learned from repeated rejections): every candidate runs against an output-cost cap - a skill whose text makes the agent MORE verbose on exact-output cases (it restates, adds preambles, or explains around the answer) is auto-rejected even when correctness improves. Write TERSE imperative steps (target under 25 lines): checklists and example commands only; never instruct the agent to add explanations, context, or restatements; the skill itself must teach doing LESS talking, not more.',
         'Respond with ONLY a JSON array: [{"name":"...","description":"...","skill_md":"..."}] - no prose, no code fences.',
     ].join('\n');
     // GEPA ancestor context: vary around a past rejection (its reason is the
@@ -507,6 +508,26 @@ async function distillMemory(provider, recentNotes, say) {
         return null;
     }
 }
+/** Cost-regression veto (pass-by-rambling detector). Multiplicative cap
+ *  alone is a HAIR TRIGGER on microscopic outputs: a case whose expected
+ *  answer is ~4 tokens (cjk-ex-9: 等待中……) allows ~1 token of slack at
+ *  1.3x, so ANY skill that perturbs output by one character vetoes the
+ *  candidate — ten consecutive candidates died exactly there. An absolute
+ *  slack floor (+8 tokens) keeps the intent (real rambling is 2-10x) while
+ *  letting tiny-output cases tolerate needle-scale perturbation. */
+export function isCostRegression(base, cand, cap = 1.3) {
+    if (base <= 0)
+        return false;
+    return cand > Math.max(base * cap, base + 8);
+}
+/** Dual-metric veto helper: names of cases where the candidate passed but
+ *  cost more than its baseline cap (rambling). Pure - unit-tested. */
+export function costRegressionNames(train, baseCost, candCost) {
+    return train.filter((c) => {
+        const cap = c.costCap ?? 1.3;
+        return isCostRegression(baseCost[c.name] ?? 0, candCost[c.name] ?? 0, cap);
+    }).map((c) => c.name);
+}
 /** First balanced JSON array in the text; tolerates fences and prose around it. */
 function parseJsonArray(text) {
     const cleaned = text.replace(/```(?:json)?/g, '');