@hmharness/evolution 0.8.4 → 0.9.0

package/dist/dataset.d.ts

@@ -0,0 +1,64 @@
+export interface DatasetSample {
+    runId: string;
+    task: string;
+    outcome: string;
+    turns: number;
+    toolUses: number;
+    toolFailRate: number;
+    toolsUsed: string[];
+    model: string;
+    /** evidence rank from the M2 ladder when the run was evaluated (0 = none) */
+    evidenceRank: number;
+    /** interpretable reward in [0,1]; llm-judged runs never reach 1.0 (M2 cap) */
+    reward: number;
+    /** evaluation notes / failure snippets attach here (bounded) */
+    evidence?: string;
+    split: 'train' | 'eval';
+}
+export interface DatasetManifest {
+    version: string;
+    createdAt: string;
+    /** inclusive run-id scan window (first..last seen, lexicographic) */
+    runWindow: {
+        first: string | null;
+        last: string | null;
+    };
+    filterFingerprint: string;
+    splitSeed: number;
+    splitRatio: number;
+    counts: {
+        scanned: number;
+        kept: number;
+        dropped: number;
+        duplicates: number;
+        train: number;
+        eval: number;
+    };
+    rewardHistogram: Record<string, number>;
+}
+export interface DatasetBuildResult {
+    manifest: DatasetManifest;
+    dir: string;
+}
+/** Interpretable reward mapping (ADR-0004): ok runs start at 1.0 and lose
+ *  0.1 per 20% tool-failure rate; non-ok runs cap at 0.3. Evidence rank
+ *  from the M2 ladder nudges confidence but never creates a free perfect. */
+export declare function rewardFor(outcome: string, toolFailRate: number, evidenceRank?: number): number;
+/** Stable fingerprint of the filter config - manifest must be reproducible. */
+export declare function filterFingerprint(opts: {
+    keepOutcome?: string[];
+    minTurns?: number;
+}): string;
+/** Deterministic hash-based split (same run always lands in the same half). */
+export declare function splitFor(runId: string, seed: number, evalRatio?: number): 'train' | 'eval';
+export interface BuildDatasetOptions {
+    version?: string;
+    /** outcomes to keep; default keeps everything labeled */
+    keepOutcome?: string[];
+    minTurns?: number;
+    evalRatio?: number;
+    splitSeed?: number;
+}
+export declare function buildDataset(home: string, opts?: BuildDatasetOptions): Promise<DatasetBuildResult>;
+export declare function listDatasets(home: string): Promise<DatasetManifest[]>;
+export declare function loadDataset(home: string, version: string, split?: 'train' | 'eval'): Promise<DatasetSample[]>;

package/dist/dataset.js

@@ -0,0 +1,184 @@
+/**
+ * @hmharness/evolution - dataset (V2 M10: Trajectory → Dataset Version)
+ * Pure data engineering, zero model calls. Pipeline per blueprint:
+ *   Trajectory → Filter → Deduplicate → Label → Evidence Attach → Reward
+ *   → Dataset Version → Train/Eval Split
+ * Source of truth: HMH_HOME/runs/<run-id>/ (M1 trajectory recorder's
+ * summary.json + trajectory.jsonl). Output: versioned, redacted, split
+ * under evolution/datasets/<version>/. See ADR-0004.
+ */
+import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { redactSecrets } from "./insights.js";
+const RUNS_DIR = 'runs';
+/** Interpretable reward mapping (ADR-0004): ok runs start at 1.0 and lose
+ *  0.1 per 20% tool-failure rate; non-ok runs cap at 0.3. Evidence rank
+ *  from the M2 ladder nudges confidence but never creates a free perfect. */
+export function rewardFor(outcome, toolFailRate, evidenceRank = 0) {
+    let r;
+    if (outcome === 'ok') {
+        r = 1.0 - Math.min(0.6, Math.round(toolFailRate * 10) / 10);
+    }
+    else {
+        r = Math.max(0, 0.3 - Math.round(toolFailRate * 10) / 10);
+    }
+    // llm-judge ladder ranks (5+) can never mint a perfect sample
+    if (evidenceRank >= 5)
+        r = Math.min(r, 0.7);
+    return Math.max(0, Math.min(1, r));
+}
+/** Stable fingerprint of the filter config - manifest must be reproducible. */
+export function filterFingerprint(opts) {
+    return createHash('sha256').update(JSON.stringify({
+        keepOutcome: opts.keepOutcome ?? ['ok', 'turn-budget', 'error'],
+        minTurns: opts.minTurns ?? 1,
+        redact: true,
+    })).digest('hex').slice(0, 12);
+}
+/** Normalize both the recorder schema (outcome object + metrics block) and
+ *  the flat test/hand-written shape into one view. */
+function normalizeSummary(s) {
+    const outcome = typeof s.outcome === 'string'
+        ? s.outcome
+        : (s.outcome?.success ? 'ok' : String(s.outcome?.reason ?? ''));
+    return {
+        task: String(s.task ?? ''),
+        outcome,
+        turns: Number(s.turns ?? s.metrics?.turns ?? 0),
+        toolUses: Number(s.toolUses ?? s.metrics?.toolUses ?? 0),
+        toolFailures: Number(s.toolFailures ?? s.metrics?.toolFailures ?? 0),
+        toolsUsed: s.toolsUsed ?? [],
+        model: String(s.model ?? ''),
+        rank: Number(s.evidence?.rank ?? 0),
+        detail: String(s.evidence?.detail ?? ''),
+    };
+}
+async function readRunSummaries(home) {
+    const root = join(home, RUNS_DIR);
+    let ids = [];
+    try {
+        ids = (await readdir(root)).filter((d) => !d.startsWith('.'));
+    }
+    catch {
+        return [];
+    }
+    ids.sort();
+    const out = [];
+    for (const id of ids) {
+        try {
+            const s = JSON.parse(await readFile(join(root, id, 'summary.json'), 'utf8'));
+            out.push({ runId: id, summary: s });
+        }
+        catch { /* torn or absent summary - not a dataset candidate */ }
+    }
+    return out;
+}
+/** Deterministic hash-based split (same run always lands in the same half). */
+export function splitFor(runId, seed, evalRatio = 0.2) {
+    const h = createHash('sha256').update(`${seed}:${runId}`).digest();
+    const v = h.readUInt32BE(0) / 0xffffffff;
+    return v < evalRatio ? 'eval' : 'train';
+}
+export async function buildDataset(home, opts = {}) {
+    const keep = opts.keepOutcome ?? ['ok', 'turn-budget', 'error'];
+    const minTurns = opts.minTurns ?? 1;
+    const evalRatio = opts.evalRatio ?? 0.2;
+    const seed = opts.splitSeed ?? 20260913;
+    const runs = await readRunSummaries(home);
+    const fp = filterFingerprint({ keepOutcome: keep, minTurns });
+    const byFingerprint = new Map();
+    let dropped = 0;
+    let duplicates = 0;
+    for (const { runId, summary } of runs) {
+        const n = normalizeSummary(summary);
+        if (!n.outcome || !keep.includes(n.outcome) || n.turns < minTurns) {
+            dropped++;
+            continue;
+        }
+        const fpRun = createHash('sha256').update(`${n.task}|${n.outcome}|${n.turns}|${n.toolsUsed.join(',')}`, 'utf8').digest('hex').slice(0, 16);
+        if (byFingerprint.has(fpRun)) {
+            duplicates++;
+            continue;
+        } // keep first (oldest) - newest is a rerun of the same shape
+        byFingerprint.set(fpRun, { runId, summary });
+    }
+    const samples = [];
+    const histogram = {};
+    for (const { runId, summary } of byFingerprint.values()) {
+        const n = normalizeSummary(summary);
+        const failRate = n.toolUses > 0 ? n.toolFailures / n.toolUses : 0;
+        const reward = rewardFor(n.outcome, failRate, n.rank);
+        const bucket = reward.toFixed(1);
+        histogram[bucket] = (histogram[bucket] ?? 0) + 1;
+        samples.push({
+            runId,
+            task: redactSecrets(n.task),
+            outcome: n.outcome,
+            turns: n.turns,
+            toolUses: n.toolUses,
+            toolFailRate: Math.round(failRate * 100) / 100,
+            toolsUsed: n.toolsUsed,
+            model: n.model,
+            evidenceRank: n.rank,
+            reward,
+            ...(n.detail ? { evidence: redactSecrets(n.detail).slice(0, 400) } : {}),
+            split: splitFor(runId, seed, evalRatio),
+        });
+    }
+    samples.sort((a, b) => (a.runId < b.runId ? -1 : 1));
+    const version = opts.version ?? `v-${new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)}`;
+    const dir = join(home, 'evolution', 'datasets', version);
+    await mkdir(dir, { recursive: true });
+    await writeFile(join(dir, 'samples.jsonl'), samples.map((s) => JSON.stringify(s)).join('\n') + (samples.length ? '\n' : ''), 'utf8');
+    const manifest = {
+        version,
+        createdAt: new Date().toISOString(),
+        runWindow: { first: runs[0]?.runId ?? null, last: runs[runs.length - 1]?.runId ?? null },
+        filterFingerprint: fp,
+        splitSeed: seed,
+        splitRatio: evalRatio,
+        counts: {
+            scanned: runs.length,
+            kept: samples.length,
+            dropped,
+            duplicates,
+            train: samples.filter((s) => s.split === 'train').length,
+            eval: samples.filter((s) => s.split === 'eval').length,
+        },
+        rewardHistogram: histogram,
+    };
+    await writeFile(join(dir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8');
+    return { manifest, dir };
+}
+export async function listDatasets(home) {
+    const root = join(home, 'evolution', 'datasets');
+    let vers = [];
+    try {
+        vers = (await readdir(root)).filter((d) => d.startsWith('v-'));
+    }
+    catch {
+        return [];
+    }
+    vers.sort();
+    const out = [];
+    for (const v of vers) {
+        try {
+            out.push(JSON.parse(await readFile(join(root, v, 'manifest.json'), 'utf8')));
+        }
+        catch { /* skip torn */ }
+    }
+    return out;
+}
+export async function loadDataset(home, version, split) {
+    const file = join(home, 'evolution', 'datasets', version, 'samples.jsonl');
+    let text;
+    try {
+        text = await readFile(file, 'utf8');
+    }
+    catch {
+        return [];
+    }
+    return text.split('\n').filter(Boolean).map((l) => JSON.parse(l))
+        .filter((s) => !split || s.split === split);
+}

package/dist/index.d.ts

@@ -1,6 +1,8 @@
 export * from './memory.ts';
 export * from './ranker.ts';
 export * from './candidates.ts';
+export * from './dataset.ts';
+export * from './readiness.ts';
 export * from './insights.ts';
 export * from './skills.ts';
 export * from './bench.ts';

package/dist/index.js

@@ -1,6 +1,8 @@
 export * from "./memory.js";
 export * from "./ranker.js";
 export * from "./candidates.js";
+export * from "./dataset.js";
+export * from "./readiness.js";
 export * from "./insights.js";
 export * from "./skills.js";
 export * from "./bench.js";

package/dist/insights.d.ts

@@ -10,6 +10,7 @@ export interface Insight {
      *  session's system prompt - the join key for canary A/B comparison. */
     skillsInjected?: string[];
 }
+export declare function redactSecrets(text: string): string;
 export declare function recordInsight(home: string, insight: Insight): Promise<void>;
 /** Read recent insights as structured records (the evolve loop's raw feed). */
 export declare function readInsights(home: string, limit?: number): Promise<Insight[]>;

package/dist/insights.js

@@ -7,10 +7,31 @@
  */
 import { appendFile, mkdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
+/**
+ * Secret redaction for everything that leaves the machine (insights feed the
+ * PUBLIC evidence page). Users paste API keys into task text ("新增 provider,
+ * sk-… 给 hmharness") and the audit trail must never publish them. Cover the
+ * shapes seen in the wild: OpenAI-style sk-, VolcEngine ark-<uuid>-<hex>,
+ * GitHub ghp_/npm_ tokens. GitHub Push Protection caught this class once
+ * (GH013, VolcEngine Ark) - it must be caught here first.
+ */
+const SECRET_PATTERNS = [
+    { re: /sk-[A-Za-z0-9]{20,}/g, label: 'sk-[REDACTED]' },
+    { re: /ark-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}-[0-9a-fA-F]{4,}/g, label: 'ark-[REDACTED]' },
+    { re: /gh[pousr]_[A-Za-z0-9]{20,}/g, label: 'ghp_[REDACTED]' },
+    { re: /npm_[A-Za-z0-9]{20,}/g, label: 'npm_[REDACTED]' },
+];
+export function redactSecrets(text) {
+    let out = text;
+    for (const p of SECRET_PATTERNS)
+        out = out.replace(p.re, p.label);
+    return out;
+}
 export async function recordInsight(home, insight) {
     const dir = join(home, 'insights');
     await mkdir(dir, { recursive: true });
-    await appendFile(join(dir, 'insights.jsonl'), JSON.stringify(insight) + '\n', 'utf8');
+    const clean = { ...insight, task: redactSecrets(insight.task) };
+    await appendFile(join(dir, 'insights.jsonl'), JSON.stringify(clean) + '\n', 'utf8');
 }
 /** Read recent insights as structured records (the evolve loop's raw feed). */
 export async function readInsights(home, limit = 40) {

package/dist/readiness.d.ts

@@ -0,0 +1,15 @@
+export interface ReadinessCondition {
+    id: string;
+    met: boolean;
+    current: number | string | boolean;
+    threshold: number | string;
+    evidence: string;
+}
+export interface ReadinessReport {
+    verdict: 'rl-eligible' | 'optimize-first';
+    conditions: ReadinessCondition[];
+    /** when optimize-first: which lever to pull first (weakest condition) */
+    recommendedLever?: 'data-collection' | 'bench-expansion' | 'reward-calibration' | 'eval-suite-stabilization' | 'version-provenance';
+    at: string;
+}
+export declare function rlReadiness(home: string): Promise<ReadinessReport>;

package/dist/readiness.js

@@ -0,0 +1,155 @@
+/**
+ * @hmharness/evolution - readiness (V2 M11: RL 前置条件检查器)
+ * The blueprint's gate: RL is allowed ONLY when all six conditions hold.
+ * Every condition is measured from real on-disk evidence - nothing is
+ * estimated, nothing can be waved through. When the gate is closed the
+ * report says exactly which optimization to do instead. See ADR-0005.
+ */
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { listCases } from "./bench.js";
+const QUALITY_MIN_REWARD = 0.5;
+async function countRunTrajectories(home) {
+    const root = join(home, 'runs');
+    let ids = [];
+    try {
+        ids = (await readdir(root)).filter((d) => !d.startsWith('.'));
+    }
+    catch {
+        return { total: 0, highQuality: 0 };
+    }
+    let high = 0;
+    for (const id of ids) {
+        try {
+            const s = JSON.parse(await readFile(join(root, id, 'summary.json'), 'utf8'));
+            const outcome = typeof s.outcome === 'string' ? s.outcome : (s.outcome?.success ? 'ok' : String(s.outcome?.reason ?? ''));
+            const uses = Number(s.toolUses ?? s.metrics?.toolUses ?? 0);
+            const fails = Number(s.toolFailures ?? s.metrics?.toolFailures ?? 0);
+            const failRate = uses > 0 ? fails / uses : 0;
+            const reward = outcome === 'ok' ? 1 - Math.min(0.6, Math.round(failRate * 10) / 10) : 0.3;
+            if (outcome && (s.turns ?? s.metrics?.turns ?? 0) >= 1 && reward >= QUALITY_MIN_REWARD)
+                high++;
+        }
+        catch { /* torn */ }
+    }
+    return { total: ids.length, highQuality: high };
+}
+export async function rlReadiness(home) {
+    const conditions = [];
+    // 1. >= 1000 high-quality trajectories
+    const runs = await countRunTrajectories(home);
+    conditions.push({
+        id: 'high-quality-trajectories',
+        met: runs.highQuality >= 1000,
+        current: runs.highQuality,
+        threshold: 1000,
+        evidence: `runs/ scan: ${runs.total} trajectories, ${runs.highQuality} with outcome + turns>=1 + reward>=${QUALITY_MIN_REWARD}`,
+    });
+    // 2. >= 100 stable benchmark tasks
+    let cases = 0;
+    try {
+        cases = (await listCases(home)).length;
+    }
+    catch { /* none */ }
+    conditions.push({
+        id: 'stable-benchmark-tasks',
+        met: cases >= 100,
+        current: cases,
+        threshold: 100,
+        evidence: `bench/cases/ scan: ${cases} cases (holdout included; stability tracked by bench history)`,
+    });
+    // 3. reward vs human judgment correlation VERIFIED - no human-labeling
+    //    channel exists yet, so this is honestly not met (never a guess)
+    let humanLabels = 0;
+    try {
+        const f = await stat(join(home, 'evolution', 'reward-human-labels.jsonl'));
+        if (f.isFile())
+            humanLabels = (await readFile(join(home, 'evolution', 'reward-human-labels.jsonl'), 'utf8')).split('\n').filter(Boolean).length;
+    }
+    catch { /* absent */ }
+    conditions.push({
+        id: 'reward-human-correlation',
+        met: humanLabels >= 100,
+        current: humanLabels,
+        threshold: 100,
+        evidence: 'evolution/reward-human-labels.jsonl (human-scored samples; dataset label field reserves the slot)',
+    });
+    // 4. evaluation regression suite STABLE - last two bench records, no regression
+    let suiteStable = false;
+    let suiteEvidence = 'no bench history records (evolution/benches/ empty)';
+    try {
+        const dir = join(home, 'evolution', 'benches');
+        const files = (await readdir(dir)).filter((f) => f.endsWith('.json')).sort();
+        if (files.length >= 2) {
+            const last = JSON.parse(await readFile(join(dir, files[files.length - 1]), 'utf8'));
+            const prev = JSON.parse(await readFile(join(dir, files[files.length - 2]), 'utf8'));
+            if (typeof last.passRate === 'number' && typeof prev.passRate === 'number') {
+                suiteStable = last.passRate >= prev.passRate - 0.05;
+                suiteEvidence = `last two bench passRates: ${(prev.passRate * 100).toFixed(0)}% -> ${(last.passRate * 100).toFixed(0)}% (tolerance -5%)`;
+            }
+        }
+        else if (files.length === 1) {
+            suiteEvidence = 'only one bench record - stability needs at least two';
+        }
+    }
+    catch { /* absent */ }
+    conditions.push({
+        id: 'eval-regression-stable',
+        met: suiteStable,
+        current: suiteStable,
+        threshold: 'no >5% passRate regression across last two bench runs',
+        evidence: suiteEvidence,
+    });
+    // 5. model/skill/workflow version provenance
+    const provenance = {
+        model: true, // every run's summary carries model; rollout session_meta pins it
+        skills: false,
+        workflows: false,
+    };
+    try {
+        const skills = await readdir(join(home, 'skills'));
+        provenance.skills = skills.some((s) => s.endsWith('.md'));
+    }
+    catch { /* none */ }
+    try {
+        const wf = await readdir(join(home, 'evolution', 'workflows'));
+        provenance.workflows = wf.length > 0;
+    }
+    catch { /* none */ }
+    conditions.push({
+        id: 'version-provenance',
+        met: provenance.skills && provenance.workflows,
+        current: provenance.skills && provenance.workflows,
+        threshold: 'skills + workflows under version control (model provenance ships with every run)',
+        evidence: `skills dir: ${provenance.skills ? 'versioned .md files' : 'empty'}; workflows dir: ${provenance.workflows ? 'present' : 'empty'}`,
+    });
+    // 6. offline evaluation + holdout set exist
+    let holdout = 0;
+    try {
+        holdout = (await listCases(home)).filter((c) => c.holdout).length;
+    }
+    catch { /* none */ }
+    conditions.push({
+        id: 'offline-eval-holdout',
+        met: holdout > 0 && cases > 0,
+        current: holdout,
+        threshold: '>0 holdout cases in a non-empty bench suite',
+        evidence: `bench/cases/: ${cases} total, ${holdout} holdout`,
+    });
+    const allMet = conditions.every((c) => c.met);
+    const weakest = conditions.find((c) => !c.met);
+    const leverMap = {
+        'high-quality-trajectories': 'data-collection',
+        'stable-benchmark-tasks': 'bench-expansion',
+        'reward-human-correlation': 'reward-calibration',
+        'eval-regression-stable': 'eval-suite-stabilization',
+        'version-provenance': 'version-provenance',
+        'offline-eval-holdout': 'bench-expansion',
+    };
+    return {
+        verdict: allMet ? 'rl-eligible' : 'optimize-first',
+        conditions,
+        ...(allMet ? {} : { recommendedLever: weakest ? leverMap[weakest.id] : undefined }),
+        at: new Date().toISOString(),
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evolution",
-  "version": "0.8.4",
+  "version": "0.9.0",
   "description": "hmharness evolution subsystem: persistent memory, insight capture, skill library, and the bench that gives evolution its fitness signal. First-class kernel citizen, not a plugin.",
   "type": "module",
   "main": "dist/index.js",
@@ -15,7 +15,7 @@
     "build": "tsc -p tsconfig.build.json"
   },
   "dependencies": {
-    "@hmharness/kernel": "0.8.2"
+    "@hmharness/kernel": "0.9.0"
   },
   "files": [
     "dist"