@hmharness/evolution 0.23.43 → 0.23.45

package/dist/sanitize.d.ts

@@ -0,0 +1,29 @@
+export declare const PLACEHOLDER = "<user>";
+/** The username to strip, derived from the home directory's last segment.
+ *  Returned lowercase-free (as-is); matching is case-insensitive anyway. */
+export declare function currentUsername(): string;
+/** Replace every case-insensitive occurrence of `username` with `<user>`.
+ *  Pure string surgery — no regex feature beyond the escaped literal. */
+export declare function sanitizeText(text: string, username: string): string;
+/** Heuristic text/binary split: a NUL byte or a high share of lone-ish
+ *  control bytes means binary. Text evidence (md/json/task/jsonl/log/txt)
+ *  sails through; png/zips copy verbatim. */
+export declare function isProbablyText(buf: Buffer): boolean;
+export declare function sanitizeBuffer(buf: Buffer, username: string): Buffer;
+/** True when the buffer still contains the username (case-insensitive). */
+export declare function containsUsername(buf: Buffer, username: string): boolean;
+export interface SanitizeTreeResult {
+    filesScanned: number;
+    filesReplaced: number;
+    binaryCopied: number;
+}
+/** Copy `src` -> `dst` recursively, sanitizing every text file in flight.
+ *  The destination is always rebuilt from source (idempotent export). */
+export declare function sanitizeTree(src: string, dst: string, username: string): SanitizeTreeResult;
+/** Sanitize a single file into `dstPath` (parent dirs created). */
+export declare function sanitizeFileTo(srcPath: string, dstPath: string, username: string): {
+    replaced: boolean;
+};
+/** Placeholder twin of basename for sanitized sidecar files:
+ *  manifest.json -> manifest.sanitized.json */
+export declare function sanitizedFileName(name: string): string;

package/dist/sanitize.js

@@ -0,0 +1,114 @@
+/**
+ * @hmharness/evolution - evidence-export username sanitizer.
+ *
+ * Root-cause fix for the publish-hook rejections (2026-10-28): experiment
+ * exports (skill snapshots, gate-home copies, manifests) carried the local
+ * Windows username inside absolute paths and bare mentions, and the repo's
+ * pre-push hook blocks any pushed content matching the username. The fix is
+ * at the EXPORT pipeline: whatever gets copied into website/evidence has the
+ * username replaced by the `<user>` placeholder BEFORE it lands on disk.
+ *
+ * Design constraints:
+ *  - the username is NEVER hardcoded here (this file is itself pushed); it is
+ *    passed in by the caller, typically derived from os.homedir()
+ *  - replacement is case-insensitive and covers every spelling shape at once:
+ *    `C:\Users\<name>\`, `C:/Users/<name>/`, JSON-escaped `C:\\Users\\<name>\\`
+ *    and bare mentions — they are all just the username token in context
+ *  - already-sanitized text is idempotent (`<user>` contains no username)
+ *  - binary files are copied verbatim (never re-encoded)
+ */
+import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { basename, dirname, extname, join } from 'node:path';
+export const PLACEHOLDER = '<user>';
+/** The username to strip, derived from the home directory's last segment.
+ *  Returned lowercase-free (as-is); matching is case-insensitive anyway. */
+export function currentUsername() {
+    const home = process.env.HMH_HOME_OVERRIDE_FOR_SANITIZER ?? process.env.USERPROFILE ?? process.env.HOME ?? '';
+    // USERPROFILE is `C:\Users\<name>`; HOME on unix is `/home/<name>`
+    const seg = home.split(/[\\/]/).filter(Boolean).pop() ?? '';
+    return seg;
+}
+function escapeRegExp(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/** Replace every case-insensitive occurrence of `username` with `<user>`.
+ *  Pure string surgery — no regex feature beyond the escaped literal. */
+export function sanitizeText(text, username) {
+    if (!username || username.length < 2)
+        return text; // refuse to nuke 1-char names
+    const re = new RegExp(escapeRegExp(username), 'gi');
+    return text.replace(re, PLACEHOLDER);
+}
+/** Heuristic text/binary split: a NUL byte or a high share of lone-ish
+ *  control bytes means binary. Text evidence (md/json/task/jsonl/log/txt)
+ *  sails through; png/zips copy verbatim. */
+export function isProbablyText(buf) {
+    if (buf.includes(0))
+        return false;
+    let ctrl = 0;
+    const sample = buf.subarray(0, Math.min(buf.length, 4096));
+    for (const b of sample)
+        if (b < 9 || (b > 13 && b < 32))
+            ctrl++;
+    return ctrl / Math.max(1, sample.length) < 0.02;
+}
+export function sanitizeBuffer(buf, username) {
+    if (!isProbablyText(buf))
+        return buf;
+    return Buffer.from(sanitizeText(buf.toString('utf8'), username), 'utf8');
+}
+/** True when the buffer still contains the username (case-insensitive). */
+export function containsUsername(buf, username) {
+    if (!username)
+        return false;
+    if (!isProbablyText(buf))
+        return false;
+    const re = new RegExp(escapeRegExp(username), 'i');
+    return re.test(buf.toString('utf8'));
+}
+/** Copy `src` -> `dst` recursively, sanitizing every text file in flight.
+ *  The destination is always rebuilt from source (idempotent export). */
+export function sanitizeTree(src, dst, username) {
+    const result = { filesScanned: 0, filesReplaced: 0, binaryCopied: 0 };
+    const walk = (from, to) => {
+        mkdirSync(to, { recursive: true });
+        for (const entry of readdirSync(from).sort()) {
+            const fp = join(from, entry);
+            const tp = join(to, entry);
+            if (statSync(fp).isDirectory()) {
+                walk(fp, tp);
+                continue;
+            }
+            result.filesScanned++;
+            const buf = readFileSync(fp);
+            if (!isProbablyText(buf)) {
+                cpSync(fp, tp); // verbatim binary
+                result.binaryCopied++;
+                continue;
+            }
+            const out = sanitizeText(buf.toString('utf8'), username);
+            if (out !== buf.toString('utf8'))
+                result.filesReplaced++;
+            writeFileSync(tp, out, 'utf8');
+        }
+    };
+    if (!existsSync(src))
+        throw new Error(`sanitizeTree: source missing: ${src}`);
+    walk(src, dst);
+    return result;
+}
+/** Sanitize a single file into `dstPath` (parent dirs created). */
+export function sanitizeFileTo(srcPath, dstPath, username) {
+    const buf = readFileSync(srcPath);
+    const out = sanitizeBuffer(buf, username);
+    mkdirSync(dirname(dstPath), { recursive: true });
+    writeFileSync(dstPath, out);
+    return { replaced: !out.equals(buf) };
+}
+/** Placeholder twin of basename for sanitized sidecar files:
+ *  manifest.json -> manifest.sanitized.json */
+export function sanitizedFileName(name) {
+    const ext = extname(name);
+    const base = basename(name, ext);
+    return `${base}.sanitized${ext}`;
+}

package/dist/three-arm-v2.d.ts

@@ -0,0 +1,69 @@
+import { type ExperimentTask, type RunRecord, type ComparisonResult } from './three-arm.js';
+export declare const EXPERIMENT_ID_V2 = "three-arm-v2-2026-10-28";
+/** Seeds 3 -> 5 (the v1 report's power recommendation). */
+export declare const SEEDS_V2: readonly [1, 2, 3, 4, 5];
+/** Promotion gate: same bars as v1 (inherited by value, not by reference to
+ *  keep the v1 module frozen for its own reproduction). */
+export declare const GATE_V2: {
+    readonly ciLevel: 0.95;
+    readonly mcnemarAlpha: 0.05;
+    readonly maxProtectFallback: 0.03;
+    readonly maxCostGrowth: 0.1;
+    readonly minTasksPerSplit: 15;
+    readonly minDiscordantCells: 8;
+};
+/** Execution-order shuffle material (per-seed). */
+export declare const SHUFFLE_SEED_MATERIAL_V2: string;
+/** The shape the evaluation package's HarmonyBench cases are read into. */
+export interface HarmonyBenchSourceCase {
+    id: string;
+    prompt: string;
+    assertion: {
+        type: 'exact' | 'contains' | 'not_contains' | 'regex';
+        value: string;
+    };
+    category: string;
+    needsTools: boolean;
+}
+/** Serialize a HarmonyBench case into the bench .task format so the protect
+ *  copy on disk is byte-identical to what fileSha/datasetHash were built
+ *  from (deterministic, single source of truth for its bytes). expect-exact
+ *  is written UNQUOTED: the .task parser strips only surrounding quotes, so
+ *  quoting a value that itself contains quotes would break the round-trip. */
+export declare function harmonyBenchTaskBytes(c: HarmonyBenchSourceCase): string;
+/** Map one HarmonyBench case to an experiment protect task. Returns null for
+ *  needsTools cases: the experiment's frozen tool whitelist (read_file /
+ *  list_dir) cannot serve them — same exclusion rationale as v1's
+ *  toolchain-* cases (unfreezable environment), pre-registered here. */
+export declare function harmonyBenchToTask(c: HarmonyBenchSourceCase): ExperimentTask | null;
+/** Failure classes for candidate generation. Deterministic on (task, record):
+ *  judge detail prefixes + the toolCalls signal. Empirical anchor from v1
+ *  phase-0: the only beh failure (beh-t8-n) had toolCalls=0 — the model
+ *  answered without reading the file it was told to read. */
+export type FailureClass = 'exact-output' | 'format' | 'content-marker' | 'tool-use' | 'forbidden';
+export declare const FAILURE_CLASSES: readonly FailureClass[];
+export declare function classifyFailure(task: Pick<ExperimentTask, 'tools'>, r: Pick<RunRecord, 'reason'> & {
+    outcome: {
+        success: boolean | null;
+    };
+    metrics: {
+        toolCalls: number;
+    };
+}): FailureClass | null;
+export declare const POPULATION_RULE: {
+    readonly perClassCandidates: 2;
+    readonly classSelection: "classes ordered by distinct failed-task count (desc), tie by class name; top classes taken, all with >=1 failed task eligible";
+    readonly maxClasses: number;
+    readonly minTasksPerClassPrompt: 1;
+    readonly gate: "the whole population enters ONE runEvolution call as presetProposals with maxProposals = population size; thresholds unchanged (quality floor 0.6, zero bench regression, double sampling, holdout re-verification, cost cap)";
+    readonly cArmPick: "if any candidate is promoted: deploy the one with the highest gate train pass rate; tie broken by lexicographically smallest name (deterministic, pre-registered)";
+    readonly allRejected: "population fully rejected => C arm not deployable, main endpoint NEEDS-DATA (honest outcome, same as v1 cArmRule)";
+};
+export declare function buildTaskSetHashV2(tasks: ExperimentTask[]): {
+    setId: string;
+    datasetHash: string;
+    canonical: string;
+};
+/** compareArms with the seed list as a parameter (v1 hardcodes [1,2,3]; the
+ *  bootstrap/McNemar/gate logic is byte-for-byte the v1 protocol). */
+export declare function compareArmsV2(records: RunRecord[], armX: 'A' | 'B' | 'C', armY: 'A' | 'B' | 'C', split: 'train' | 'protect', seeds?: readonly number[]): ComparisonResult;

package/dist/three-arm-v2.js

@@ -0,0 +1,160 @@
+/**
+ * @hmharness/evolution - three-arm v2: expanded-sample rerun (2026-10-28).
+ *
+ * Continuation of three-arm-main-experiment-2026-10-27 (verdict NEEDS-DATA).
+ * v2 changes over v1, frozen BEFORE any run (the v1 REPORT §六 next-step):
+ *   1. GEPA-style candidate POPULATION: phase-0 TRAIN failures are typed by
+ *      failure class; each class proposes its own candidates (2 per class),
+ *      and the WHOLE population enters the same real evolution gate.
+ *   2. Expanded sample: protect set 23 -> 69 tasks (23 existing hmb holdouts +
+ *      the 46 offline-judgeable HarmonyBench v1.0.0 cases), seeds 3 -> 5.
+ *   3. Everything else - gate thresholds, statistics protocol, cArmRule,
+ *      crashRule, arm construction - is inherited from v1 verbatim.
+ *
+ * Pure core again: constants, failure typing, HarmonyBench task mapping,
+ * seeds-parameterized statistics. All model IO stays in the driver scripts.
+ */
+import { createHash } from 'node:crypto';
+import { GATE, mcnemarExactP, bootstrapPairedCI, } from "./three-arm.js";
+export const EXPERIMENT_ID_V2 = 'three-arm-v2-2026-10-28';
+/** Seeds 3 -> 5 (the v1 report's power recommendation). */
+export const SEEDS_V2 = [1, 2, 3, 4, 5];
+/** Promotion gate: same bars as v1 (inherited by value, not by reference to
+ *  keep the v1 module frozen for its own reproduction). */
+export const GATE_V2 = GATE;
+/** Execution-order shuffle material (per-seed). */
+export const SHUFFLE_SEED_MATERIAL_V2 = EXPERIMENT_ID_V2 + ':exec-order';
+/** Serialize a HarmonyBench case into the bench .task format so the protect
+ *  copy on disk is byte-identical to what fileSha/datasetHash were built
+ *  from (deterministic, single source of truth for its bytes). expect-exact
+ *  is written UNQUOTED: the .task parser strips only surrounding quotes, so
+ *  quoting a value that itself contains quotes would break the round-trip. */
+export function harmonyBenchTaskBytes(c) {
+    const lines = [c.prompt];
+    if (c.assertion.type === 'exact')
+        lines.push(`expect-exact: ${c.assertion.value}`);
+    else if (c.assertion.type === 'regex')
+        lines.push(`expect-regex: ${c.assertion.value}`);
+    else if (c.assertion.type === 'contains')
+        lines.push(`expect: ${c.assertion.value}`);
+    else
+        lines.push(`expect-none: ${c.assertion.value}`);
+    lines.push('holdout: true');
+    return lines.join('\n') + '\n';
+}
+/** Map one HarmonyBench case to an experiment protect task. Returns null for
+ *  needsTools cases: the experiment's frozen tool whitelist (read_file /
+ *  list_dir) cannot serve them — same exclusion rationale as v1's
+ *  toolchain-* cases (unfreezable environment), pre-registered here. */
+export function harmonyBenchToTask(c) {
+    if (c.needsTools)
+        return null;
+    const raw = harmonyBenchTaskBytes(c);
+    return {
+        id: `hb-${c.id}`,
+        prompt: c.prompt,
+        assertion: c.assertion.type === 'exact' ? { expect: [], expectExact: c.assertion.value }
+            : c.assertion.type === 'regex' ? { expect: [], expectRegex: c.assertion.value }
+                : c.assertion.type === 'contains' ? { expect: [c.assertion.value] }
+                    : { expect: [], expectNone: [c.assertion.value] },
+        tools: false,
+        holdout: true,
+        split: 'protect',
+        family: `hb-${c.category}`,
+        fileSha: createHash('sha256').update(raw).digest('hex'),
+    };
+}
+export const FAILURE_CLASSES = ['exact-output', 'format', 'content-marker', 'tool-use', 'forbidden'];
+export function classifyFailure(task, r) {
+    if (r.outcome.success === true)
+        return null;
+    // a tools-required task that never called a tool: answered from nothing
+    if (task.tools && r.metrics.toolCalls === 0)
+        return 'tool-use';
+    const d = r.reason;
+    if (d.startsWith('exact mismatch'))
+        return 'exact-output';
+    if (d.startsWith('regex mismatch'))
+        return 'format';
+    if (d.startsWith('missing'))
+        return 'content-marker';
+    if (d.startsWith('none of the allowed markers'))
+        return 'content-marker';
+    if (d.startsWith('forbidden marker'))
+        return 'forbidden';
+    // tools tasks that DID call tools but still failed an exact/marker check
+    if (task.tools)
+        return 'tool-use';
+    return 'exact-output';
+}
+/* ---------------- population rule (pre-registered) ---------------- */
+export const POPULATION_RULE = {
+    perClassCandidates: 2,
+    classSelection: 'classes ordered by distinct failed-task count (desc), tie by class name; top classes taken, all with >=1 failed task eligible',
+    maxClasses: FAILURE_CLASSES.length,
+    minTasksPerClassPrompt: 1,
+    gate: 'the whole population enters ONE runEvolution call as presetProposals with maxProposals = population size; thresholds unchanged (quality floor 0.6, zero bench regression, double sampling, holdout re-verification, cost cap)',
+    cArmPick: 'if any candidate is promoted: deploy the one with the highest gate train pass rate; tie broken by lexicographically smallest name (deterministic, pre-registered)',
+    allRejected: 'population fully rejected => C arm not deployable, main endpoint NEEDS-DATA (honest outcome, same as v1 cArmRule)',
+};
+/* ---------------- datasetHash (v2 canonical) ---------------- */
+export function buildTaskSetHashV2(tasks) {
+    const primaryKind = (t) => t.assertion.expectExact !== undefined ? 'expect-exact'
+        : t.assertion.expectRegex !== undefined ? 'expect-regex'
+            : t.assertion.expectAny?.length ? 'expect-any'
+                : t.assertion.expectNone?.length ? 'expect-none'
+                    : 'expect';
+    const rows = [...tasks]
+        .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
+        .map((t) => ({ id: t.id, prompt: t.prompt, assertion: { kind: primaryKind(t), expect: JSON.stringify(t.assertion) }, holdout: t.split === 'protect' }));
+    const canonical = JSON.stringify({ kind: 'hmharness-eval-taskset', version: 1, id: 'three-arm-v2-2026-10-28', revision: 1, tasks: rows });
+    return { setId: 'three-arm-v2-2026-10-28', datasetHash: createHash('sha256').update(canonical).digest('hex'), canonical };
+}
+/* ---------------- statistics (v1 protocol, seeds parameterized) ---------------- */
+/** compareArms with the seed list as a parameter (v1 hardcodes [1,2,3]; the
+ *  bootstrap/McNemar/gate logic is byte-for-byte the v1 protocol). */
+export function compareArmsV2(records, armX, armY, split, seeds = SEEDS_V2) {
+    const xs = new Map(records.filter((r) => r.arm === armX && r.holdoutClass === split && r.decision === 'measured').map((r) => [`${r.seed}:${r.taskId}`, r.outcome.success === true]));
+    const ys = new Map(records.filter((r) => r.arm === armY && r.holdoutClass === split && r.decision === 'measured').map((r) => [`${r.seed}:${r.taskId}`, r.outcome.success === true]));
+    const cellKeys = [...xs.keys()].filter((k) => ys.has(k));
+    const byTask = new Map();
+    const mcnemarPerSeed = [];
+    for (const seed of seeds) {
+        let xOnly = 0, yOnly = 0;
+        for (const k of cellKeys) {
+            const [s, task] = k.split(':');
+            if (Number(s) !== seed)
+                continue;
+            const x = xs.get(k), y = ys.get(k);
+            if (x && !y)
+                xOnly++;
+            else if (!x && y)
+                yOnly++;
+            const t = byTask.get(task) ?? { x: [], y: [] };
+            t.x.push(x ? 1 : 0);
+            t.y.push(y ? 1 : 0);
+            byTask.set(task, t);
+        }
+        const { p, discordant } = mcnemarExactP(xOnly, yOnly);
+        mcnemarPerSeed.push({ seed, p, discordant, xOnly, yOnly });
+    }
+    const diffs = [...byTask.values()].map((t) => avg(t.x) - avg(t.y));
+    const rateX = cellKeys.filter((k) => xs.get(k)).length / Math.max(1, cellKeys.length);
+    const rateY = cellKeys.filter((k) => ys.get(k)).length / Math.max(1, cellKeys.length);
+    const bootstrap = bootstrapPairedCI(diffs, { level: GATE_V2.ciLevel, seed: 42 });
+    const discordantCells = mcnemarPerSeed.reduce((s, m) => s + m.discordant, 0);
+    const needsData = [];
+    if (diffs.length < GATE_V2.minTasksPerSplit)
+        needsData.push(`only ${diffs.length} paired tasks (<${GATE_V2.minTasksPerSplit})`);
+    if (discordantCells < GATE_V2.minDiscordantCells)
+        needsData.push(`only ${discordantCells} discordant cells (<${GATE_V2.minDiscordantCells}) — exact test underpowered`);
+    const significant = bootstrap.ciLo > 0 || bootstrap.ciHi < 0 || mcnemarPerSeed.some((m) => m.p < GATE_V2.mcnemarAlpha);
+    return {
+        comparison: `${armX}-vs-${armY}`,
+        split, armX, armY,
+        rateX: Number(rateX.toFixed(4)), rateY: Number(rateY.toFixed(4)),
+        diff: Number((rateX - rateY).toFixed(4)),
+        bootstrap, mcnemarPerSeed, discordantCells, significant, needsData,
+    };
+}
+function avg(xs) { return xs.reduce((s, x) => s + x, 0) / xs.length; }

package/dist/three-arm.d.ts

@@ -0,0 +1,195 @@
+export declare const EXPERIMENT_ID = "three-arm-main-experiment-2026-10-27";
+/** Family quotas for the train subset. Families are the first '-' segment of
+ *  the bench case id. toolchain-* and reply-determinism are excluded (they
+ *  need the live toolchain, i.e. an unfreezable environment). */
+export declare const FAMILY_QUOTAS: Record<string, number>;
+/** beh + behave-* are one family (both are read-file-then-answer cases). */
+export declare const FAMILY_ALIASES: Record<string, string>;
+export declare const SEEDS: readonly [1, 2, 3];
+export declare const SHUFFLE_SEED_MATERIAL: string;
+/** gate split of the train subset: last 8 shuffled tasks become gate-val */
+export declare const GATE_VAL_SIZE = 8;
+/** Promotion gate (pre-registered; the task brief's suggested bars):
+ *  significance = bootstrap 95% CI of the paired task-level diff excludes 0
+ *  OR any per-seed exact McNemar p < 0.05 on the MAIN endpoint only
+ *  (protect-set C-vs-A; the other three comparisons are exploratory).
+ *  AND protect fallback <= 3pp, AND total-token growth <= 10%. */
+export declare const GATE: {
+    readonly ciLevel: 0.95;
+    readonly mcnemarAlpha: 0.05;
+    readonly maxProtectFallback: 0.03;
+    readonly maxCostGrowth: 0.1;
+    readonly minTasksPerSplit: 15;
+    readonly minDiscordantCells: 8;
+};
+export interface ExperimentTask {
+    id: string;
+    prompt: string;
+    /** the full original assertion spec (all modes) — judging input */
+    assertion: {
+        expect: string[];
+        expectExact?: string;
+        expectRegex?: string;
+        expectNone?: string[];
+        expectAny?: string[];
+    };
+    tools: boolean;
+    holdout: boolean;
+    split: 'train' | 'protect';
+    family: string;
+    /** sha256 of the original .task file bytes — content identity */
+    fileSha: string;
+}
+export declare function familyOf(id: string): string;
+/** Deterministic stratified selection: per family, seededOrder(quota). */
+export declare function selectTasks(all: Array<{
+    id: string;
+    holdout: boolean;
+    tools: boolean;
+}>, seed: number): {
+    train: string[];
+    protect: string[];
+    excluded: Array<{
+        id: string;
+        why: string;
+    }>;
+};
+/** datasetHash: sha256 over the canonical registered task set (tasks sorted
+ *  by id; the FULL assertion spec serialized into evalset's assertion.expect
+ *  so the hash is content-sensitive to every mode, not just the primary). */
+export declare function buildTaskSetHash(tasks: ExperimentTask[]): {
+    setId: string;
+    datasetHash: string;
+    canonical: string;
+};
+/** Same fence convention as bench.matchCase: assertions test the fenced
+ *  inner body when the reply is a fenced block. Re-implemented here (not
+ *  imported) so the judge is a self-contained, unit-testable unit. */
+export declare function fenceInner(output: string): string;
+export interface JudgeVerdict {
+    pass: boolean;
+    detail: string;
+    /** claimed-done-but-failed: assertion failed AND the reply contains a
+     *  completion marker (pre-registered false-completion definition) */
+    falseCompletion: boolean;
+}
+export declare function judgeOutput(output: string, t: Pick<ExperimentTask, 'assertion'>): JudgeVerdict;
+export interface SkillSnapshotEntry {
+    name: string;
+    description: string;
+}
+/** The A-arm / gate-baseline injection: the frozen skill-library snapshot
+ *  (same shape as skills.skillsToPrompt). */
+export declare function snapshotToPrompt(entries: SkillSnapshotEntry[]): string;
+/** The C-arm / gate-candidate injection: snapshot + the candidate draft,
+ *  byte-identical in construction to evolve.ts's candidateInjection. */
+export declare function candidateInjection(snapshot: SkillSnapshotEntry[], draftName: string, draftMd: string): string;
+export interface TaskLesson {
+    taskId: string;
+    promptHead: string;
+    failedOutputHead: string;
+    failureDetail: string;
+}
+/** The B-arm adaptation block: deterministic lessons from TRAIN-only null
+ *  failures of the SAME family (never protect tasks — no leakage), max 3.
+ *  Task-scoped by construction: the driver injects it into this task's
+ *  system prompt only and drops it afterwards. */
+export declare function buildAdaptiveBlock(family: string, lessons: TaskLesson[]): string;
+export interface RunRecord {
+    experimentId: string;
+    repoCommit: string;
+    datasetHash: string;
+    baseModel: {
+        provider: string;
+        id: string;
+        sampling: string;
+    };
+    arm: 'A' | 'B' | 'C';
+    seed: number;
+    taskId: string;
+    holdoutClass: 'train' | 'protect';
+    outcome: {
+        success: boolean | null;
+        environmentDeltaHash: string;
+    };
+    metrics: {
+        cost: {
+            promptTokens: number | null;
+            completionTokens: number | null;
+            totalTokens: number | null;
+            usd: null;
+        };
+        wallMs: number;
+        toolCalls: number;
+        recoveries: number;
+        crashes: number;
+        falseCompletion: number;
+        predictedP: number | null;
+        brier: number | null;
+    };
+    safety: {
+        backend: string;
+        isolationLevel: string;
+    };
+    decision: 'measured' | 'crash';
+    reason: string;
+}
+/** Exact two-sided binomial p (McNemar on discordant pairs). */
+export declare function mcnemarExactP(aOnly: number, bOnly: number): {
+    p: number;
+    discordant: number;
+};
+/** Deterministic bootstrap (seeded PRNG so the CI is reproducible) of the
+ *  mean task-level paired difference. Analysis unit = task; each task's
+ *  diff = mean over seeds of (armX success) − mean over seeds of (armY). */
+export declare function bootstrapPairedCI(diffs: number[], opts?: {
+    level?: number;
+    iters?: number;
+    seed?: number;
+}): {
+    mean: number;
+    ciLo: number;
+    ciHi: number;
+    width: number;
+    nTasks: number;
+};
+export interface ComparisonResult {
+    comparison: string;
+    split: 'train' | 'protect';
+    armX: 'A' | 'B' | 'C';
+    armY: 'A' | 'B' | 'C';
+    rateX: number;
+    rateY: number;
+    diff: number;
+    bootstrap: ReturnType<typeof bootstrapPairedCI>;
+    mcnemarPerSeed: Array<{
+        seed: number;
+        p: number;
+        discordant: number;
+        xOnly: number;
+        yOnly: number;
+    }>;
+    discordantCells: number;
+    significant: boolean;
+    needsData: string[];
+}
+/** One arm-vs-arm comparison on one split. Records for the two arms must
+ *  cover the same (seed, taskId) cells — the paired discipline. */
+export declare function compareArms(records: RunRecord[], armX: 'A' | 'B' | 'C', armY: 'A' | 'B' | 'C', split: 'train' | 'protect'): ComparisonResult;
+/** A-arm flip rate across seeds (the null/control variance estimate): share
+ *  of tasks whose outcome is not identical across all null runs. */
+export declare function nullFlux(records: RunRecord[], split: 'train' | 'protect'): {
+    flipRate: number;
+    tasks: number;
+    flippedTaskIds: string[];
+};
+/** The pre-registered promotion gate over the analysis inputs. */
+export declare function evaluateGate(main: ComparisonResult, protectFallback: number, costRatioGrowth: number): {
+    verdict: 'PROMOTE' | 'REJECT' | 'NEEDS-DATA';
+    gates: Array<{
+        name: string;
+        pass: boolean;
+        detail: string;
+    }>;
+    reason: string;
+};

package/dist/three-arm.js

@@ -0,0 +1,295 @@
+/**
+ * @hmharness/evolution - three-arm main experiment (Issue #22, P1, 2026-10-27)
+ *
+ * The question: does ONE persistable harness/skill change raise task
+ * success under a FROZEN base model, without breaking the protection set?
+ * Three arms, everything else pinned:
+ *   A  Baseline          - the harness as-is (frozen skill-library snapshot)
+ *   B  Task-adaptive     - per-task temporary adaptation block, discarded
+ *                          after the task (built from TRAIN failures only)
+ *   C  Persistent-evolved - the candidate skill, deployed only after passing
+ *                          the real evolution gate, injected everywhere
+ *
+ * This module is the PURE core: task selection, judging, injection
+ * building, and the pre-registered statistics. All model IO lives in the
+ * driver scripts. The promotion gate thresholds are constants here so the
+ * analysis cannot tune them after seeing results (pre-registration rule).
+ */
+import { createHash } from 'node:crypto';
+/* ---------------- pre-registered constants (frozen before any run) ---------------- */
+export const EXPERIMENT_ID = 'three-arm-main-experiment-2026-10-27';
+/** Family quotas for the train subset. Families are the first '-' segment of
+ *  the bench case id. toolchain-* and reply-determinism are excluded (they
+ *  need the live toolchain, i.e. an unfreezable environment). */
+export const FAMILY_QUOTAS = { beh: 8, cjk: 6, adv: 6, train: 6, fadv: 3, teach: 3 };
+/** beh + behave-* are one family (both are read-file-then-answer cases). */
+export const FAMILY_ALIASES = { behave: 'beh' };
+export const SEEDS = [1, 2, 3];
+export const SHUFFLE_SEED_MATERIAL = EXPERIMENT_ID + ':exec-order';
+/** gate split of the train subset: last 8 shuffled tasks become gate-val */
+export const GATE_VAL_SIZE = 8;
+/** Promotion gate (pre-registered; the task brief's suggested bars):
+ *  significance = bootstrap 95% CI of the paired task-level diff excludes 0
+ *  OR any per-seed exact McNemar p < 0.05 on the MAIN endpoint only
+ *  (protect-set C-vs-A; the other three comparisons are exploratory).
+ *  AND protect fallback <= 3pp, AND total-token growth <= 10%. */
+export const GATE = {
+    ciLevel: 0.95,
+    mcnemarAlpha: 0.05,
+    maxProtectFallback: 0.03,
+    maxCostGrowth: 0.10,
+    minTasksPerSplit: 15,
+    minDiscordantCells: 8,
+};
+export function familyOf(id) {
+    const seg = id.split('-')[0] ?? id;
+    return FAMILY_ALIASES[seg] ?? seg;
+}
+/** Deterministic stratified selection: per family, seededOrder(quota). */
+export function selectTasks(all, seed) {
+    const excluded = [];
+    const eligible = [];
+    for (const c of all) {
+        if (c.id.startsWith('toolchain-')) {
+            excluded.push({ id: c.id, why: 'needs the live HarmonyOS toolchain (unfreezable environment)' });
+            continue;
+        }
+        if (c.id === 'reply-determinism') {
+            excluded.push({ id: c.id, why: 'not in a sampled family' });
+            continue;
+        }
+        eligible.push(c.id);
+    }
+    const train = [];
+    for (const [fam, quota] of Object.entries(FAMILY_QUOTAS)) {
+        const pool = eligible.filter((id) => familyOf(id) === fam && !all.find((c) => c.id === id)?.holdout);
+        const picked = seededOrderStr(pool, seed).slice(0, quota);
+        if (picked.length < quota)
+            excluded.push({ id: `(${fam} family)`, why: `only ${picked.length}/${quota} train cases available` });
+        train.push(...picked);
+    }
+    train.sort();
+    const protect = eligible.filter((id) => all.find((c) => c.id === id)?.holdout).sort();
+    return { train, protect, excluded };
+}
+function seededOrderStr(items, seed) {
+    // mulberry32 from a string seed — same contract as evalset.seededOrder
+    let a = createHash('sha256').update(String(seed)).digest().readUInt32BE(0);
+    const rand = () => {
+        a = (a + 0x6d2b79f5) | 0;
+        let t = Math.imul(a ^ (a >>> 15), 1 | a);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+    const arr = [...items];
+    for (let i = arr.length - 1; i > 0; i--) {
+        const j = Math.floor(rand() * (i + 1));
+        [arr[i], arr[j]] = [arr[j], arr[i]];
+    }
+    return arr;
+}
+/** datasetHash: sha256 over the canonical registered task set (tasks sorted
+ *  by id; the FULL assertion spec serialized into evalset's assertion.expect
+ *  so the hash is content-sensitive to every mode, not just the primary). */
+export function buildTaskSetHash(tasks) {
+    const rows = [...tasks]
+        .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
+        .map((t) => ({ id: t.id, prompt: t.prompt, assertion: { kind: primaryKind(t), expect: JSON.stringify(t.assertion) }, holdout: t.split === 'protect' }));
+    const canonical = JSON.stringify({ kind: 'hmharness-eval-taskset', version: 1, id: 'three-arm-2026-10-27', revision: 1, tasks: rows });
+    return { setId: 'three-arm-2026-10-27', datasetHash: createHash('sha256').update(canonical).digest('hex'), canonical };
+}
+function primaryKind(t) {
+    if (t.assertion.expectExact)
+        return 'expect-exact';
+    if (t.assertion.expectRegex)
+        return 'expect-regex';
+    if (t.assertion.expectAny?.length)
+        return 'expect-any';
+    if (t.assertion.expectNone?.length)
+        return 'expect-none';
+    return 'expect';
+}
+/* ---------------- judging (environment-truth, never model self-report) ---------------- */
+/** Same fence convention as bench.matchCase: assertions test the fenced
+ *  inner body when the reply is a fenced block. Re-implemented here (not
+ *  imported) so the judge is a self-contained, unit-testable unit. */
+export function fenceInner(output) {
+    const m = output.match(/```[a-zA-Z]*[ \t]*\r?\n([\s\S]*?)(?:\r?\n)?```/);
+    return m ? m[1] : output;
+}
+const DONE_RE = /(已完成|完成|done|已处理|passed|成功)/i;
+export function judgeOutput(output, t) {
+    const body = fenceInner(output);
+    const a = t.assertion;
+    if (a.expectExact !== undefined && body.trim() !== a.expectExact.trim()) {
+        return { pass: false, detail: `exact mismatch: got "${body.trim().slice(0, 80)}"`, falseCompletion: DONE_RE.test(body) };
+    }
+    if (a.expectRegex !== undefined) {
+        try {
+            if (!new RegExp(a.expectRegex).test(body))
+                return { pass: false, detail: `regex mismatch: /${a.expectRegex.slice(0, 60)}/`, falseCompletion: DONE_RE.test(body) };
+        }
+        catch {
+            return { pass: false, detail: `invalid regex in case: ${a.expectRegex.slice(0, 40)}`, falseCompletion: false };
+        }
+    }
+    if (a.expectNone && a.expectNone.some((e) => body.toLowerCase().includes(e.toLowerCase()))) {
+        return { pass: false, detail: `forbidden marker present: ${a.expectNone.join(' && ')}`, falseCompletion: DONE_RE.test(body) };
+    }
+    if (a.expectAny && !a.expectAny.some((e) => body.toLowerCase().includes(e.toLowerCase()))) {
+        return { pass: false, detail: `none of the allowed markers found: ${a.expectAny.join(' || ')}`, falseCompletion: DONE_RE.test(body) };
+    }
+    const missing = a.expect.filter((e) => !body.toLowerCase().includes(e.toLowerCase()));
+    if (missing.length > 0) {
+        return { pass: false, detail: `missing "${missing.join('" && "')}" in output`, falseCompletion: DONE_RE.test(body) };
+    }
+    return { pass: true, detail: 'ok', falseCompletion: false };
+}
+/** The A-arm / gate-baseline injection: the frozen skill-library snapshot
+ *  (same shape as skills.skillsToPrompt). */
+export function snapshotToPrompt(entries) {
+    if (entries.length === 0)
+        return '';
+    return entries.map((s) => `- ${s.name}: ${s.description || '(no description)'}`).join('\n');
+}
+/** The C-arm / gate-candidate injection: snapshot + the candidate draft,
+ *  byte-identical in construction to evolve.ts's candidateInjection. */
+export function candidateInjection(snapshot, draftName, draftMd) {
+    const base = snapshotToPrompt(snapshot);
+    const draftBlock = `## Draft skill under evaluation: ${draftName}\n\n${draftMd.slice(0, 4000)}`;
+    return base ? `${base}\n${draftBlock}` : draftBlock;
+}
+/** The B-arm adaptation block: deterministic lessons from TRAIN-only null
+ *  failures of the SAME family (never protect tasks — no leakage), max 3.
+ *  Task-scoped by construction: the driver injects it into this task's
+ *  system prompt only and drops it afterwards. */
+export function buildAdaptiveBlock(family, lessons) {
+    if (lessons.length === 0)
+        return '';
+    const lines = lessons.slice(0, 3).map((l, i) => `${i + 1}. Task ${l.taskId} asked: "${l.promptHead}" — the attempt replied "${l.failedOutputHead}" and FAILED its assertion (${l.failureDetail}).`);
+    return [
+        '## Task adaptation (temporary — this task only, not persisted)',
+        `Lessons from failed attempts at similar tasks (family: ${family}):`,
+        ...lines,
+        'Apply the lesson: make the final reply satisfy exactly what the task asks for.',
+    ].join('\n');
+}
+/* ---------------- statistics ---------------- */
+/** Exact two-sided binomial p (McNemar on discordant pairs). */
+export function mcnemarExactP(aOnly, bOnly) {
+    const n = aOnly + bOnly;
+    if (n === 0)
+        return { p: 1, discordant: 0 };
+    const k = Math.min(aOnly, bOnly);
+    const logChoose = (nn, kk) => {
+        let s = 0;
+        for (let i = 1; i <= kk; i++)
+            s += Math.log(nn - kk + i) - Math.log(i);
+        return s;
+    };
+    const pmf = (i) => Math.exp(logChoose(n, i) + i * Math.log(0.5) + (n - i) * Math.log(0.5));
+    const pk = pmf(k);
+    let sum = 0;
+    for (let i = 0; i <= n; i++)
+        if (pmf(i) <= pk + 1e-12)
+            sum += pmf(i);
+    return { p: Math.min(1, Number(sum.toFixed(6))), discordant: n };
+}
+/** Deterministic bootstrap (seeded PRNG so the CI is reproducible) of the
+ *  mean task-level paired difference. Analysis unit = task; each task's
+ *  diff = mean over seeds of (armX success) − mean over seeds of (armY). */
+export function bootstrapPairedCI(diffs, opts = {}) {
+    const iters = opts.iters ?? 10_000;
+    const level = opts.level ?? 0.95;
+    let a = createHash('sha256').update(String(opts.seed ?? 0)).digest().readUInt32BE(0);
+    const rand = () => {
+        a = (a + 0x6d2b79f5) | 0;
+        let t = Math.imul(a ^ (a >>> 15), 1 | a);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+    const mean = diffs.reduce((s, x) => s + x, 0) / diffs.length;
+    const means = [];
+    for (let i = 0; i < iters; i++) {
+        let s = 0;
+        for (let j = 0; j < diffs.length; j++)
+            s += diffs[Math.floor(rand() * diffs.length)];
+        means.push(s / diffs.length);
+    }
+    means.sort((x, y) => x - y);
+    const lo = means[Math.floor(((1 - level) / 2) * iters)];
+    const hi = means[Math.ceil((1 + level) / 2 * iters) - 1];
+    return { mean: Number(mean.toFixed(4)), ciLo: Number(lo.toFixed(4)), ciHi: Number(hi.toFixed(4)), width: Number((hi - lo).toFixed(4)), nTasks: diffs.length };
+}
+/** One arm-vs-arm comparison on one split. Records for the two arms must
+ *  cover the same (seed, taskId) cells — the paired discipline. */
+export function compareArms(records, armX, armY, split) {
+    const xs = new Map(records.filter((r) => r.arm === armX && r.holdoutClass === split && r.decision === 'measured').map((r) => [`${r.seed}:${r.taskId}`, r.outcome.success === true]));
+    const ys = new Map(records.filter((r) => r.arm === armY && r.holdoutClass === split && r.decision === 'measured').map((r) => [`${r.seed}:${r.taskId}`, r.outcome.success === true]));
+    const cellKeys = [...xs.keys()].filter((k) => ys.has(k));
+    const byTask = new Map();
+    const mcnemarPerSeed = [];
+    for (const seed of SEEDS) {
+        let xOnly = 0, yOnly = 0;
+        for (const k of cellKeys) {
+            const [s, task] = k.split(':');
+            if (Number(s) !== seed)
+                continue;
+            const x = xs.get(k), y = ys.get(k);
+            if (x && !y)
+                xOnly++;
+            else if (!x && y)
+                yOnly++;
+            const t = byTask.get(task) ?? { x: [], y: [] };
+            t.x.push(x ? 1 : 0);
+            t.y.push(y ? 1 : 0);
+            byTask.set(task, t);
+        }
+        const { p, discordant } = mcnemarExactP(xOnly, yOnly);
+        mcnemarPerSeed.push({ seed, p, discordant, xOnly, yOnly });
+    }
+    const diffs = [...byTask.values()].map((t) => avg(t.x) - avg(t.y));
+    const rateX = cellKeys.filter((k) => xs.get(k)).length / Math.max(1, cellKeys.length);
+    const rateY = cellKeys.filter((k) => ys.get(k)).length / Math.max(1, cellKeys.length);
+    const bootstrap = bootstrapPairedCI(diffs, { level: GATE.ciLevel, seed: 42 });
+    const discordantCells = mcnemarPerSeed.reduce((s, m) => s + m.discordant, 0);
+    const needsData = [];
+    if (diffs.length < GATE.minTasksPerSplit)
+        needsData.push(`only ${diffs.length} paired tasks (<${GATE.minTasksPerSplit})`);
+    if (discordantCells < GATE.minDiscordantCells)
+        needsData.push(`only ${discordantCells} discordant cells (<${GATE.minDiscordantCells}) — exact test underpowered`);
+    const significant = bootstrap.ciLo > 0 || bootstrap.ciHi < 0 || mcnemarPerSeed.some((m) => m.p < GATE.mcnemarAlpha);
+    return {
+        comparison: `${armX}-vs-${armY}`,
+        split, armX, armY,
+        rateX: Number(rateX.toFixed(4)), rateY: Number(rateY.toFixed(4)),
+        diff: Number((rateX - rateY).toFixed(4)),
+        bootstrap, mcnemarPerSeed, discordantCells, significant, needsData,
+    };
+}
+function avg(xs) { return xs.reduce((s, x) => s + x, 0) / xs.length; }
+/** A-arm flip rate across seeds (the null/control variance estimate): share
+ *  of tasks whose outcome is not identical across all null runs. */
+export function nullFlux(records, split) {
+    const byTask = new Map();
+    for (const r of records) {
+        if (r.arm !== 'A' || r.holdoutClass !== split)
+            continue;
+        const s = byTask.get(r.taskId) ?? new Set();
+        s.add(r.outcome.success === true);
+        byTask.set(r.taskId, s);
+    }
+    const flipped = [...byTask.entries()].filter(([, s]) => s.size > 1).map(([id]) => id);
+    return { flipRate: byTask.size ? Number((flipped.length / byTask.size).toFixed(4)) : 0, tasks: byTask.size, flippedTaskIds: flipped.sort() };
+}
+/** The pre-registered promotion gate over the analysis inputs. */
+export function evaluateGate(main, protectFallback, costRatioGrowth) {
+    const gates = [];
+    const supportOK = main.needsData.length === 0;
+    gates.push({ name: 'significance (95% CI excludes 0 OR exact McNemar p<0.05)', pass: main.significant && supportOK, detail: `CI=[${main.bootstrap.ciLo},${main.bootstrap.ciHi}] diff=${main.diff}; McNemar per seed: ${main.mcnemarPerSeed.map((m) => `p=${m.p}`).join(', ')}` });
+    gates.push({ name: `protect fallback <= ${(GATE.maxProtectFallback * 100).toFixed(0)}pp`, pass: protectFallback >= -GATE.maxProtectFallback, detail: `protect diff (C-A) = ${protectFallback}` });
+    gates.push({ name: `cost growth <= ${(GATE.maxCostGrowth * 100).toFixed(0)}%`, pass: costRatioGrowth <= GATE.maxCostGrowth, detail: `token growth = ${(costRatioGrowth * 100).toFixed(2)}%` });
+    const allPass = gates.every((g) => g.pass);
+    const verdict = !supportOK ? 'NEEDS-DATA' : allPass ? 'PROMOTE' : 'REJECT';
+    return { verdict, gates, reason: supportOK ? gates.filter((g) => !g.pass).map((g) => `${g.name}: ${g.detail}`).join('; ') || 'all gates passed' : `low support: ${main.needsData.join('; ')}` };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/evolution",
-  "version": "0.23.43",
+  "version": "0.23.45",
   "description": "hmharness evolution subsystem: persistent memory, insight capture, skill library, and the bench that gives evolution its fitness signal. First-class kernel citizen, not a plugin.",
   "type": "module",
   "main": "dist/index.js",
@@ -15,8 +15,8 @@
     "build": "tsc -p tsconfig.build.json"
   },
   "dependencies": {
-    "@hmharness/cognitive": "0.23.43",
-    "@hmharness/kernel": "0.23.43"
+    "@hmharness/cognitive": "0.23.45",
+    "@hmharness/kernel": "0.23.45"
   },
   "files": [
     "dist"