akm-cli 0.6.1 → 0.7.0

package/dist/tests/bench/leakage.test.js

@@ -0,0 +1,228 @@
+/**
+ * Leakage smoke test for the seeded bench corpus (spec §7.4).
+ *
+ * Gated behind `AKM_BENCH_FIXTURE_TESTS=1`. This is a corpus-content
+ * validator (it inspects the seeded fixture stashes and verifier files,
+ * not the bench framework code itself), so it ships skipped by default —
+ * matching the `AKM_SEMANTIC_TESTS` / `AKM_DOCKER_TESTS` pattern. Run it
+ * locally when you change a fixture stash or a verifier:
+ *
+ *   AKM_BENCH_FIXTURE_TESTS=1 bun test tests/bench/leakage.test.ts
+ *
+ * For every task that declares a `gold_ref` of the form `skill:<name>`,
+ * locate the SKILL.md inside the named fixture stash and assert that the
+ * verifier's *structural assertions* do not appear verbatim in the gold-ref
+ * content. The gold ref is allowed (and expected) to discuss the topic in
+ * general terms — what it must NOT do is hand the agent a copy-pasteable
+ * fragment that satisfies the verifier directly.
+ *
+ * The check extracts:
+ *   • for `regex` verifiers — the literal segments of `expected_match`
+ *     between regex meta-characters (these are the substrings the agent
+ *     must produce);
+ *   • for `pytest` verifiers — Python-style structural assertion paths and
+ *     dictionary lookups (e.g., `services.redis.healthcheck.test`,
+ *     `redis["healthcheck"]["test"]`);
+ *   • for `script` (shell) verifiers — single-quoted `grep` patterns and
+ *     `jq -e` expressions, which encode the exact assertion shape.
+ *
+ * Each fragment is checked individually. Lone tokens that legitimately
+ * appear in any reasonable description of the topic (e.g., `redis-cli`,
+ * `akm`, `bridge`, `feedback`) are filtered out by a minimum-length and
+ * minimum-token-count rule.
+ */
+import { describe, expect, test } from "bun:test";
+import fs from "node:fs";
+import path from "node:path";
+import { effectiveSlice, getTasksRoot, listTasks } from "./corpus";
+const FIXTURE_TESTS = !!process.env.AKM_BENCH_FIXTURE_TESTS;
+const STASHES_ROOT = path.resolve(getTasksRoot(), "..", "..", "stashes");
+/** Resolve `skill:<name>` against the named stash; returns SKILL.md path or `undefined`. */
+function resolveGoldRefPath(stashName, goldRef) {
+    const match = /^skill:([a-z0-9][a-z0-9-]*)$/.exec(goldRef);
+    if (!match)
+        return undefined;
+    const skillDir = path.join(STASHES_ROOT, stashName, "skills", match[1]);
+    const skillFile = path.join(skillDir, "SKILL.md");
+    return fs.existsSync(skillFile) ? skillFile : undefined;
+}
+/**
+ * Pull the literal segments out of a regex pattern. Splits on regex
+ * meta-characters and discards short fragments. The remaining strings are
+ * what the agent's stdout must contain — and therefore what the gold ref
+ * must NOT spell out verbatim.
+ */
+function regexLiterals(pattern) {
+    return pattern
+        .split(/[.*+?^${}()|[\]\\]/)
+        .map((s) => s.trim())
+        .filter((s) => s.length >= 6 && s.includes(" "));
+}
+/** Pull structural assertion fragments out of a pytest verifier file. */
+function pytestStructuralFragments(text) {
+    const out = new Set();
+    // Subscript chains like compose["services"]["redis"]["healthcheck"]["test"].
+    const subscriptRe = /(?:\["[a-z0-9_]+"\]){2,}/g;
+    for (const m of text.matchAll(subscriptRe))
+        out.add(m[0]);
+    // Dotted attribute paths used in error messages, e.g. services.redis.healthcheck.test.
+    const dottedRe = /[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*){2,}/g;
+    for (const m of text.matchAll(dottedRe))
+        out.add(m[0]);
+    return [...out];
+}
+/** Pull shell-verifier assertions: single-quoted greps and jq -e expressions. */
+function shellAssertionFragments(text) {
+    const out = new Set();
+    // grep -q '<pattern>' or grep -qi '<pattern>'.
+    const grepRe = /grep\s+-[a-zA-Z]+\s+'([^']{4,})'/g;
+    for (const m of text.matchAll(grepRe))
+        out.add(m[1]);
+    // jq -e '<expr>'.
+    const jqRe = /jq\s+-e\s+'([^']{4,})'/g;
+    for (const m of text.matchAll(jqRe))
+        out.add(m[1]);
+    return [...out];
+}
+function readVerifierFiles(task) {
+    let combined = "";
+    if (task.verifier === "pytest") {
+        const testsDir = path.join(task.taskDir, "tests");
+        if (fs.existsSync(testsDir)) {
+            for (const entry of fs.readdirSync(testsDir)) {
+                if (entry.endsWith(".py"))
+                    combined += `${fs.readFileSync(path.join(testsDir, entry), "utf8")}\n`;
+            }
+        }
+    }
+    else if (task.verifier === "script") {
+        const verify = path.join(task.taskDir, "verify.sh");
+        if (fs.existsSync(verify))
+            combined += fs.readFileSync(verify, "utf8");
+    }
+    return combined;
+}
+/**
+ * Return the verifier assertion fragments for a task, applying an additional
+ * filter suitable for cross-task comparisons. Short two-word domain phrases
+ * (e.g. `akm feedback`, `akm search`) naturally recur across tasks that share
+ * a domain — they are NOT meaningful leakage signals. A fragment is considered
+ * meaningful only when it either:
+ *   • contains at least two spaces (three or more tokens), or
+ *   • contains a structural character (`=`, `[`, `(`) that marks it as a
+ *     complex expression unlikely to appear by coincidence.
+ *
+ * This is more precise than a raw length threshold because it captures the
+ * difference between `akm feedback` (12 chars, 2 tokens, no structure) and
+ * `.model == "anthropic/claude-opus-4-7"` (37 chars, structural `==`).
+ */
+function crossTaskFragments(task) {
+    const isMeaningful = (f) => {
+        const spaceCount = (f.match(/ /g) ?? []).length;
+        return spaceCount >= 2 || /[=[(]/.test(f);
+    };
+    const raw = [];
+    if (task.verifier === "regex" && task.expectedMatch) {
+        raw.push(...regexLiterals(task.expectedMatch));
+    }
+    else {
+        const verifierText = readVerifierFiles(task);
+        raw.push(...pytestStructuralFragments(verifierText));
+        raw.push(...shellAssertionFragments(verifierText));
+    }
+    return raw.filter(isMeaningful);
+}
+describe.skipIf(!FIXTURE_TESTS)("cross-task eval/train verifier leakage check", () => {
+    const allTasks = listTasks();
+    // Group tasks by stash name.
+    const byStash = new Map();
+    for (const task of allTasks) {
+        const group = byStash.get(task.stash) ?? [];
+        group.push(task);
+        byStash.set(task.stash, group);
+    }
+    // Only stashes that have BOTH train and eval tasks are interesting.
+    const mixedStashes = [...byStash.entries()].filter(([, tasks]) => {
+        const hasTrain = tasks.some((t) => effectiveSlice(t) === "train");
+        const hasEval = tasks.some((t) => effectiveSlice(t) === "eval");
+        return hasTrain && hasEval;
+    });
+    test("at least one stash has both train and eval tasks", () => {
+        expect(mixedStashes.length).toBeGreaterThan(0);
+    });
+    for (const [stashName, tasks] of mixedStashes) {
+        const trainTasks = tasks.filter((t) => effectiveSlice(t) === "train");
+        const evalTasks = tasks.filter((t) => effectiveSlice(t) === "eval");
+        // Train → Eval: train verifier fragments must not appear in eval verifier text.
+        // Skip pairs that are intentional train/eval variants of the same task family
+        // (e.g. inkwell/add-healthcheck-train vs inkwell/add-healthcheck) — they share
+        // field-access patterns by design, just with different expected values.
+        const isVariantPair = (trainId, evalId) => {
+            const trainBase = trainId.replace(/-train$/, "");
+            return trainBase === evalId || evalId.startsWith(`${trainBase}-`);
+        };
+        for (const trainTask of trainTasks) {
+            const trainFragments = crossTaskFragments(trainTask);
+            if (trainFragments.length === 0)
+                continue;
+            for (const evalTask of evalTasks) {
+                if (isVariantPair(trainTask.id, evalTask.id))
+                    continue;
+                const evalVerifierText = readVerifierFiles(evalTask);
+                test(`stash:${stashName} — train:${trainTask.id} fragments not in eval:${evalTask.id} verifier`, () => {
+                    const leaked = trainFragments.filter((frag) => evalVerifierText.includes(frag));
+                    expect(leaked, `fragments leaked from train verifier to eval verifier: ${JSON.stringify(leaked)}`).toEqual([]);
+                });
+            }
+        }
+        // Eval → Train: eval verifier fragments must not appear in train verifier text.
+        for (const evalTask of evalTasks) {
+            const evalFragments = crossTaskFragments(evalTask);
+            if (evalFragments.length === 0)
+                continue;
+            for (const trainTask of trainTasks) {
+                if (isVariantPair(trainTask.id, evalTask.id))
+                    continue;
+                const trainVerifierText = readVerifierFiles(trainTask);
+                test(`stash:${stashName} — eval:${evalTask.id} fragments not in train:${trainTask.id} verifier`, () => {
+                    const leaked = evalFragments.filter((frag) => trainVerifierText.includes(frag));
+                    expect(leaked, `fragments leaked from eval verifier to train verifier: ${JSON.stringify(leaked)}`).toEqual([]);
+                });
+            }
+        }
+    }
+});
+describe.skipIf(!FIXTURE_TESTS)("gold-ref leakage check", () => {
+    const tasks = listTasks().filter((t) => t.goldRef);
+    test("at least one task ships with a gold_ref", () => {
+        expect(tasks.length).toBeGreaterThan(0);
+    });
+    for (const task of tasks) {
+        test(`${task.id}: verifier text does not appear in gold-ref content`, () => {
+            const goldRef = task.goldRef;
+            const goldPath = resolveGoldRefPath(task.stash, goldRef);
+            // A declared gold_ref MUST resolve to an existing fixture asset. Silent
+            // skipping here previously masked typos and stash-name drift; we now
+            // fail loudly so the corpus author is forced to fix the reference.
+            if (!goldPath) {
+                // Non-skill refs (workflow:, command:, etc.) are not leakage-checked —
+                // only skill: refs map to a SKILL.md that could leak answers.
+                if (!/^skill:/.test(goldRef))
+                    return;
+                throw new Error(`${task.id}: gold_ref "${goldRef}" against stash "${task.stash}" did not resolve to a SKILL.md under tests/fixtures/stashes/. Fix the gold_ref, fix the stash name, or remove the gold_ref.`);
+            }
+            const goldContent = fs.readFileSync(goldPath, "utf8");
+            const fragments = [];
+            if (task.verifier === "regex" && task.expectedMatch) {
+                fragments.push(...regexLiterals(task.expectedMatch));
+            }
+            else {
+                const verifierText = readVerifierFiles(task);
+                fragments.push(...pytestStructuralFragments(verifierText));
+                fragments.push(...shellAssertionFragments(verifierText));
+            }
+            const leaked = fragments.filter((frag) => goldContent.includes(frag));
+            expect(leaked).toEqual([]);
+        });
+    }
+});

package/dist/tests/bench/learning-curve.test.js

@@ -0,0 +1,134 @@
+/**
+ * Report-level coverage for Track B learning curves (issue #265).
+ *
+ * Verifies that a `LearningCurve` supplied via `EvolveReportInput.learningCurve`
+ * surfaces in both the JSON envelope (under `learning`) and the markdown
+ * body (as a "Learning curve" section). Companion unit tests for
+ * `computeLearningCurve` itself live in `metrics.test.ts`.
+ */
+import { describe, expect, test } from "bun:test";
+import { computeLearningCurve } from "./metrics";
+import { renderEvolveReport, renderLearningCurveSection, } from "./report";
+function emptyUtilityReport() {
+    return {
+        timestamp: "2026-04-27T00:00:00Z",
+        branch: "test",
+        commit: "deadbee",
+        model: "m",
+        corpus: { domains: 0, tasks: 0, slice: "all", seedsPerArm: 1 },
+        aggregateNoakm: { passRate: 0, tokensPerPass: 0, tokensPerRun: null, wallclockMs: 0 },
+        aggregateAkm: { passRate: 0, tokensPerPass: 0, tokensPerRun: null, wallclockMs: 0 },
+        aggregateDelta: {
+            passRate: 0,
+            tokensPerPass: 0,
+            tokensPerRun: null,
+            wallclockMs: 0,
+        },
+        trajectoryAkm: {
+            correctAssetLoaded: null,
+            feedbackRecorded: 0,
+        },
+        failureModes: { byLabel: {}, byTask: {} },
+        tasks: [],
+        warnings: [],
+        akmRuns: [],
+        taskMetadata: [],
+        goldRankRecords: [],
+    };
+}
+function ep(overrides) {
+    return {
+        delta_from_previous_episode: 0,
+        cumulative_feedback_events: 0,
+        cumulative_proposals_created: 0,
+        cumulative_proposals_accepted: 0,
+        cumulative_lessons_created: 0,
+        lesson_reuse_rate: null,
+        ...overrides,
+    };
+}
+function baseEvolveInput(extra = {}) {
+    return {
+        timestamp: "2026-04-27T00:00:00Z",
+        branch: "test",
+        commit: "deadbee",
+        model: "m",
+        domain: "test",
+        seedsPerArm: 1,
+        proposals: { rows: [], totalProposals: 0, totalAccepted: 0, acceptanceRate: 0, lintPassRate: 0 },
+        longitudinal: {
+            improvementSlope: 0.1,
+            overSyntheticLift: 0.05,
+            degradationCount: 0,
+            degradations: [],
+            prePassRate: 0.5,
+            postPassRate: 0.6,
+            syntheticPassRate: 0.55,
+        },
+        arms: { pre: emptyUtilityReport(), post: emptyUtilityReport(), synthetic: emptyUtilityReport() },
+        warnings: [],
+        ...extra,
+    };
+}
+describe("renderEvolveReport — learning block (#265)", () => {
+    test("emits learning.episodes[] + slope + time_to_improvement when learningCurve supplied", () => {
+        const curve = computeLearningCurve([
+            ep({ episode_index: 0, pass_rate: 0.4, cumulative_feedback_events: 10 }),
+            ep({
+                episode_index: 1,
+                pass_rate: 0.55,
+                cumulative_feedback_events: 22,
+                cumulative_proposals_created: 4,
+                cumulative_proposals_accepted: 3,
+                cumulative_lessons_created: 3,
+                lesson_reuse_rate: 0.42,
+            }),
+            ep({
+                episode_index: 2,
+                pass_rate: 0.7,
+                cumulative_feedback_events: 35,
+                cumulative_proposals_created: 7,
+                cumulative_proposals_accepted: 5,
+                cumulative_lessons_created: 5,
+                lesson_reuse_rate: 0.6,
+            }),
+        ]);
+        const { json, markdown } = renderEvolveReport(baseEvolveInput({ learningCurve: curve }));
+        const parsed = json;
+        expect(parsed.learning).toBeDefined();
+        expect(parsed.learning?.episodes).toHaveLength(3);
+        expect(parsed.learning?.pass_rate_by_episode).toEqual([0.4, 0.55, 0.7]);
+        expect(parsed.learning?.time_to_improvement).toBe(1);
+        expect(parsed.learning?.learning_slope).toBeCloseTo(0.15, 3);
+        expect(markdown).toContain("Learning curve");
+        expect(markdown).toContain("learning_slope=+0.150");
+        expect(markdown).toContain("time_to_improvement=1");
+    });
+    test("omits learning block when learningCurve absent (legacy path)", () => {
+        const { json, markdown } = renderEvolveReport(baseEvolveInput());
+        const parsed = json;
+        expect(parsed.learning).toBeUndefined();
+        expect(markdown).not.toContain("Learning curve");
+    });
+    test("renders n/a for time_to_improvement when no improvement", () => {
+        const curve = computeLearningCurve([
+            ep({ episode_index: 0, pass_rate: 0.5 }),
+            ep({ episode_index: 1, pass_rate: 0.5 }),
+        ]);
+        const { json, markdown } = renderEvolveReport(baseEvolveInput({ learningCurve: curve }));
+        const parsed = json;
+        expect(parsed.learning?.time_to_improvement).toBeNull();
+        expect(markdown).toContain("time_to_improvement=n/a");
+    });
+});
+describe("renderLearningCurveSection", () => {
+    test("emits the empty-state message when no episodes are recorded", () => {
+        const md = renderLearningCurveSection({
+            episodes: [],
+            pass_rate_by_episode: [],
+            learning_slope: 0,
+            time_to_improvement: null,
+        });
+        expect(md).toContain("No episodes recorded");
+    });
+});