akm-cli 0.7.0 → 0.7.2

package/dist/tests/bench/corpus.js

@@ -1,317 +0,0 @@
-/**
- * akm-bench corpus loader (spec §5.4 + §13.1).
- *
- * Tasks live at `tests/fixtures/bench/tasks/<domain>/<task-id>/task.yaml`. Each
- * task is a flat YAML record (see §13.1). The schema lands in #237 and grows
- * over time; #236 only ships the loader and one sample task fixture.
- *
- * The `tests/fixtures/bench/tasks/` directory may not exist on `release/0.7.0`
- * yet — that's #237's deliverable. `listTasks()` MUST return `[]` cleanly when
- * the directory is missing rather than throwing, so consumers degrade gracefully.
- */
-import { createHash } from "node:crypto";
-import fs from "node:fs";
-import path from "node:path";
-import { parse as parseYaml } from "yaml";
-/**
- * Closed set of memory-operation labels (#262).
- *
- * Each value names the kind of memory / knowledge ability the task exercises:
- * - `procedural_lookup` — find and apply a single procedural skill.
- * - `multi_asset_composition` — combine guidance from two+ assets.
- * - `temporal_update` — apply the most recent guidance over older versions.
- * - `conflict_resolution` — choose between conflicting assets.
- * - `abstention` — recognise no relevant asset exists and decline to load.
- * - `noisy_retrieval` — succeed despite distractor / irrelevant assets.
- *
- * Adding a value here is a CORPUS-schema change; callers iterate the closed
- * set when laying out coverage tables.
- */
-export const MEMORY_ABILITY_VALUES = [
-    "procedural_lookup",
-    "multi_asset_composition",
-    "temporal_update",
-    "conflict_resolution",
-    "abstention",
-    "noisy_retrieval",
-];
-const TASKS_ROOT = path.resolve(__dirname, "..", "fixtures", "bench", "tasks");
-/** Public for tests; resolves the corpus root in case callers need to assert it. */
-export function getTasksRoot() {
-    return TASKS_ROOT;
-}
-/**
- * Compute a deterministic SHA-256 hash over a selected task corpus (#250).
- *
- * Two reports with the same selected task IDs and the same task body bytes
- * produce the same hash. The hash is order-independent: callers may pass IDs
- * in any order — the function sorts them lexicographically before hashing.
- *
- * The `taskBodies` map is keyed by task id and carries the raw `task.yaml`
- * bytes (or any deterministic per-task identity payload). Tasks present in
- * `taskIds` but missing from `taskBodies` are hashed with an empty body so
- * the hash still distinguishes the selection.
- *
- * Encoding (per id, in sorted order):
- *   `<id>\0<body-bytes>\0`
- *
- * Used by `bench compare` to refuse mismatched corpora unless the operator
- * explicitly opts in via `--allow-corpus-mismatch`.
- */
-export function computeTaskCorpusHash(taskIds, taskBodies) {
-    const sortedIds = [...taskIds].sort();
-    const hash = createHash("sha256");
-    for (const id of sortedIds) {
-        hash.update(id);
-        hash.update("\0");
-        hash.update(taskBodies.get(id) ?? "");
-        hash.update("\0");
-    }
-    return hash.digest("hex");
-}
-/**
- * Read the raw `task.yaml` bytes for a task whose `taskDir` is known. Returns
- * the empty string when the file is missing — callers should still hash the
- * id so the selection's identity is preserved.
- */
-export function readTaskBody(taskDir) {
-    const yamlPath = path.join(taskDir, "task.yaml");
-    try {
-        return fs.readFileSync(yamlPath, "utf8");
-    }
-    catch {
-        return "";
-    }
-}
-/**
- * The `_example/` task tree is reserved for loader unit tests; real corpus
- * statistics must exclude it. The path-prefix check matches both POSIX (`/`)
- * and Windows (`\`) separators.
- */
-const EXAMPLE_PREFIX = `${path.sep}_example${path.sep}`;
-function isExampleTaskDir(dir) {
-    return dir.includes(EXAMPLE_PREFIX);
-}
-/**
- * Load a single task by id. Walks the tasks tree until a directory whose
- * `task.yaml` carries the requested `id` is found. By default the
- * `_example/` tree is skipped; pass `{ includeExamples: true }` to load
- * tasks under that prefix (used by loader-unit tests).
- *
- * Throws if the corpus directory is missing or no task matches.
- */
-export function loadTask(taskId, options = {}) {
-    if (!fs.existsSync(TASKS_ROOT)) {
-        throw new Error(`bench corpus directory missing: ${TASKS_ROOT}`);
-    }
-    for (const candidate of walkTaskDirs(TASKS_ROOT)) {
-        if (!options.includeExamples && isExampleTaskDir(candidate))
-            continue;
-        const meta = readTask(candidate);
-        if (meta && meta.id === taskId)
-            return meta;
-    }
-    throw new Error(`task not found: ${taskId}`);
-}
-/**
- * Return every task in the corpus, optionally filtered by `slice`. When the
- * corpus directory is missing this returns `[]`. By default tasks under the
- * `_example/` tree are excluded; pass `{ includeExamples: true }` to include
- * them.
- */
-export function listTasks(options = {}) {
-    if (!fs.existsSync(TASKS_ROOT))
-        return [];
-    const out = [];
-    for (const dir of walkTaskDirs(TASKS_ROOT)) {
-        if (!options.includeExamples && isExampleTaskDir(dir))
-            continue;
-        const meta = readTask(dir);
-        if (!meta)
-            continue;
-        if (options.slice) {
-            // Tasks without an explicit `slice:` are partitioned by id-hash so the
-            // filter behaves consistently with `partitionSlice()`. Without this,
-            // an unsliced task would always pass through both `slice: "train"` and
-            // `slice: "eval"` filters, double-counting in evaluation reports.
-            if (effectiveSlice(meta) !== options.slice)
-                continue;
-        }
-        out.push(meta);
-    }
-    out.sort((a, b) => a.id.localeCompare(b.id));
-    return out;
-}
-/**
- * Partition a list of tasks into train/eval slices. Tasks declaring an explicit
- * `slice:` are honoured; tasks that don't are placed by `id-hash mod 2 === 0`
- * → train, else eval. The hash is SHA-1 of the id (deterministic across hosts).
- */
-export function partitionSlice(tasks) {
-    const train = [];
-    const evalSlice = [];
-    for (const task of tasks) {
-        const slice = effectiveSlice(task);
-        if (slice === "train")
-            train.push(task);
-        else
-            evalSlice.push(task);
-    }
-    return { train, eval: evalSlice };
-}
-function assignSliceByHash(id) {
-    // SHA-1 first byte parity is stable, fast, and uniformly distributed enough
-    // for slice partitioning. We avoid `node:crypto.randomInt` (non-deterministic).
-    const digest = createHash("sha1").update(id).digest();
-    return digest[0] % 2 === 0 ? "train" : "eval";
-}
-/**
- * Resolve a task's effective slice. If `task.yaml` declares `slice:` we honour
- * it; otherwise the id-hash partition assigns one deterministically. Exported
- * so that consumers (and tests) can ask "which slice would this task fall in?"
- * without re-running `partitionSlice()`.
- */
-export function effectiveSlice(task) {
-    return task.slice ?? assignSliceByHash(task.id);
-}
-/** Walk the corpus tree depth-first, yielding every directory containing a `task.yaml`. */
-function* walkTaskDirs(root) {
-    const stack = [root];
-    while (stack.length > 0) {
-        const dir = stack.pop();
-        if (!dir)
-            continue;
-        let entries;
-        try {
-            entries = fs.readdirSync(dir, { withFileTypes: true });
-        }
-        catch {
-            continue;
-        }
-        if (entries.some((e) => e.isFile() && e.name === "task.yaml")) {
-            yield dir;
-            continue; // Don't recurse beneath a task directory.
-        }
-        for (const entry of entries) {
-            if (entry.isDirectory())
-                stack.push(path.join(dir, entry.name));
-        }
-    }
-}
-function readTask(taskDir) {
-    const yamlPath = path.join(taskDir, "task.yaml");
-    let text;
-    try {
-        text = fs.readFileSync(yamlPath, "utf8");
-    }
-    catch {
-        return undefined;
-    }
-    return parseTaskYaml(text, taskDir);
-}
-export function parseTaskYaml(text, taskDir) {
-    let raw;
-    try {
-        raw = parseYaml(text);
-    }
-    catch {
-        return undefined;
-    }
-    if (!raw || typeof raw !== "object")
-        return undefined;
-    const id = asString(raw.id);
-    const title = asString(raw.title);
-    const domain = asString(raw.domain);
-    const difficulty = asEnum(raw.difficulty, ["easy", "medium", "hard"]);
-    const stash = asString(raw.stash);
-    const verifier = asEnum(raw.verifier, ["script", "pytest", "regex"]);
-    const tokens = asNumber(raw.budget?.tokens);
-    const wallMs = asNumber(raw.budget?.wallMs);
-    if (!id || !title || !domain || !difficulty || !stash || !verifier || tokens === undefined || wallMs === undefined) {
-        return undefined;
-    }
-    const slice = raw.slice === undefined ? undefined : asEnum(raw.slice, ["train", "eval"]);
-    const goldRef = asString(raw.gold_ref);
-    const stashOverlay = asString(raw.stash_overlay);
-    const expectedMatch = asString(raw.expected_match);
-    const meta = {
-        id,
-        title,
-        domain,
-        difficulty,
-        stash,
-        verifier,
-        budget: { tokens, wallMs },
-        taskDir,
-    };
-    if (slice !== undefined)
-        meta.slice = slice;
-    if (goldRef !== undefined)
-        meta.goldRef = goldRef;
-    if (stashOverlay !== undefined)
-        meta.stashOverlay = stashOverlay;
-    if (expectedMatch !== undefined)
-        meta.expectedMatch = expectedMatch;
-    // #262 memory-operation tags. Each tag is OPTIONAL — invalid values are
-    // silently ignored so a malformed tag never breaks an existing task. This
-    // matches the broader "loader returns undefined on bad core fields, drops
-    // bad tags" contract documented in the §13.1 schema.
-    const memoryAbility = asEnum(raw.memory_ability, MEMORY_ABILITY_VALUES);
-    if (memoryAbility !== undefined)
-        meta.memoryAbility = memoryAbility;
-    const workflowFocus = asString(raw.workflow_focus);
-    if (workflowFocus !== undefined)
-        meta.workflowFocus = workflowFocus;
-    const taskFamily = asString(raw.task_family);
-    if (taskFamily !== undefined)
-        meta.taskFamily = taskFamily;
-    const expectedTransferFrom = asStringArray(raw.expected_transfer_from);
-    if (expectedTransferFrom !== undefined)
-        meta.expectedTransferFrom = expectedTransferFrom;
-    const abstentionCase = asBoolean(raw.abstention_case);
-    if (abstentionCase !== undefined)
-        meta.abstentionCase = abstentionCase;
-    const conflictCase = asBoolean(raw.conflict_case);
-    if (conflictCase !== undefined)
-        meta.conflictCase = conflictCase;
-    const staleGuidanceCase = asBoolean(raw.stale_guidance_case);
-    if (staleGuidanceCase !== undefined)
-        meta.staleGuidanceCase = staleGuidanceCase;
-    const akmKeywords = asString(raw.akm_keywords);
-    if (akmKeywords !== undefined)
-        meta.akmKeywords = akmKeywords;
-    return meta;
-}
-function asBoolean(value) {
-    if (typeof value === "boolean")
-        return value;
-    return undefined;
-}
-function asStringArray(value) {
-    if (!Array.isArray(value))
-        return undefined;
-    const out = [];
-    for (const entry of value) {
-        if (typeof entry === "string" && entry.length > 0)
-            out.push(entry);
-    }
-    return out;
-}
-function asString(value) {
-    return typeof value === "string" && value.length > 0 ? value : undefined;
-}
-function asNumber(value) {
-    if (typeof value === "number" && Number.isFinite(value))
-        return value;
-    if (typeof value === "string") {
-        const parsed = Number.parseFloat(value);
-        if (Number.isFinite(parsed))
-            return parsed;
-    }
-    return undefined;
-}
-function asEnum(value, allowed) {
-    if (typeof value !== "string")
-        return undefined;
-    return allowed.includes(value) ? value : undefined;
-}

package/dist/tests/bench/corpus.test.js

@@ -1,258 +0,0 @@
-/**
- * Unit tests for the bench corpus loader.
- *
- *   • `listTasks()` returns `[]` cleanly when the corpus dir is missing.
- *   • The shipped sample task at `_example/example-task` is excluded by
- *     default but loadable via `{ includeExamples: true }`.
- *   • The seeded corpus contains 23 tasks (issue #237 seeded 17 across
- *     three domains; #259 added six workflow-compliance tasks) and every
- *     entry validates against the §13.1 schema.
- *   • `partitionSlice` is deterministic — same input → same partitioning
- *     across calls.
- */
-import { describe, expect, test } from "bun:test";
-import fs from "node:fs";
-import path from "node:path";
-import { computeTaskCorpusHash, effectiveSlice, getTasksRoot, listTasks, loadTask, MEMORY_ABILITY_VALUES, parseTaskYaml, partitionSlice, readTaskBody, } from "./corpus";
-import { benchMkdtemp } from "./tmp";
-describe("listTasks", () => {
-    test("the corpus root resolves under tests/fixtures/bench/tasks", () => {
-        expect(getTasksRoot()).toMatch(/tests[\\/]+fixtures[\\/]+bench[\\/]+tasks$/);
-    });
-    test("returns an array (possibly empty) without throwing", () => {
-        const tasks = listTasks();
-        expect(Array.isArray(tasks)).toBe(true);
-    });
-    test("excludes `_example/` tasks by default", () => {
-        const tasks = listTasks();
-        expect(tasks.find((t) => t.id.startsWith("_example/"))).toBeUndefined();
-    });
-    test("loads `_example/` when includeExamples is set", () => {
-        const tasks = listTasks({ includeExamples: true });
-        const sample = tasks.find((t) => t.id === "_example/example-task");
-        expect(sample).toBeDefined();
-        expect(sample?.title).toContain("Example task");
-        expect(sample?.stash).toBe("minimal");
-        expect(sample?.verifier).toBe("script");
-        expect(sample?.budget.tokens).toBe(1000);
-        expect(sample?.budget.wallMs).toBe(30_000);
-    });
-    test("seeds hand-authored tasks across all domains", () => {
-        const tasks = listTasks();
-        expect(tasks).toHaveLength(40);
-        const byDomain = new Map();
-        for (const task of tasks) {
-            const list = byDomain.get(task.domain) ?? [];
-            list.push(task);
-            byDomain.set(task.domain, list);
-        }
-        expect(new Set(byDomain.keys())).toEqual(new Set(["docker-homelab", "az-cli", "opencode", "workflow-compliance", "drillbit", "inkwell"]));
-        expect(byDomain.get("docker-homelab")).toHaveLength(6);
-        expect(byDomain.get("az-cli")).toHaveLength(6);
-        expect(byDomain.get("opencode")).toHaveLength(6);
-        expect(byDomain.get("workflow-compliance")).toHaveLength(6);
-        expect(byDomain.get("drillbit")).toHaveLength(7);
-        expect(byDomain.get("inkwell")).toHaveLength(9);
-    });
-    test("every task validates against the §13.1 schema", () => {
-        const ID_RE = /^[a-z0-9][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/;
-        for (const task of listTasks()) {
-            expect(task.id).toMatch(ID_RE);
-            expect(task.title.length).toBeGreaterThan(0);
-            expect(["easy", "medium", "hard"]).toContain(task.difficulty);
-            expect(["train", "eval"]).toContain(task.slice);
-            expect(["pytest", "script", "regex"]).toContain(task.verifier);
-            expect(typeof task.stash).toBe("string");
-            expect(task.budget.tokens).toBeGreaterThan(0);
-            expect(task.budget.wallMs).toBeGreaterThan(0);
-            if (task.verifier === "regex") {
-                expect(task.expectedMatch).toBeDefined();
-                expect((task.expectedMatch ?? "").length).toBeGreaterThan(0);
-            }
-        }
-    });
-    test("filters by slice when requested", () => {
-        const train = listTasks({ slice: "train" });
-        const evalTasks = listTasks({ slice: "eval" });
-        expect(train.every((t) => t.slice === "train")).toBe(true);
-        expect(evalTasks.every((t) => t.slice === "eval")).toBe(true);
-        // 23 train (19 + 2 drillbit train + 2 inkwell train)
-        // 17 eval  (15 prior + 2 new: inkwell/full-config + opencode/select-correct-skill)
-        expect(train).toHaveLength(23);
-        expect(evalTasks).toHaveLength(17);
-    });
-});
-describe("loadTask", () => {
-    test("loads a real corpus task by id", () => {
-        const meta = loadTask("docker-homelab/redis-healthcheck");
-        expect(meta.title).toContain("Redis healthcheck");
-        expect(meta.taskDir).toContain("docker-homelab/redis-healthcheck");
-        expect(meta.verifier).toBe("pytest");
-    });
-    test("loads the example task only with includeExamples", () => {
-        expect(() => loadTask("_example/example-task")).toThrow();
-        const meta = loadTask("_example/example-task", { includeExamples: true });
-        expect(meta.taskDir).toContain("_example/example-task");
-    });
-    test("throws on unknown id", () => {
-        expect(() => loadTask("does/not/exist")).toThrow();
-    });
-});
-describe("partitionSlice", () => {
-    function fakeTask(id, slice) {
-        return {
-            id,
-            title: id,
-            domain: "test",
-            difficulty: "easy",
-            stash: "minimal",
-            verifier: "regex",
-            budget: { tokens: 1000, wallMs: 1000 },
-            taskDir: "/tmp/none",
-            ...(slice ? { slice } : {}),
-        };
-    }
-    test("explicit slice fields are honoured", () => {
-        const tasks = [fakeTask("a", "train"), fakeTask("b", "eval"), fakeTask("c", "train")];
-        const { train, eval: evalSlice } = partitionSlice(tasks);
-        expect(train.map((t) => t.id).sort()).toEqual(["a", "c"]);
-        expect(evalSlice.map((t) => t.id)).toEqual(["b"]);
-    });
-    test("tasks without explicit slice get a deterministic assignment", () => {
-        const tasks = [fakeTask("alpha"), fakeTask("beta"), fakeTask("gamma"), fakeTask("delta"), fakeTask("epsilon")];
-        const a = partitionSlice(tasks);
-        const b = partitionSlice(tasks);
-        expect(a.train.map((t) => t.id)).toEqual(b.train.map((t) => t.id));
-        expect(a.eval.map((t) => t.id)).toEqual(b.eval.map((t) => t.id));
-        // Sanity: every task ends up in exactly one slice.
-        expect(a.train.length + a.eval.length).toBe(tasks.length);
-    });
-    test("partition of the real corpus is stable across calls", () => {
-        const corpus = listTasks();
-        const a = partitionSlice(corpus);
-        const b = partitionSlice(corpus);
-        expect(a.train.map((t) => t.id)).toEqual(b.train.map((t) => t.id));
-        expect(a.eval.map((t) => t.id)).toEqual(b.eval.map((t) => t.id));
-    });
-    test("computeTaskCorpusHash is deterministic and order-independent (#250)", () => {
-        const bodies = new Map([
-            ["a/one", "id: a/one\ntitle: One\n"],
-            ["b/two", "id: b/two\ntitle: Two\n"],
-        ]);
-        const h1 = computeTaskCorpusHash(["a/one", "b/two"], bodies);
-        const h2 = computeTaskCorpusHash(["b/two", "a/one"], bodies);
-        expect(h1).toBe(h2);
-        expect(h1).toMatch(/^[0-9a-f]{64}$/);
-        // Body change → hash change.
-        const altBodies = new Map(bodies);
-        altBodies.set("a/one", "id: a/one\ntitle: Different\n");
-        const h3 = computeTaskCorpusHash(["a/one", "b/two"], altBodies);
-        expect(h3).not.toBe(h1);
-        // ID-set change → hash change.
-        const h4 = computeTaskCorpusHash(["a/one"], bodies);
-        expect(h4).not.toBe(h1);
-        // readTaskBody returns "" for a missing taskDir without throwing.
-        expect(readTaskBody("/does/not/exist")).toBe("");
-    });
-    test("a task without explicit slice is bucketed deterministically and goes to exactly one slice", () => {
-        // This guards against the regression where `listTasks({ slice })` would
-        // pass an unsliced task through *both* the train and eval filters because
-        // the early code only excluded tasks whose explicit slice differed.
-        const synthetic = {
-            id: "synthetic/no-slice-task",
-            title: "Synthetic task with no explicit slice",
-            domain: "synthetic",
-            difficulty: "easy",
-            stash: "minimal",
-            verifier: "regex",
-            budget: { tokens: 1000, wallMs: 1000 },
-            taskDir: "/tmp/none",
-        };
-        const slice = effectiveSlice(synthetic);
-        expect(["train", "eval"]).toContain(slice);
-        // Re-running must give the same bucket.
-        expect(effectiveSlice(synthetic)).toBe(slice);
-        // partitionSlice and effectiveSlice must agree.
-        const { train, eval: evalSlice } = partitionSlice([synthetic]);
-        if (slice === "train") {
-            expect(train.map((t) => t.id)).toEqual([synthetic.id]);
-            expect(evalSlice).toEqual([]);
-        }
-        else {
-            expect(evalSlice.map((t) => t.id)).toEqual([synthetic.id]);
-            expect(train).toEqual([]);
-        }
-    });
-});
-// ── Memory-operation tags (#262) ───────────────────────────────────────────
-describe("memory-operation tags (#262)", () => {
-    test("MEMORY_ABILITY_VALUES is the documented closed set", () => {
-        expect(new Set(MEMORY_ABILITY_VALUES)).toEqual(new Set([
-            "procedural_lookup",
-            "multi_asset_composition",
-            "temporal_update",
-            "conflict_resolution",
-            "abstention",
-            "noisy_retrieval",
-        ]));
-    });
-    test("loader leaves tag fields undefined for legacy tasks (without new fields)", () => {
-        // The shipped `_example/example-task` carries no #262 tags — it
-        // continues to load cleanly with every new field undefined.
-        const meta = loadTask("_example/example-task", { includeExamples: true });
-        expect(meta.memoryAbility).toBeUndefined();
-        expect(meta.taskFamily).toBeUndefined();
-        expect(meta.workflowFocus).toBeUndefined();
-        expect(meta.expectedTransferFrom).toBeUndefined();
-        expect(meta.abstentionCase).toBeUndefined();
-        expect(meta.conflictCase).toBeUndefined();
-        expect(meta.staleGuidanceCase).toBeUndefined();
-    });
-    test("loader parses memory_ability + task_family from a tagged task", () => {
-        // Every seeded corpus task is tagged with at least these two fields by
-        // #262. Pick a representative entry and assert the round-trip.
-        const meta = loadTask("docker-homelab/restart-policy");
-        expect(meta.memoryAbility).toBe("procedural_lookup");
-        expect(meta.taskFamily).toBe("docker-homelab/compose-basics");
-    });
-    test("every seeded corpus task carries memory_ability + task_family", () => {
-        for (const task of listTasks()) {
-            expect(task.memoryAbility).toBeDefined();
-            expect(MEMORY_ABILITY_VALUES).toContain(task.memoryAbility);
-            expect(task.taskFamily).toBeDefined();
-            expect(task.taskFamily).toMatch(/^[a-z0-9][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/);
-        }
-    });
-    test("invalid memory_ability values are dropped (loader stays permissive)", () => {
-        const dir = benchMkdtemp("akm-bench-tag-");
-        try {
-            const taskDir = path.join(dir, "_example", "bad-tag");
-            fs.mkdirSync(taskDir, { recursive: true });
-            // Write a clone of `_example/example-task` augmented with a bogus tag.
-            const yaml = [
-                "id: _example/bad-tag",
-                'title: "Bogus tag"',
-                "domain: _example",
-                "difficulty: easy",
-                "slice: train",
-                "stash: minimal",
-                "verifier: script",
-                "budget:",
-                "  tokens: 1000",
-                "  wallMs: 30000",
-                "memory_ability: not_a_real_ability",
-                "task_family: _example/bad",
-                "",
-            ].join("\n");
-            fs.writeFileSync(path.join(taskDir, "task.yaml"), yaml, "utf8");
-            const parsed = parseTaskYaml(readTaskBody(taskDir), taskDir);
-            expect(parsed).toBeDefined();
-            expect(parsed?.id).toBe("_example/bad-tag");
-            expect(parsed?.memoryAbility).toBeUndefined();
-            expect(parsed?.taskFamily).toBe("_example/bad");
-        }
-        finally {
-            fs.rmSync(dir, { recursive: true });
-        }
-    });
-});