akm-cli 0.7.0 → 0.7.2

package/dist/tests/bench/run-config.js

@@ -1,355 +0,0 @@
-/**
- * akm-bench run-config loader.
- *
- * A bench run config (`tests/bench/configs/*.json`) is a single-file
- * description of a utility/evolve invocation: providers, default model,
- * tasks, arms, seeds, budgets, parallel, baseline. Loading a config
- * resolves the providers file (from explicit `providers` / `providersRef`
- * fields, the `BENCH_OPENCODE_CONFIG` env var, or
- * `${XDG_CONFIG_HOME:-~/.config}/akm/bench-providers.json`), looks up the
- * effective default model, and resolves the task selector + baseline file
- * paths.
- *
- * Self-contained — does not import from `src/` so the bench framework
- * stays liftable to a standalone repo.
- */
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { listTasks, loadTask } from "./corpus";
-import { BenchConfigError, loadOpencodeProviders, } from "./opencode-config";
-import { benchMkdtemp } from "./tmp";
-/**
- * Resolve a path string supporting `~` expansion and `${VAR}` env-var
- * expansion. Relative paths are resolved against `baseDir`.
- */
-export function resolvePathString(value, baseDir) {
-    let s = value;
-    // Expand ${VAR} and $VAR forms — matches the conventional shell forms.
-    s = s.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_m, name) => process.env[name] ?? "");
-    s = s.replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (_m, name) => process.env[name] ?? "");
-    // Tilde expansion. `~` alone or `~/...`; we don't support `~user/`.
-    if (s === "~")
-        s = os.homedir();
-    else if (s.startsWith("~/"))
-        s = path.join(os.homedir(), s.slice(2));
-    if (path.isAbsolute(s))
-        return s;
-    return path.resolve(baseDir, s);
-}
-/** Default per-operator providers location: `${XDG_CONFIG_HOME:-~/.config}/akm/bench-providers.json`. */
-export function defaultUserProvidersPath() {
-    const xdg = process.env.XDG_CONFIG_HOME;
-    const root = xdg && xdg.length > 0 ? xdg : path.join(os.homedir(), ".config");
-    return path.join(root, "akm", "bench-providers.json");
-}
-/**
- * Resolve the providers file using the §A discovery chain and load it.
- *
- *   1. `BENCH_OPENCODE_CONFIG` env var (absolute path).
- *   2. `providers` inline in the config (materialised to a synthetic
- *      `LoadedOpencodeProviders` without touching disk).
- *   3. `providersRef` in the config (with tilde / env-var expansion).
- *   4. `${XDG_CONFIG_HOME:-~/.config}/akm/bench-providers.json`.
- *   5. Throw — the caller is expected to map this to exit code 2.
- *
- * Returns `{ providers, source }` where `source` is the absolute path the
- * providers came from (or `"<inline>"` for the inline case).
- */
-export function resolveProviders(config, configDir) {
-    // 1. BENCH_OPENCODE_CONFIG env var wins.
-    const envPath = process.env.BENCH_OPENCODE_CONFIG;
-    if (envPath && envPath.length > 0) {
-        return loadOpencodeProviders(path.isAbsolute(envPath) ? envPath : path.resolve(envPath));
-    }
-    // 2. Inline providers in the config.
-    if (config.providers !== undefined) {
-        if (config.providersRef !== undefined) {
-            throw new BenchConfigError("bench run config: only one of `providers` or `providersRef` may be set", true);
-        }
-        return materialiseInlineProviders(config);
-    }
-    // 3. Explicit providersRef.
-    if (config.providersRef !== undefined) {
-        const resolved = resolvePathString(config.providersRef, configDir);
-        return loadOpencodeProviders(resolved);
-    }
-    // 4. Per-operator default location.
-    const userPath = defaultUserProvidersPath();
-    if (fs.existsSync(userPath)) {
-        return loadOpencodeProviders(userPath);
-    }
-    // 5. Repo-local fallbacks — the same locations the legacy
-    //    `discoverOpencodeProviders` checks. The gitignored `.local.json`
-    //    overlay wins over the committed fixture so an operator's local
-    //    overrides survive a `git pull` without needing a config edit.
-    const repoLocalPath = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
-    if (fs.existsSync(repoLocalPath)) {
-        return loadOpencodeProviders(repoLocalPath);
-    }
-    const repoFixturePath = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
-    if (fs.existsSync(repoFixturePath)) {
-        return loadOpencodeProviders(repoFixturePath);
-    }
-    // 6. No providers found.
-    throw new BenchConfigError(`bench run config: no opencode providers found. Set \`providers\` or \`providersRef\` in the config, set BENCH_OPENCODE_CONFIG, or create ${userPath}.`, true);
-}
-/**
- * Build a `LoadedOpencodeProviders` from an inline `providers` map without
- * round-tripping through disk. We still validate via `loadOpencodeProviders`
- * by writing to a tmp file? No — that would risk leaving secrets on disk.
- * Instead, do a minimal in-memory validation that matches what the on-disk
- * loader checks (forbidden top-level keys are not applicable here, since
- * the inline providers already live inside a `providers` object; but the
- * credential heuristic still applies).
- */
-function materialiseInlineProviders(config) {
-    if (config.providers === null || typeof config.providers !== "object" || Array.isArray(config.providers)) {
-        throw new BenchConfigError("bench run config: `providers` must be an object", false);
-    }
-    // Reuse `loadOpencodeProviders` indirectly by stamping a synthetic
-    // BenchOpencodeProvidersFile — without touching disk we still want the
-    // credential scan applied. The simplest path is: write a tmp file mode
-    // 0o600 and load it, then unlink. That keeps the credential-scan logic
-    // co-located in opencode-config.ts.
-    const file = {
-        schemaVersion: 1,
-        providers: config.providers,
-        ...(config.defaultModel !== undefined ? { defaultModel: config.defaultModel } : {}),
-    };
-    // Per #276: bench tmp dirs MUST live under `${AKM_CACHE_DIR}/bench/`,
-    // never the OS-default tmp root. `benchMkdtemp` is the drop-in.
-    const tmpDir = benchMkdtemp("akm-bench-inline-");
-    const tmpPath = path.join(tmpDir, "providers.json");
-    try {
-        fs.writeFileSync(tmpPath, JSON.stringify(file), { mode: 0o600 });
-        const loaded = loadOpencodeProviders(tmpPath);
-        return { ...loaded, source: "<inline>" };
-    }
-    finally {
-        try {
-            fs.rmSync(tmpDir, { recursive: true, force: true });
-        }
-        catch {
-            // best-effort cleanup
-        }
-    }
-}
-/** Load + validate a baseline JSON file: `{ taskId: passRate (0..1) }`. */
-export function loadBaseline(absPath) {
-    let raw;
-    try {
-        raw = fs.readFileSync(absPath, "utf8");
-    }
-    catch (err) {
-        throw new BenchConfigError(`bench run config: cannot read baseline file "${absPath}": ${err instanceof Error ? err.message : String(err)}`, true);
-    }
-    let parsed;
-    try {
-        parsed = JSON.parse(raw);
-    }
-    catch (err) {
-        throw new BenchConfigError(`bench run config: baseline file "${absPath}" is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, false);
-    }
-    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
-        throw new BenchConfigError(`bench run config: baseline file "${absPath}" must be a JSON object of taskId → passRate`, false);
-    }
-    const out = {};
-    for (const [key, value] of Object.entries(parsed)) {
-        if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
-            throw new BenchConfigError(`bench run config: baseline entry ${JSON.stringify(key)} in "${absPath}" must be a number in [0, 1]; got ${JSON.stringify(value)}`, false);
-        }
-        out[key] = value;
-    }
-    return out;
-}
-/**
- * Resolve the `tasks` selector to a concrete `TaskMetadata[]` plus a slice
- * label for the report's `corpus.slice` field.
- */
-export function resolveTasks(selector) {
-    // Default = "all" when the field is omitted entirely.
-    if (selector === undefined) {
-        return { tasks: listTasks(), slice: "all" };
-    }
-    if (typeof selector === "string") {
-        if (selector === "all" || selector === "train" || selector === "eval") {
-            const sliceFilter = selector === "all" ? undefined : selector;
-            const tasks = listTasks(sliceFilter ? { slice: sliceFilter } : {});
-            return { tasks, slice: selector };
-        }
-        // Single task id ("domain/name") — try direct lookup first.
-        if (selector.includes("/")) {
-            try {
-                return { tasks: [loadTask(selector)], slice: "all" };
-            }
-            catch {
-                // Fall through to "no match" error below.
-            }
-            throw new BenchConfigError(`bench run config: tasks: no task matched "${selector}"`, true);
-        }
-        // Domain prefix (no slash).
-        const all = listTasks();
-        const matches = all.filter((t) => t.domain === selector);
-        if (matches.length === 0) {
-            throw new BenchConfigError(`bench run config: tasks: no task matched domain "${selector}". Available domains: ${[...new Set(all.map((t) => t.domain))].sort().join(", ") || "(none)"}`, true);
-        }
-        return { tasks: matches, slice: "all" };
-    }
-    // Array of task ids.
-    if (selector.length === 0) {
-        throw new BenchConfigError("bench run config: tasks: array must be non-empty", true);
-    }
-    const out = [];
-    for (const id of selector) {
-        try {
-            out.push(loadTask(id));
-        }
-        catch {
-            throw new BenchConfigError(`bench run config: tasks: no task matched "${id}"`, true);
-        }
-    }
-    return { tasks: out, slice: "all" };
-}
-/**
- * Validate the parsed config against the v1 schema (in-code, no JSON
- * Schema runtime — keeps the bench self-contained). Throws BenchConfigError
- * on the first violation.
- */
-function validateConfig(parsed, source) {
-    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
-        throw new BenchConfigError(`bench run config: root of ${source} must be a JSON object`, false);
-    }
-    const obj = parsed;
-    if (obj.schemaVersion !== 1) {
-        throw new BenchConfigError(`bench run config: ${source}: unsupported schemaVersion ${JSON.stringify(obj.schemaVersion)}; expected 1`, false);
-    }
-    const allowed = new Set([
-        "$schema",
-        "schemaVersion",
-        "name",
-        "description",
-        "providers",
-        "providersRef",
-        "defaultModel",
-        "tasks",
-        "arms",
-        "seeds",
-        "budgetTokens",
-        "budgetWallMs",
-        "parallel",
-        "forceParallel",
-        "baseline",
-    ]);
-    for (const key of Object.keys(obj)) {
-        if (!allowed.has(key)) {
-            throw new BenchConfigError(`bench run config: ${source}: unknown field "${key}"`, false);
-        }
-    }
-    if (obj.providers !== undefined && obj.providersRef !== undefined) {
-        throw new BenchConfigError(`bench run config: ${source}: only one of "providers" or "providersRef" may be set`, true);
-    }
-    if (obj.tasks !== undefined) {
-        if (typeof obj.tasks !== "string" && !Array.isArray(obj.tasks)) {
-            throw new BenchConfigError(`bench run config: ${source}: "tasks" must be a string or array of strings`, false);
-        }
-        if (Array.isArray(obj.tasks)) {
-            for (const t of obj.tasks) {
-                if (typeof t !== "string") {
-                    throw new BenchConfigError(`bench run config: ${source}: every entry in "tasks" must be a string`, false);
-                }
-            }
-        }
-    }
-    if (obj.arms !== undefined) {
-        if (!Array.isArray(obj.arms) || obj.arms.length === 0) {
-            throw new BenchConfigError(`bench run config: ${source}: "arms" must be a non-empty array`, false);
-        }
-        for (const a of obj.arms) {
-            if (a !== "noakm" && a !== "akm" && a !== "synthetic") {
-                throw new BenchConfigError(`bench run config: ${source}: invalid arm ${JSON.stringify(a)}; expected one of "noakm", "akm", "synthetic"`, false);
-            }
-        }
-    }
-    for (const numField of ["seeds", "budgetTokens", "budgetWallMs", "parallel"]) {
-        const val = obj[numField];
-        if (val !== undefined) {
-            if (typeof val !== "number" || !Number.isInteger(val) || val < 1) {
-                throw new BenchConfigError(`bench run config: ${source}: "${numField}" must be a positive integer`, false);
-            }
-        }
-    }
-    return obj;
-}
-/**
- * Load and resolve a bench run config from disk.
- *
- * @param configPath  Absolute or relative path to the config JSON file.
- * @param overrides   CLI-derived overrides applied on top of the config.
- */
-export function loadBenchRunConfig(configPath, overrides = {}) {
-    const absPath = path.isAbsolute(configPath) ? configPath : path.resolve(configPath);
-    if (!fs.existsSync(absPath)) {
-        throw new BenchConfigError(`bench run config: file not found: ${absPath}`, true);
-    }
-    let raw;
-    try {
-        raw = fs.readFileSync(absPath, "utf8");
-    }
-    catch (err) {
-        throw new BenchConfigError(`bench run config: cannot read ${absPath}: ${err instanceof Error ? err.message : String(err)}`, true);
-    }
-    let parsed;
-    try {
-        parsed = JSON.parse(raw);
-    }
-    catch (err) {
-        throw new BenchConfigError(`bench run config: ${absPath}: invalid JSON: ${err instanceof Error ? err.message : String(err)}`, false);
-    }
-    const config = validateConfig(parsed, absPath);
-    const configDir = path.dirname(absPath);
-    const providers = resolveProviders(config, configDir);
-    const envModel = process.env.BENCH_OPENCODE_MODEL;
-    const model = (envModel && envModel.length > 0 ? envModel : undefined) ?? config.defaultModel ?? providers.defaultModel;
-    if (!model) {
-        throw new BenchConfigError(`bench run config: ${absPath}: no model specified. Set "defaultModel" in the config, set "defaultModel" in the providers file, or set BENCH_OPENCODE_MODEL.`, true);
-    }
-    // Resolve tasks (with optional CLI list override).
-    let resolved = resolveTasks(config.tasks);
-    if (overrides.tasksList && overrides.tasksList.length > 0) {
-        const set = new Set(overrides.tasksList);
-        const filtered = resolved.tasks.filter((t) => set.has(t.id));
-        const missing = overrides.tasksList.filter((id) => !resolved.tasks.some((t) => t.id === id));
-        if (missing.length > 0) {
-            throw new BenchConfigError(`bench run config: --tasks override: no task in the config matched ${JSON.stringify(missing.join(", "))}`, true);
-        }
-        resolved = { tasks: filtered, slice: resolved.slice };
-    }
-    if (resolved.tasks.length === 0) {
-        throw new BenchConfigError(`bench run config: ${absPath}: task selector matched zero tasks`, true);
-    }
-    let baselineByTaskId;
-    if (config.baseline) {
-        const baselinePath = resolvePathString(config.baseline, configDir);
-        baselineByTaskId = loadBaseline(baselinePath);
-    }
-    const arms = config.arms ?? ["noakm", "akm"];
-    const seedsPerArm = overrides.seedsPerArm ?? config.seeds;
-    const parallel = overrides.parallel ?? config.parallel;
-    const name = config.name ?? path.basename(absPath, path.extname(absPath));
-    return {
-        source: absPath,
-        name,
-        providers,
-        model,
-        tasks: resolved.tasks,
-        arms,
-        ...(seedsPerArm !== undefined ? { seedsPerArm } : {}),
-        ...(config.budgetTokens !== undefined ? { budgetTokens: config.budgetTokens } : {}),
-        ...(config.budgetWallMs !== undefined ? { budgetWallMs: config.budgetWallMs } : {}),
-        ...(parallel !== undefined ? { parallel } : {}),
-        ...(config.forceParallel ? { forceParallel: true } : {}),
-        ...(baselineByTaskId ? { baselineByTaskId } : {}),
-        slice: resolved.slice,
-    };
-}

package/dist/tests/bench/run-config.test.js

@@ -1,298 +0,0 @@
-/**
- * Unit tests for the bench run-config loader (`tests/bench/run-config.ts`).
- *
- * Covers the parts that don't require spawning a process:
- * - Schema validation (unknown fields, missing schemaVersion, bad arms).
- * - Path resolution (~ expansion, ${VAR} expansion, relative vs absolute).
- * - Provider discovery chain (env > inline > providersRef > XDG default).
- * - Baseline-file loading + range checks.
- * - Task selector resolution (slice / domain / id / array).
- *
- * The CLI-level dispatch is exercised by `cli.test.ts` via spawned bench
- * runs — keep those for end-to-end coverage; this file is unit-grade.
- */
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { defaultUserProvidersPath, loadBaseline, loadBenchRunConfig, resolvePathString } from "./run-config";
-import { benchMkdtemp } from "./tmp";
-const REPO_ROOT = path.resolve(__dirname, "..", "..");
-let workDir;
-let savedEnv;
-beforeEach(() => {
-    // Per #276 invariant: bench tmp dirs live under `${AKM_CACHE_DIR}/bench/`,
-    // never the OS-default tmp root. `benchMkdtemp` is the drop-in.
-    workDir = benchMkdtemp("akm-bench-runconfig-test-");
-    savedEnv = {
-        BENCH_OPENCODE_CONFIG: process.env.BENCH_OPENCODE_CONFIG,
-        BENCH_OPENCODE_MODEL: process.env.BENCH_OPENCODE_MODEL,
-        AKM_TEST_VAR: process.env.AKM_TEST_VAR,
-    };
-    delete process.env.BENCH_OPENCODE_CONFIG;
-    delete process.env.BENCH_OPENCODE_MODEL;
-});
-afterEach(() => {
-    fs.rmSync(workDir, { recursive: true, force: true });
-    for (const [k, v] of Object.entries(savedEnv)) {
-        if (v === undefined)
-            delete process.env[k];
-        else
-            process.env[k] = v;
-    }
-});
-function writeProvidersFile(filePath, defaultModel = "p/m") {
-    fs.mkdirSync(path.dirname(filePath), { recursive: true });
-    fs.writeFileSync(filePath, JSON.stringify({
-        schemaVersion: 1,
-        defaultModel,
-        providers: { p: { npm: "@ai-sdk/openai-compatible" } },
-    }));
-}
-describe("resolvePathString", () => {
-    test("resolves a relative path against the supplied base dir", () => {
-        expect(resolvePathString("foo.json", "/work")).toBe("/work/foo.json");
-    });
-    test("returns absolute paths unchanged", () => {
-        expect(resolvePathString("/abs/path.json", "/work")).toBe("/abs/path.json");
-    });
-    test("expands `~` to the operator's home dir", () => {
-        expect(resolvePathString("~/.config/akm/foo.json", "/work")).toBe(path.join(os.homedir(), ".config/akm/foo.json"));
-    });
-    test("expands env-var references", () => {
-        // Build the input with concatenation rather than a string literal to avoid
-        // biome's noTemplateCurlyInString flag on the `\${VAR}` form.
-        process.env.AKM_TEST_VAR = "/somewhere";
-        const input = `${"$"}{AKM_TEST_VAR}/providers.json`;
-        expect(resolvePathString(input, "/work")).toBe("/somewhere/providers.json");
-    });
-});
-describe("defaultUserProvidersPath", () => {
-    test("respects XDG_CONFIG_HOME when set", () => {
-        const saved = process.env.XDG_CONFIG_HOME;
-        process.env.XDG_CONFIG_HOME = "/xdg-test";
-        try {
-            expect(defaultUserProvidersPath()).toBe("/xdg-test/akm/bench-providers.json");
-        }
-        finally {
-            if (saved === undefined)
-                delete process.env.XDG_CONFIG_HOME;
-            else
-                process.env.XDG_CONFIG_HOME = saved;
-        }
-    });
-    test("falls back to ~/.config when XDG_CONFIG_HOME is unset", () => {
-        const saved = process.env.XDG_CONFIG_HOME;
-        delete process.env.XDG_CONFIG_HOME;
-        try {
-            expect(defaultUserProvidersPath()).toBe(path.join(os.homedir(), ".config/akm/bench-providers.json"));
-        }
-        finally {
-            if (saved !== undefined)
-                process.env.XDG_CONFIG_HOME = saved;
-        }
-    });
-});
-describe("loadBaseline", () => {
-    test("loads a `{ taskId: passRate }` map", () => {
-        const filePath = path.join(workDir, "baseline.json");
-        fs.writeFileSync(filePath, JSON.stringify({ "domain/a": 0.8, "domain/b": 1.0 }));
-        expect(loadBaseline(filePath)).toEqual({ "domain/a": 0.8, "domain/b": 1.0 });
-    });
-    test("rejects pass rates outside [0, 1]", () => {
-        const filePath = path.join(workDir, "bad.json");
-        fs.writeFileSync(filePath, JSON.stringify({ "x/y": 1.5 }));
-        expect(() => loadBaseline(filePath)).toThrow(/must be a number in \[0, 1\]/);
-    });
-    test("rejects non-number values", () => {
-        const filePath = path.join(workDir, "non-number.json");
-        fs.writeFileSync(filePath, JSON.stringify({ "x/y": "not a number" }));
-        expect(() => loadBaseline(filePath)).toThrow(/must be a number/);
-    });
-});
-describe("loadBenchRunConfig — schema validation", () => {
-    test("rejects unknown top-level fields", () => {
-        const cfgPath = path.join(workDir, "bad.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({ schemaVersion: 1, name: "x", weirdField: 42 }));
-        expect(() => loadBenchRunConfig(cfgPath)).toThrow(/unknown field "weirdField"/);
-    });
-    test("rejects missing schemaVersion", () => {
-        const cfgPath = path.join(workDir, "noversion.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({ name: "x" }));
-        expect(() => loadBenchRunConfig(cfgPath)).toThrow(/unsupported schemaVersion/);
-    });
-    test("rejects providers AND providersRef both set", () => {
-        const cfgPath = path.join(workDir, "both.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providers: { p: { npm: "x" } },
-            providersRef: "./other.json",
-        }));
-        expect(() => loadBenchRunConfig(cfgPath)).toThrow(/only one of "providers" or "providersRef"/);
-    });
-    test("rejects bad arm values", () => {
-        const cfgPath = path.join(workDir, "badarm.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({ schemaVersion: 1, arms: ["nope"] }));
-        expect(() => loadBenchRunConfig(cfgPath)).toThrow(/invalid arm/);
-    });
-    test("missing config file exits with usage error", () => {
-        expect(() => loadBenchRunConfig(path.join(workDir, "ghost.json"))).toThrow(/file not found/);
-    });
-});
-describe("loadBenchRunConfig — provider discovery", () => {
-    test("BENCH_OPENCODE_CONFIG env var wins over providersRef", () => {
-        const envProviders = path.join(workDir, "env-providers.json");
-        const refProviders = path.join(workDir, "ref-providers.json");
-        writeProvidersFile(envProviders, "env/model");
-        writeProvidersFile(refProviders, "ref/model");
-        process.env.BENCH_OPENCODE_CONFIG = envProviders;
-        const cfgPath = path.join(workDir, "config.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./ref-providers.json",
-            tasks: "all",
-        }));
-        // No tasks resolved so we can't actually load — just verify provider
-        // resolution. We restrict to a single committed task to satisfy the
-        // selector. The bench corpus exists at fixtures/bench/tasks; we use
-        // "all" as the selector and skip past the `tasks=0` exit by writing a
-        // selector that matches a real task.
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./ref-providers.json",
-            tasks: ["drillbit/backup-policy"],
-        }));
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.providers.source).toBe(envProviders);
-        expect(resolved.model).toBe("env/model");
-    });
-    test("`providersRef` is resolved relative to the config file", () => {
-        const refProviders = path.join(workDir, "subdir", "providers.json");
-        writeProvidersFile(refProviders, "ref/model");
-        const cfgPath = path.join(workDir, "config.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./subdir/providers.json",
-            tasks: ["drillbit/backup-policy"],
-        }));
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.providers.source).toBe(refProviders);
-        expect(resolved.model).toBe("ref/model");
-    });
-    test("config `defaultModel` overrides the providers file's defaultModel", () => {
-        const refProviders = path.join(workDir, "providers.json");
-        writeProvidersFile(refProviders, "ref/model");
-        const cfgPath = path.join(workDir, "config.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./providers.json",
-            defaultModel: "config/model",
-            tasks: ["drillbit/backup-policy"],
-        }));
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.model).toBe("config/model");
-    });
-    test("BENCH_OPENCODE_MODEL env wins over both", () => {
-        const refProviders = path.join(workDir, "providers.json");
-        writeProvidersFile(refProviders, "ref/model");
-        process.env.BENCH_OPENCODE_MODEL = "env/model";
-        const cfgPath = path.join(workDir, "config.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./providers.json",
-            defaultModel: "config/model",
-            tasks: ["drillbit/backup-policy"],
-        }));
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.model).toBe("env/model");
-    });
-});
-describe("loadBenchRunConfig — task resolution", () => {
-    test("tasks=array selects exactly the listed ids", () => {
-        const refProviders = path.join(workDir, "providers.json");
-        writeProvidersFile(refProviders);
-        const cfgPath = path.join(workDir, "config.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./providers.json",
-            tasks: ["drillbit/backup-policy", "drillbit/canary-enable"],
-        }));
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.tasks.map((t) => t.id).sort()).toEqual(["drillbit/backup-policy", "drillbit/canary-enable"]);
-    });
-    test("tasks=domain matches every task whose domain matches", () => {
-        const refProviders = path.join(workDir, "providers.json");
-        writeProvidersFile(refProviders);
-        const cfgPath = path.join(workDir, "config.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./providers.json",
-            tasks: "drillbit",
-        }));
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.tasks.length).toBeGreaterThan(0);
-        for (const t of resolved.tasks)
-            expect(t.domain).toBe("drillbit");
-    });
-    test("tasks=single-id matches exactly that task", () => {
-        const refProviders = path.join(workDir, "providers.json");
-        writeProvidersFile(refProviders);
-        const cfgPath = path.join(workDir, "config.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./providers.json",
-            tasks: "drillbit/backup-policy",
-        }));
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.tasks.map((t) => t.id)).toEqual(["drillbit/backup-policy"]);
-    });
-    test("--tasks override (CLI) restricts to a subset of the config's selection", () => {
-        const refProviders = path.join(workDir, "providers.json");
-        writeProvidersFile(refProviders);
-        const cfgPath = path.join(workDir, "config.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./providers.json",
-            tasks: ["drillbit/backup-policy", "drillbit/canary-enable"],
-        }));
-        const resolved = loadBenchRunConfig(cfgPath, { tasksList: ["drillbit/canary-enable"] });
-        expect(resolved.tasks.map((t) => t.id)).toEqual(["drillbit/canary-enable"]);
-    });
-    test("baseline path is resolved relative to the config file", () => {
-        const refProviders = path.join(workDir, "providers.json");
-        writeProvidersFile(refProviders);
-        const baselinePath = path.join(workDir, "baseline.json");
-        fs.writeFileSync(baselinePath, JSON.stringify({ "drillbit/backup-policy": 0.8 }));
-        const cfgPath = path.join(workDir, "config.json");
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            schemaVersion: 1,
-            providersRef: "./providers.json",
-            tasks: ["drillbit/backup-policy"],
-            baseline: "./baseline.json",
-        }));
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.baselineByTaskId).toEqual({ "drillbit/backup-policy": 0.8 });
-    });
-});
-describe("loadBenchRunConfig — committed configs validate", () => {
-    test("tests/bench/configs/nano-quick.json loads cleanly", () => {
-        const cfgPath = path.join(REPO_ROOT, "tests", "bench", "configs", "nano-quick.json");
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.name).toBe("nano-quick");
-        expect(resolved.arms).toEqual(["akm"]);
-        expect(resolved.seedsPerArm).toBe(2);
-        expect(resolved.tasks.length).toBe(5);
-    });
-    test("tests/bench/configs/full.json loads cleanly and carries the baseline", () => {
-        const cfgPath = path.join(REPO_ROOT, "tests", "bench", "configs", "full.json");
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.name).toBe("full");
-        expect(resolved.baselineByTaskId).toBeDefined();
-        expect(typeof resolved.baselineByTaskId?.["drillbit/backup-policy"]).toBe("number");
-    });
-    test("tests/bench/configs/curate-test.json restricts to one task", () => {
-        const cfgPath = path.join(REPO_ROOT, "tests", "bench", "configs", "curate-test.json");
-        const resolved = loadBenchRunConfig(cfgPath);
-        expect(resolved.tasks.map((t) => t.id)).toEqual(["inkwell/configure-scaling"]);
-    });
-});