akm-cli 0.6.1 → 0.7.0

package/dist/tests/bench/run-config.js

@@ -0,0 +1,355 @@
+/**
+ * akm-bench run-config loader.
+ *
+ * A bench run config (`tests/bench/configs/*.json`) is a single-file
+ * description of a utility/evolve invocation: providers, default model,
+ * tasks, arms, seeds, budgets, parallel, baseline. Loading a config
+ * resolves the providers file (from explicit `providers` / `providersRef`
+ * fields, the `BENCH_OPENCODE_CONFIG` env var, or
+ * `${XDG_CONFIG_HOME:-~/.config}/akm/bench-providers.json`), looks up the
+ * effective default model, and resolves the task selector + baseline file
+ * paths.
+ *
+ * Self-contained — does not import from `src/` so the bench framework
+ * stays liftable to a standalone repo.
+ */
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { listTasks, loadTask } from "./corpus";
+import { BenchConfigError, loadOpencodeProviders, } from "./opencode-config";
+import { benchMkdtemp } from "./tmp";
+/**
+ * Resolve a path string supporting `~` expansion and `${VAR}` env-var
+ * expansion. Relative paths are resolved against `baseDir`.
+ */
+export function resolvePathString(value, baseDir) {
+    let s = value;
+    // Expand ${VAR} and $VAR forms — matches the conventional shell forms.
+    s = s.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_m, name) => process.env[name] ?? "");
+    s = s.replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (_m, name) => process.env[name] ?? "");
+    // Tilde expansion. `~` alone or `~/...`; we don't support `~user/`.
+    if (s === "~")
+        s = os.homedir();
+    else if (s.startsWith("~/"))
+        s = path.join(os.homedir(), s.slice(2));
+    if (path.isAbsolute(s))
+        return s;
+    return path.resolve(baseDir, s);
+}
+/** Default per-operator providers location: `${XDG_CONFIG_HOME:-~/.config}/akm/bench-providers.json`. */
+export function defaultUserProvidersPath() {
+    const xdg = process.env.XDG_CONFIG_HOME;
+    const root = xdg && xdg.length > 0 ? xdg : path.join(os.homedir(), ".config");
+    return path.join(root, "akm", "bench-providers.json");
+}
+/**
+ * Resolve the providers file using the §A discovery chain and load it.
+ *
+ *   1. `BENCH_OPENCODE_CONFIG` env var (absolute path).
+ *   2. `providers` inline in the config (materialised to a synthetic
+ *      `LoadedOpencodeProviders` without touching disk).
+ *   3. `providersRef` in the config (with tilde / env-var expansion).
+ *   4. `${XDG_CONFIG_HOME:-~/.config}/akm/bench-providers.json`.
+ *   5. Throw — the caller is expected to map this to exit code 2.
+ *
+ * Returns `{ providers, source }` where `source` is the absolute path the
+ * providers came from (or `"<inline>"` for the inline case).
+ */
+export function resolveProviders(config, configDir) {
+    // 1. BENCH_OPENCODE_CONFIG env var wins.
+    const envPath = process.env.BENCH_OPENCODE_CONFIG;
+    if (envPath && envPath.length > 0) {
+        return loadOpencodeProviders(path.isAbsolute(envPath) ? envPath : path.resolve(envPath));
+    }
+    // 2. Inline providers in the config.
+    if (config.providers !== undefined) {
+        if (config.providersRef !== undefined) {
+            throw new BenchConfigError("bench run config: only one of `providers` or `providersRef` may be set", true);
+        }
+        return materialiseInlineProviders(config);
+    }
+    // 3. Explicit providersRef.
+    if (config.providersRef !== undefined) {
+        const resolved = resolvePathString(config.providersRef, configDir);
+        return loadOpencodeProviders(resolved);
+    }
+    // 4. Per-operator default location.
+    const userPath = defaultUserProvidersPath();
+    if (fs.existsSync(userPath)) {
+        return loadOpencodeProviders(userPath);
+    }
+    // 5. Repo-local fallbacks — the same locations the legacy
+    //    `discoverOpencodeProviders` checks. The gitignored `.local.json`
+    //    overlay wins over the committed fixture so an operator's local
+    //    overrides survive a `git pull` without needing a config edit.
+    const repoLocalPath = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.local.json");
+    if (fs.existsSync(repoLocalPath)) {
+        return loadOpencodeProviders(repoLocalPath);
+    }
+    const repoFixturePath = path.resolve(__dirname, "..", "fixtures", "bench", "opencode-providers.json");
+    if (fs.existsSync(repoFixturePath)) {
+        return loadOpencodeProviders(repoFixturePath);
+    }
+    // 6. No providers found.
+    throw new BenchConfigError(`bench run config: no opencode providers found. Set \`providers\` or \`providersRef\` in the config, set BENCH_OPENCODE_CONFIG, or create ${userPath}.`, true);
+}
+/**
+ * Build a `LoadedOpencodeProviders` from an inline `providers` map without
+ * round-tripping through disk. We still validate via `loadOpencodeProviders`
+ * by writing to a tmp file? No — that would risk leaving secrets on disk.
+ * Instead, do a minimal in-memory validation that matches what the on-disk
+ * loader checks (forbidden top-level keys are not applicable here, since
+ * the inline providers already live inside a `providers` object; but the
+ * credential heuristic still applies).
+ */
+function materialiseInlineProviders(config) {
+    if (config.providers === null || typeof config.providers !== "object" || Array.isArray(config.providers)) {
+        throw new BenchConfigError("bench run config: `providers` must be an object", false);
+    }
+    // Reuse `loadOpencodeProviders` indirectly by stamping a synthetic
+    // BenchOpencodeProvidersFile — without touching disk we still want the
+    // credential scan applied. The simplest path is: write a tmp file mode
+    // 0o600 and load it, then unlink. That keeps the credential-scan logic
+    // co-located in opencode-config.ts.
+    const file = {
+        schemaVersion: 1,
+        providers: config.providers,
+        ...(config.defaultModel !== undefined ? { defaultModel: config.defaultModel } : {}),
+    };
+    // Per #276: bench tmp dirs MUST live under `${AKM_CACHE_DIR}/bench/`,
+    // never the OS-default tmp root. `benchMkdtemp` is the drop-in.
+    const tmpDir = benchMkdtemp("akm-bench-inline-");
+    const tmpPath = path.join(tmpDir, "providers.json");
+    try {
+        fs.writeFileSync(tmpPath, JSON.stringify(file), { mode: 0o600 });
+        const loaded = loadOpencodeProviders(tmpPath);
+        return { ...loaded, source: "<inline>" };
+    }
+    finally {
+        try {
+            fs.rmSync(tmpDir, { recursive: true, force: true });
+        }
+        catch {
+            // best-effort cleanup
+        }
+    }
+}
+/** Load + validate a baseline JSON file: `{ taskId: passRate (0..1) }`. */
+export function loadBaseline(absPath) {
+    let raw;
+    try {
+        raw = fs.readFileSync(absPath, "utf8");
+    }
+    catch (err) {
+        throw new BenchConfigError(`bench run config: cannot read baseline file "${absPath}": ${err instanceof Error ? err.message : String(err)}`, true);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (err) {
+        throw new BenchConfigError(`bench run config: baseline file "${absPath}" is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, false);
+    }
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+        throw new BenchConfigError(`bench run config: baseline file "${absPath}" must be a JSON object of taskId → passRate`, false);
+    }
+    const out = {};
+    for (const [key, value] of Object.entries(parsed)) {
+        if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
+            throw new BenchConfigError(`bench run config: baseline entry ${JSON.stringify(key)} in "${absPath}" must be a number in [0, 1]; got ${JSON.stringify(value)}`, false);
+        }
+        out[key] = value;
+    }
+    return out;
+}
+/**
+ * Resolve the `tasks` selector to a concrete `TaskMetadata[]` plus a slice
+ * label for the report's `corpus.slice` field.
+ */
+export function resolveTasks(selector) {
+    // Default = "all" when the field is omitted entirely.
+    if (selector === undefined) {
+        return { tasks: listTasks(), slice: "all" };
+    }
+    if (typeof selector === "string") {
+        if (selector === "all" || selector === "train" || selector === "eval") {
+            const sliceFilter = selector === "all" ? undefined : selector;
+            const tasks = listTasks(sliceFilter ? { slice: sliceFilter } : {});
+            return { tasks, slice: selector };
+        }
+        // Single task id ("domain/name") — try direct lookup first.
+        if (selector.includes("/")) {
+            try {
+                return { tasks: [loadTask(selector)], slice: "all" };
+            }
+            catch {
+                // Fall through to "no match" error below.
+            }
+            throw new BenchConfigError(`bench run config: tasks: no task matched "${selector}"`, true);
+        }
+        // Domain prefix (no slash).
+        const all = listTasks();
+        const matches = all.filter((t) => t.domain === selector);
+        if (matches.length === 0) {
+            throw new BenchConfigError(`bench run config: tasks: no task matched domain "${selector}". Available domains: ${[...new Set(all.map((t) => t.domain))].sort().join(", ") || "(none)"}`, true);
+        }
+        return { tasks: matches, slice: "all" };
+    }
+    // Array of task ids.
+    if (selector.length === 0) {
+        throw new BenchConfigError("bench run config: tasks: array must be non-empty", true);
+    }
+    const out = [];
+    for (const id of selector) {
+        try {
+            out.push(loadTask(id));
+        }
+        catch {
+            throw new BenchConfigError(`bench run config: tasks: no task matched "${id}"`, true);
+        }
+    }
+    return { tasks: out, slice: "all" };
+}
+/**
+ * Validate the parsed config against the v1 schema (in-code, no JSON
+ * Schema runtime — keeps the bench self-contained). Throws BenchConfigError
+ * on the first violation.
+ */
+function validateConfig(parsed, source) {
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+        throw new BenchConfigError(`bench run config: root of ${source} must be a JSON object`, false);
+    }
+    const obj = parsed;
+    if (obj.schemaVersion !== 1) {
+        throw new BenchConfigError(`bench run config: ${source}: unsupported schemaVersion ${JSON.stringify(obj.schemaVersion)}; expected 1`, false);
+    }
+    const allowed = new Set([
+        "$schema",
+        "schemaVersion",
+        "name",
+        "description",
+        "providers",
+        "providersRef",
+        "defaultModel",
+        "tasks",
+        "arms",
+        "seeds",
+        "budgetTokens",
+        "budgetWallMs",
+        "parallel",
+        "forceParallel",
+        "baseline",
+    ]);
+    for (const key of Object.keys(obj)) {
+        if (!allowed.has(key)) {
+            throw new BenchConfigError(`bench run config: ${source}: unknown field "${key}"`, false);
+        }
+    }
+    if (obj.providers !== undefined && obj.providersRef !== undefined) {
+        throw new BenchConfigError(`bench run config: ${source}: only one of "providers" or "providersRef" may be set`, true);
+    }
+    if (obj.tasks !== undefined) {
+        if (typeof obj.tasks !== "string" && !Array.isArray(obj.tasks)) {
+            throw new BenchConfigError(`bench run config: ${source}: "tasks" must be a string or array of strings`, false);
+        }
+        if (Array.isArray(obj.tasks)) {
+            for (const t of obj.tasks) {
+                if (typeof t !== "string") {
+                    throw new BenchConfigError(`bench run config: ${source}: every entry in "tasks" must be a string`, false);
+                }
+            }
+        }
+    }
+    if (obj.arms !== undefined) {
+        if (!Array.isArray(obj.arms) || obj.arms.length === 0) {
+            throw new BenchConfigError(`bench run config: ${source}: "arms" must be a non-empty array`, false);
+        }
+        for (const a of obj.arms) {
+            if (a !== "noakm" && a !== "akm" && a !== "synthetic") {
+                throw new BenchConfigError(`bench run config: ${source}: invalid arm ${JSON.stringify(a)}; expected one of "noakm", "akm", "synthetic"`, false);
+            }
+        }
+    }
+    for (const numField of ["seeds", "budgetTokens", "budgetWallMs", "parallel"]) {
+        const val = obj[numField];
+        if (val !== undefined) {
+            if (typeof val !== "number" || !Number.isInteger(val) || val < 1) {
+                throw new BenchConfigError(`bench run config: ${source}: "${numField}" must be a positive integer`, false);
+            }
+        }
+    }
+    return obj;
+}
+/**
+ * Load and resolve a bench run config from disk.
+ *
+ * @param configPath  Absolute or relative path to the config JSON file.
+ * @param overrides   CLI-derived overrides applied on top of the config.
+ */
+export function loadBenchRunConfig(configPath, overrides = {}) {
+    const absPath = path.isAbsolute(configPath) ? configPath : path.resolve(configPath);
+    if (!fs.existsSync(absPath)) {
+        throw new BenchConfigError(`bench run config: file not found: ${absPath}`, true);
+    }
+    let raw;
+    try {
+        raw = fs.readFileSync(absPath, "utf8");
+    }
+    catch (err) {
+        throw new BenchConfigError(`bench run config: cannot read ${absPath}: ${err instanceof Error ? err.message : String(err)}`, true);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (err) {
+        throw new BenchConfigError(`bench run config: ${absPath}: invalid JSON: ${err instanceof Error ? err.message : String(err)}`, false);
+    }
+    const config = validateConfig(parsed, absPath);
+    const configDir = path.dirname(absPath);
+    const providers = resolveProviders(config, configDir);
+    const envModel = process.env.BENCH_OPENCODE_MODEL;
+    const model = (envModel && envModel.length > 0 ? envModel : undefined) ?? config.defaultModel ?? providers.defaultModel;
+    if (!model) {
+        throw new BenchConfigError(`bench run config: ${absPath}: no model specified. Set "defaultModel" in the config, set "defaultModel" in the providers file, or set BENCH_OPENCODE_MODEL.`, true);
+    }
+    // Resolve tasks (with optional CLI list override).
+    let resolved = resolveTasks(config.tasks);
+    if (overrides.tasksList && overrides.tasksList.length > 0) {
+        const set = new Set(overrides.tasksList);
+        const filtered = resolved.tasks.filter((t) => set.has(t.id));
+        const missing = overrides.tasksList.filter((id) => !resolved.tasks.some((t) => t.id === id));
+        if (missing.length > 0) {
+            throw new BenchConfigError(`bench run config: --tasks override: no task in the config matched ${JSON.stringify(missing.join(", "))}`, true);
+        }
+        resolved = { tasks: filtered, slice: resolved.slice };
+    }
+    if (resolved.tasks.length === 0) {
+        throw new BenchConfigError(`bench run config: ${absPath}: task selector matched zero tasks`, true);
+    }
+    let baselineByTaskId;
+    if (config.baseline) {
+        const baselinePath = resolvePathString(config.baseline, configDir);
+        baselineByTaskId = loadBaseline(baselinePath);
+    }
+    const arms = config.arms ?? ["noakm", "akm"];
+    const seedsPerArm = overrides.seedsPerArm ?? config.seeds;
+    const parallel = overrides.parallel ?? config.parallel;
+    const name = config.name ?? path.basename(absPath, path.extname(absPath));
+    return {
+        source: absPath,
+        name,
+        providers,
+        model,
+        tasks: resolved.tasks,
+        arms,
+        ...(seedsPerArm !== undefined ? { seedsPerArm } : {}),
+        ...(config.budgetTokens !== undefined ? { budgetTokens: config.budgetTokens } : {}),
+        ...(config.budgetWallMs !== undefined ? { budgetWallMs: config.budgetWallMs } : {}),
+        ...(parallel !== undefined ? { parallel } : {}),
+        ...(config.forceParallel ? { forceParallel: true } : {}),
+        ...(baselineByTaskId ? { baselineByTaskId } : {}),
+        slice: resolved.slice,
+    };
+}

package/dist/tests/bench/run-config.test.js

@@ -0,0 +1,298 @@
+/**
+ * Unit tests for the bench run-config loader (`tests/bench/run-config.ts`).
+ *
+ * Covers the parts that don't require spawning a process:
+ * - Schema validation (unknown fields, missing schemaVersion, bad arms).
+ * - Path resolution (~ expansion, ${VAR} expansion, relative vs absolute).
+ * - Provider discovery chain (env > inline > providersRef > XDG default).
+ * - Baseline-file loading + range checks.
+ * - Task selector resolution (slice / domain / id / array).
+ *
+ * The CLI-level dispatch is exercised by `cli.test.ts` via spawned bench
+ * runs — keep those for end-to-end coverage; this file is unit-grade.
+ */
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { defaultUserProvidersPath, loadBaseline, loadBenchRunConfig, resolvePathString } from "./run-config";
+import { benchMkdtemp } from "./tmp";
+const REPO_ROOT = path.resolve(__dirname, "..", "..");
+let workDir;
+let savedEnv;
+beforeEach(() => {
+    // Per #276 invariant: bench tmp dirs live under `${AKM_CACHE_DIR}/bench/`,
+    // never the OS-default tmp root. `benchMkdtemp` is the drop-in.
+    workDir = benchMkdtemp("akm-bench-runconfig-test-");
+    savedEnv = {
+        BENCH_OPENCODE_CONFIG: process.env.BENCH_OPENCODE_CONFIG,
+        BENCH_OPENCODE_MODEL: process.env.BENCH_OPENCODE_MODEL,
+        AKM_TEST_VAR: process.env.AKM_TEST_VAR,
+    };
+    delete process.env.BENCH_OPENCODE_CONFIG;
+    delete process.env.BENCH_OPENCODE_MODEL;
+});
+afterEach(() => {
+    fs.rmSync(workDir, { recursive: true, force: true });
+    for (const [k, v] of Object.entries(savedEnv)) {
+        if (v === undefined)
+            delete process.env[k];
+        else
+            process.env[k] = v;
+    }
+});
+function writeProvidersFile(filePath, defaultModel = "p/m") {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, JSON.stringify({
+        schemaVersion: 1,
+        defaultModel,
+        providers: { p: { npm: "@ai-sdk/openai-compatible" } },
+    }));
+}
+describe("resolvePathString", () => {
+    test("resolves a relative path against the supplied base dir", () => {
+        expect(resolvePathString("foo.json", "/work")).toBe("/work/foo.json");
+    });
+    test("returns absolute paths unchanged", () => {
+        expect(resolvePathString("/abs/path.json", "/work")).toBe("/abs/path.json");
+    });
+    test("expands `~` to the operator's home dir", () => {
+        expect(resolvePathString("~/.config/akm/foo.json", "/work")).toBe(path.join(os.homedir(), ".config/akm/foo.json"));
+    });
+    test("expands env-var references", () => {
+        // Build the input with concatenation rather than a string literal to avoid
+        // biome's noTemplateCurlyInString flag on the `\${VAR}` form.
+        process.env.AKM_TEST_VAR = "/somewhere";
+        const input = `${"$"}{AKM_TEST_VAR}/providers.json`;
+        expect(resolvePathString(input, "/work")).toBe("/somewhere/providers.json");
+    });
+});
+describe("defaultUserProvidersPath", () => {
+    test("respects XDG_CONFIG_HOME when set", () => {
+        const saved = process.env.XDG_CONFIG_HOME;
+        process.env.XDG_CONFIG_HOME = "/xdg-test";
+        try {
+            expect(defaultUserProvidersPath()).toBe("/xdg-test/akm/bench-providers.json");
+        }
+        finally {
+            if (saved === undefined)
+                delete process.env.XDG_CONFIG_HOME;
+            else
+                process.env.XDG_CONFIG_HOME = saved;
+        }
+    });
+    test("falls back to ~/.config when XDG_CONFIG_HOME is unset", () => {
+        const saved = process.env.XDG_CONFIG_HOME;
+        delete process.env.XDG_CONFIG_HOME;
+        try {
+            expect(defaultUserProvidersPath()).toBe(path.join(os.homedir(), ".config/akm/bench-providers.json"));
+        }
+        finally {
+            if (saved !== undefined)
+                process.env.XDG_CONFIG_HOME = saved;
+        }
+    });
+});
+describe("loadBaseline", () => {
+    test("loads a `{ taskId: passRate }` map", () => {
+        const filePath = path.join(workDir, "baseline.json");
+        fs.writeFileSync(filePath, JSON.stringify({ "domain/a": 0.8, "domain/b": 1.0 }));
+        expect(loadBaseline(filePath)).toEqual({ "domain/a": 0.8, "domain/b": 1.0 });
+    });
+    test("rejects pass rates outside [0, 1]", () => {
+        const filePath = path.join(workDir, "bad.json");
+        fs.writeFileSync(filePath, JSON.stringify({ "x/y": 1.5 }));
+        expect(() => loadBaseline(filePath)).toThrow(/must be a number in \[0, 1\]/);
+    });
+    test("rejects non-number values", () => {
+        const filePath = path.join(workDir, "non-number.json");
+        fs.writeFileSync(filePath, JSON.stringify({ "x/y": "not a number" }));
+        expect(() => loadBaseline(filePath)).toThrow(/must be a number/);
+    });
+});
+describe("loadBenchRunConfig — schema validation", () => {
+    test("rejects unknown top-level fields", () => {
+        const cfgPath = path.join(workDir, "bad.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({ schemaVersion: 1, name: "x", weirdField: 42 }));
+        expect(() => loadBenchRunConfig(cfgPath)).toThrow(/unknown field "weirdField"/);
+    });
+    test("rejects missing schemaVersion", () => {
+        const cfgPath = path.join(workDir, "noversion.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({ name: "x" }));
+        expect(() => loadBenchRunConfig(cfgPath)).toThrow(/unsupported schemaVersion/);
+    });
+    test("rejects providers AND providersRef both set", () => {
+        const cfgPath = path.join(workDir, "both.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providers: { p: { npm: "x" } },
+            providersRef: "./other.json",
+        }));
+        expect(() => loadBenchRunConfig(cfgPath)).toThrow(/only one of "providers" or "providersRef"/);
+    });
+    test("rejects bad arm values", () => {
+        const cfgPath = path.join(workDir, "badarm.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({ schemaVersion: 1, arms: ["nope"] }));
+        expect(() => loadBenchRunConfig(cfgPath)).toThrow(/invalid arm/);
+    });
+    test("missing config file exits with usage error", () => {
+        expect(() => loadBenchRunConfig(path.join(workDir, "ghost.json"))).toThrow(/file not found/);
+    });
+});
+describe("loadBenchRunConfig — provider discovery", () => {
+    test("BENCH_OPENCODE_CONFIG env var wins over providersRef", () => {
+        const envProviders = path.join(workDir, "env-providers.json");
+        const refProviders = path.join(workDir, "ref-providers.json");
+        writeProvidersFile(envProviders, "env/model");
+        writeProvidersFile(refProviders, "ref/model");
+        process.env.BENCH_OPENCODE_CONFIG = envProviders;
+        const cfgPath = path.join(workDir, "config.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./ref-providers.json",
+            tasks: "all",
+        }));
+        // No tasks resolved so we can't actually load — just verify provider
+        // resolution. We restrict to a single committed task to satisfy the
+        // selector. The bench corpus exists at fixtures/bench/tasks; we use
+        // "all" as the selector and skip past the `tasks=0` exit by writing a
+        // selector that matches a real task.
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./ref-providers.json",
+            tasks: ["drillbit/backup-policy"],
+        }));
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.providers.source).toBe(envProviders);
+        expect(resolved.model).toBe("env/model");
+    });
+    test("`providersRef` is resolved relative to the config file", () => {
+        const refProviders = path.join(workDir, "subdir", "providers.json");
+        writeProvidersFile(refProviders, "ref/model");
+        const cfgPath = path.join(workDir, "config.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./subdir/providers.json",
+            tasks: ["drillbit/backup-policy"],
+        }));
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.providers.source).toBe(refProviders);
+        expect(resolved.model).toBe("ref/model");
+    });
+    test("config `defaultModel` overrides the providers file's defaultModel", () => {
+        const refProviders = path.join(workDir, "providers.json");
+        writeProvidersFile(refProviders, "ref/model");
+        const cfgPath = path.join(workDir, "config.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./providers.json",
+            defaultModel: "config/model",
+            tasks: ["drillbit/backup-policy"],
+        }));
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.model).toBe("config/model");
+    });
+    test("BENCH_OPENCODE_MODEL env wins over both", () => {
+        const refProviders = path.join(workDir, "providers.json");
+        writeProvidersFile(refProviders, "ref/model");
+        process.env.BENCH_OPENCODE_MODEL = "env/model";
+        const cfgPath = path.join(workDir, "config.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./providers.json",
+            defaultModel: "config/model",
+            tasks: ["drillbit/backup-policy"],
+        }));
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.model).toBe("env/model");
+    });
+});
+describe("loadBenchRunConfig — task resolution", () => {
+    test("tasks=array selects exactly the listed ids", () => {
+        const refProviders = path.join(workDir, "providers.json");
+        writeProvidersFile(refProviders);
+        const cfgPath = path.join(workDir, "config.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./providers.json",
+            tasks: ["drillbit/backup-policy", "drillbit/canary-enable"],
+        }));
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.tasks.map((t) => t.id).sort()).toEqual(["drillbit/backup-policy", "drillbit/canary-enable"]);
+    });
+    test("tasks=domain matches every task whose domain matches", () => {
+        const refProviders = path.join(workDir, "providers.json");
+        writeProvidersFile(refProviders);
+        const cfgPath = path.join(workDir, "config.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./providers.json",
+            tasks: "drillbit",
+        }));
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.tasks.length).toBeGreaterThan(0);
+        for (const t of resolved.tasks)
+            expect(t.domain).toBe("drillbit");
+    });
+    test("tasks=single-id matches exactly that task", () => {
+        const refProviders = path.join(workDir, "providers.json");
+        writeProvidersFile(refProviders);
+        const cfgPath = path.join(workDir, "config.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./providers.json",
+            tasks: "drillbit/backup-policy",
+        }));
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.tasks.map((t) => t.id)).toEqual(["drillbit/backup-policy"]);
+    });
+    test("--tasks override (CLI) restricts to a subset of the config's selection", () => {
+        const refProviders = path.join(workDir, "providers.json");
+        writeProvidersFile(refProviders);
+        const cfgPath = path.join(workDir, "config.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./providers.json",
+            tasks: ["drillbit/backup-policy", "drillbit/canary-enable"],
+        }));
+        const resolved = loadBenchRunConfig(cfgPath, { tasksList: ["drillbit/canary-enable"] });
+        expect(resolved.tasks.map((t) => t.id)).toEqual(["drillbit/canary-enable"]);
+    });
+    test("baseline path is resolved relative to the config file", () => {
+        const refProviders = path.join(workDir, "providers.json");
+        writeProvidersFile(refProviders);
+        const baselinePath = path.join(workDir, "baseline.json");
+        fs.writeFileSync(baselinePath, JSON.stringify({ "drillbit/backup-policy": 0.8 }));
+        const cfgPath = path.join(workDir, "config.json");
+        fs.writeFileSync(cfgPath, JSON.stringify({
+            schemaVersion: 1,
+            providersRef: "./providers.json",
+            tasks: ["drillbit/backup-policy"],
+            baseline: "./baseline.json",
+        }));
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.baselineByTaskId).toEqual({ "drillbit/backup-policy": 0.8 });
+    });
+});
+describe("loadBenchRunConfig — committed configs validate", () => {
+    test("tests/bench/configs/nano-quick.json loads cleanly", () => {
+        const cfgPath = path.join(REPO_ROOT, "tests", "bench", "configs", "nano-quick.json");
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.name).toBe("nano-quick");
+        expect(resolved.arms).toEqual(["akm"]);
+        expect(resolved.seedsPerArm).toBe(2);
+        expect(resolved.tasks.length).toBe(5);
+    });
+    test("tests/bench/configs/full.json loads cleanly and carries the baseline", () => {
+        const cfgPath = path.join(REPO_ROOT, "tests", "bench", "configs", "full.json");
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.name).toBe("full");
+        expect(resolved.baselineByTaskId).toBeDefined();
+        expect(typeof resolved.baselineByTaskId?.["drillbit/backup-policy"]).toBe("number");
+    });
+    test("tests/bench/configs/curate-test.json restricts to one task", () => {
+        const cfgPath = path.join(REPO_ROOT, "tests", "bench", "configs", "curate-test.json");
+        const resolved = loadBenchRunConfig(cfgPath);
+        expect(resolved.tasks.map((t) => t.id)).toEqual(["inkwell/configure-scaling"]);
+    });
+});