openmerit 0.4.1 → 0.4.2

package/dist/setup.d.ts

@@ -0,0 +1,44 @@
+import type { Prompter } from "./cli/prompt.js";
+import { type Endpoint, type HarnessKind, type HarnessSettings } from "./state/config.js";
+import type { State } from "./state/db.js";
+import type { Repo } from "./state/paths.js";
+export interface SetupFlags {
+    harness?: HarnessKind;
+    harnessModel?: string;
+    apiProvider?: string;
+    apiModel?: string;
+    apiKeyEnv?: string;
+    /** Sign in to ChatGPT with a device code (for --harness codex on a server). */
+    deviceCode?: boolean;
+    /** `openrouter:ENV_VAR`, `openai:ENV_VAR` or `<id>=<baseUrl>:ENV_VAR` (OpenAI-compatible). */
+    endpoints?: string[];
+    allowedModels?: string[];
+    budget?: number;
+    registerMcp?: boolean;
+    langfuseUrl?: string;
+    langfusePublicKeyEnv?: string;
+    langfuseSecretKeyEnv?: string;
+    /** Accept defaults and never prompt. */
+    nonInteractive?: boolean;
+}
+interface SetupContext {
+    repo: Repo;
+    state: State;
+    prompter: Prompter | null;
+    log: (line: string) => void;
+    env: Record<string, string | undefined>;
+    fetch?: typeof fetch;
+    flags: SetupFlags;
+    mcpCommand: string[];
+}
+export declare function verifyEndpoint(endpoint: Endpoint, env: Record<string, string | undefined>, fetcher?: typeof fetch): Promise<{
+    ok: boolean;
+    detail: string;
+}>;
+export declare function runSetup(ctx: SetupContext): Promise<{
+    harness: HarnessSettings | null;
+    endpoints: Endpoint[];
+    langfuse: boolean;
+    budgetUsd: number;
+}>;
+export {};

package/dist/setup.js

@@ -0,0 +1,211 @@
+// `openmerit setup`: detect → explicit choice → verify → persist, rerunnable. Global: how OpenMerit
+// Harness is powered, and the MCP registration with Claude Code/Codex. Per repository: read-only
+// Langfuse, candidate endpoints with allowed models, and the default eval budget. Re-running keeps
+// working settings unless the user replaces them, and a failed connection is offered for retry or
+// reconfiguration; nothing switches provider on its own.
+import { spawnSync } from "node:child_process";
+import { confirm, connect, credential, HARNESS_LABEL } from "./connect.js";
+import { createLangfuseConnector } from "./connectors/langfuse/index.js";
+import { DEFAULT_BUDGET_USD, describeCredential, OPENAI_ENDPOINT, OPENROUTER_ENDPOINT, readRepoConfig, resolveCredential, updateRepoConfig, } from "./state/config.js";
+export async function verifyEndpoint(endpoint, env, fetcher = fetch) {
+    const key = resolveCredential(endpoint.apiKey, env);
+    if (!key)
+        return { ok: false, detail: `no key (${describeCredential(endpoint.apiKey)} is empty)` };
+    const url = endpoint.kind === "openrouter" ? `${endpoint.baseUrl.replace(/\/$/, "")}/key` : `${endpoint.baseUrl.replace(/\/$/, "")}/models`;
+    try {
+        const response = await fetcher(url, { headers: { authorization: `Bearer ${key}` }, signal: AbortSignal.timeout(20_000), redirect: "manual" });
+        await response.body?.cancel();
+        return response.ok ? { ok: true, detail: "key accepted" } : { ok: false, detail: `HTTP ${response.status}${response.status === 401 ? " (key rejected or expired)" : ""}` };
+    }
+    catch (error) {
+        return { ok: false, detail: error instanceof Error ? error.message : String(error) };
+    }
+}
+/** Claude Code and Codex CLIs on PATH, which can register OpenMerit's MCP server. */
+function mcpClients(env) {
+    return ["claude-code", "codex"].filter(kind => !spawnSync(kind === "claude-code" ? env.OPENMERIT_CLAUDE_BIN ?? "claude" : env.OPENMERIT_CODEX_BIN ?? "codex", ["--version"], { env, stdio: "ignore" }).error);
+}
+/** Adds OpenMerit's stdio MCP server to Claude Code and Codex through their own `mcp add` commands; existing entries are left alone. */
+function registerMcp(ctx, kinds) {
+    for (const kind of kinds) {
+        const bin = kind === "claude-code" ? ctx.env.OPENMERIT_CLAUDE_BIN ?? "claude" : ctx.env.OPENMERIT_CODEX_BIN ?? "codex";
+        const get = spawnSync(bin, ["mcp", "get", "openmerit"], { encoding: "utf8", env: ctx.env });
+        if (get.error)
+            continue;
+        if (get.status === 0) {
+            ctx.log(`  ${kind === "claude-code" ? "Claude Code" : "Codex"} already has an "openmerit" MCP server; left unchanged.`);
+            continue;
+        }
+        const args = kind === "claude-code" ? ["mcp", "add", "--scope", "user", "openmerit", "--", ...ctx.mcpCommand] : ["mcp", "add", "openmerit", "--", ...ctx.mcpCommand];
+        const add = spawnSync(bin, args, { encoding: "utf8", env: ctx.env });
+        ctx.log(add.status === 0 ? `  ✓ Registered OpenMerit's MCP server with ${kind === "claude-code" ? "Claude Code" : "Codex"}.` : `  ✗ Could not register with ${kind}: ${(add.stderr || add.stdout).trim().slice(0, 200)}`);
+    }
+}
+async function setupLangfuse(ctx) {
+    const existing = readRepoConfig(ctx.repo.root).langfuse;
+    const connector = (settings) => {
+        const publicKey = resolveCredential(settings.publicKey, ctx.env), secretKey = resolveCredential(settings.secretKey, ctx.env);
+        if (!publicKey || !secretKey)
+            throw new Error("the Viewer key pair is not available");
+        return createLangfuseConnector({ baseUrl: settings.baseUrl, publicKey, secretKey }, { ...(ctx.fetch ? { fetch: ctx.fetch } : {}), retries: 1 });
+    };
+    const verify = async (settings) => {
+        try {
+            const lf = connector(settings);
+            const project = await lf.check();
+            await lf.verifyViewer();
+            return { ok: true, detail: `project ${project.projectName}, read-only key`, lf };
+        }
+        catch (error) {
+            return { ok: false, detail: error instanceof Error ? error.message : String(error) };
+        }
+    };
+    if (existing && !ctx.flags.langfuseUrl) {
+        const check = await verify(existing);
+        ctx.log(check.ok ? `  ✓ Langfuse at ${existing.baseUrl}: ${check.detail}` : `  ✗ Saved Langfuse connection failed: ${check.detail}`);
+        if (check.ok && await confirm(ctx, "Keep this Langfuse connection?", true))
+            return { ...existing, verifiedAt: new Date().toISOString() };
+        if (!check.ok && !(await confirm(ctx, "Reconfigure Langfuse now?", false)))
+            return existing;
+    }
+    else if (!ctx.flags.langfuseUrl && !(await confirm(ctx, "Connect this repository's Langfuse (read-only aggregates)?", false)))
+        return existing;
+    for (;;) {
+        const baseUrl = ctx.flags.langfuseUrl ?? (ctx.prompter ? await ctx.prompter.ask("Langfuse URL (Cloud or self-hosted)", existing?.baseUrl ?? "https://cloud.langfuse.com") : "https://cloud.langfuse.com");
+        ctx.log("  OpenMerit needs a separate Viewer key pair (Project Settings → API Keys → Viewer); it never writes to Langfuse.");
+        const publicKey = await credential(ctx, "Langfuse Viewer public key", ["OPENMERIT_LANGFUSE_PUBLIC_KEY"], `langfuse-public-${ctx.repo.id}`, ctx.flags.langfusePublicKeyEnv);
+        const secretKey = await credential(ctx, "Langfuse Viewer secret key", ["OPENMERIT_LANGFUSE_SECRET_KEY"], `langfuse-secret-${ctx.repo.id}`, ctx.flags.langfuseSecretKeyEnv);
+        if (!publicKey || !secretKey)
+            return existing;
+        const check = await verify({ baseUrl, publicKey, secretKey });
+        if (check.ok) {
+            ctx.log(`  ✓ Langfuse: ${check.detail}`);
+            const mappings = existing?.mappings ?? [];
+            if (ctx.prompter && !ctx.flags.nonInteractive) {
+                const traces = await check.lf.traceNames().catch(() => []);
+                const latest = ctx.state.listRuns(ctx.repo.id, 1)[0];
+                const taskNames = (latest?.state.tasks ?? []).map(task => task.name);
+                if (traces.length)
+                    ctx.log(`  Trace names with generations in the last 30 days: ${traces.slice(0, 15).map(trace => `${trace.traceName} (${trace.generations})`).join(", ")}`);
+                if (taskNames.length)
+                    ctx.log(`  Tasks from the last Scout run: ${taskNames.join(", ")}`);
+                for (;;) {
+                    const line = await ctx.prompter.ask("Map a task to Langfuse evidence as <task>=<traceName>:<score,score> (empty to finish)", "");
+                    if (!line)
+                        break;
+                    const match = /^(.+?)=([^:]+)(?::(.*))?$/.exec(line);
+                    if (!match) {
+                        ctx.log("  Write <task>=<traceName>:<score names>.");
+                        continue;
+                    }
+                    const mapping = { task: match[1].trim(), traceName: match[2].trim(), scores: (match[3] ?? "").split(",").map(item => item.trim()).filter(Boolean) };
+                    mappings.splice(0, mappings.length, ...mappings.filter(item => item.task !== mapping.task), mapping);
+                }
+            }
+            return { baseUrl, publicKey, secretKey, mappings, verifiedAt: new Date().toISOString() };
+        }
+        ctx.log(`  ✗ Langfuse: ${check.detail}`);
+        if (!ctx.prompter || ctx.flags.nonInteractive || !(await ctx.prompter.confirm("Try again?", true)))
+            return existing;
+    }
+}
+function parseEndpointFlag(value, allowed) {
+    const known = /^(openrouter|openai):([A-Z_][A-Z0-9_]*)$/.exec(value);
+    if (known)
+        return { ...(known[1] === "openrouter" ? OPENROUTER_ENDPOINT : OPENAI_ENDPOINT), allowedModels: allowed, apiKey: { env: known[2] } };
+    const custom = /^([a-z0-9-]+)=(https?:\/\/[^\s]+):([A-Z_][A-Z0-9_]*)$/.exec(value);
+    if (!custom)
+        throw new Error(`--endpoint ${value}: use openrouter:ENV_VAR, openai:ENV_VAR or <id>=<baseUrl>:ENV_VAR`);
+    if (allowed === "all")
+        throw new Error("An OpenAI-compatible endpoint needs --allowed-models: the catalogue cannot tell which models it serves.");
+    return { id: custom[1], kind: "openai-compatible", baseUrl: custom[2], apiKey: { env: custom[3] }, allowedModels: allowed };
+}
+async function setupEndpoints(ctx) {
+    const existing = readRepoConfig(ctx.repo.root).endpoints ?? [];
+    const allowed = ctx.flags.allowedModels?.length ? ctx.flags.allowedModels : "all";
+    if (ctx.flags.endpoints?.length) {
+        const next = [...existing];
+        for (const flag of ctx.flags.endpoints) {
+            const endpoint = parseEndpointFlag(flag, allowed);
+            const check = await verifyEndpoint(endpoint, ctx.env, ctx.fetch);
+            if (!check.ok)
+                throw new Error(`Endpoint ${endpoint.id} failed verification: ${check.detail}. Existing endpoints are unchanged.`);
+            ctx.log(`  ✓ ${endpoint.id}: ${check.detail}; allowed models: ${endpoint.allowedModels === "all" ? "all" : endpoint.allowedModels.join(", ")}`);
+            next.splice(0, next.length, ...next.filter(item => item.id !== endpoint.id), { ...endpoint, verifiedAt: new Date().toISOString() });
+        }
+        return next;
+    }
+    const kept = [];
+    for (const endpoint of existing) {
+        const check = await verifyEndpoint(endpoint, ctx.env, ctx.fetch);
+        ctx.log(`  ${check.ok ? "✓" : "✗"} Endpoint ${endpoint.id} (${endpoint.baseUrl}): ${check.detail}; allowed models: ${endpoint.allowedModels === "all" ? "all" : endpoint.allowedModels.join(", ")}`);
+        if (check.ok) {
+            kept.push(await confirm(ctx, `Keep ${endpoint.id}?`, true) ? endpoint : await reconfigure(endpoint) ?? endpoint);
+            continue;
+        }
+        const fixed = await reconfigure(endpoint);
+        kept.push(fixed ?? endpoint);
+    }
+    async function reconfigure(endpoint) {
+        if (!ctx.prompter || ctx.flags.nonInteractive || !(await ctx.prompter.confirm(`Enter a new key for ${endpoint.id}?`, true)))
+            return null;
+        for (;;) {
+            const apiKey = await credential(ctx, `${endpoint.id} API key`, [], `endpoint-${endpoint.id}-${ctx.repo.id}`);
+            if (!apiKey)
+                return null;
+            const next = { ...endpoint, apiKey };
+            const check = await verifyEndpoint(next, ctx.env, ctx.fetch);
+            ctx.log(`  ${check.ok ? "✓" : "✗"} ${endpoint.id}: ${check.detail}`);
+            if (check.ok)
+                return { ...next, verifiedAt: new Date().toISOString() };
+            if (!(await ctx.prompter.confirm("Try again?", true)))
+                return null;
+        }
+    }
+    if (!ctx.prompter || ctx.flags.nonInteractive)
+        return kept;
+    if (!kept.some(item => item.kind === "openrouter") && await ctx.prompter.confirm("Add OpenRouter for candidates (all its models eligible)?", !kept.length)) {
+        const apiKey = await credential(ctx, "OpenRouter key", ["OPENROUTER_API_KEY"], `endpoint-openrouter-${ctx.repo.id}`);
+        if (apiKey)
+            kept.push(...await added({ ...OPENROUTER_ENDPOINT, apiKey }, "all"));
+    }
+    if (!kept.some(item => item.kind === "openai") && await ctx.prompter.confirm("Add OpenAI for candidates?", false)) {
+        const apiKey = await credential(ctx, "OpenAI key", ["OPENAI_API_KEY"], `endpoint-openai-${ctx.repo.id}`);
+        if (apiKey)
+            kept.push(...await added({ ...OPENAI_ENDPOINT, apiKey }));
+    }
+    // OpenRouter's question already says every model is eligible; other endpoints ask which.
+    async function added(endpoint, allowed) {
+        const models = allowed ?? await ctx.prompter.ask(`Allowed models on ${endpoint.id}: "all", or IDs comma-separated`, "all");
+        const next = { ...endpoint, allowedModels: /^all$/i.test(models) ? "all" : models.split(",").map(item => item.trim()).filter(Boolean) };
+        const check = await verifyEndpoint(next, ctx.env, ctx.fetch);
+        ctx.log(`  ${check.ok ? "✓" : "✗"} ${endpoint.id}: ${check.detail}`);
+        return check.ok ? [{ ...next, verifiedAt: new Date().toISOString() }] : (await ctx.prompter.confirm("Save it anyway, to fix later?", false)) ? [next] : [];
+    }
+    return kept;
+}
+export async function runSetup(ctx) {
+    ctx.log("OpenMerit setup\n\n1. OpenMerit Harness (global)");
+    const { harness: kind, harnessModel, apiProvider, apiModel, apiKeyEnv, deviceCode, nonInteractive } = ctx.flags;
+    const harness = await connect({ ...ctx, root: ctx.repo.root, flags: { ...(harnessModel ? { model: harnessModel } : {}), ...(apiProvider ? { apiProvider } : {}), ...(apiModel ? { apiModel } : {}), ...(apiKeyEnv ? { apiKeyEnv } : {}), ...(deviceCode ? { deviceCode } : {}), ...(nonInteractive ? { nonInteractive } : {}) } }, kind);
+    if (!harness)
+        return { harness: null, endpoints: [], langfuse: false, budgetUsd: DEFAULT_BUDGET_USD };
+    const agents = mcpClients(ctx.env);
+    if (agents.length && (ctx.flags.registerMcp ?? await confirm(ctx, `Register OpenMerit's MCP server with ${agents.map(kind => (kind === "claude-code" ? "Claude Code" : "Codex")).join(" and ")}?`, true)))
+        registerMcp(ctx, agents);
+    ctx.log(`\n2. This repository: ${ctx.repo.root}`);
+    const langfuse = await setupLangfuse(ctx);
+    const endpoints = await setupEndpoints(ctx);
+    if (!endpoints.length)
+        ctx.log("  No candidate endpoints: Scout needs at least one to evaluate models. Run setup again to add one.");
+    const current = readRepoConfig(ctx.repo.root).budget?.defaultUsd ?? DEFAULT_BUDGET_USD;
+    let budgetUsd = ctx.flags.budget ?? current;
+    if (ctx.flags.budget === undefined && ctx.prompter && !ctx.flags.nonInteractive) {
+        const answer = Number(await ctx.prompter.ask("Default eval budget per Scout run, USD", String(current)));
+        if (Number.isFinite(answer) && answer > 0)
+            budgetUsd = answer;
+    }
+    updateRepoConfig(ctx.repo.root, { endpoints, budget: { defaultUsd: budgetUsd }, ...(langfuse ? { langfuse } : {}) });
+    ctx.log(`\nSaved. Harness: ${HARNESS_LABEL[harness.kind]}. Endpoints: ${endpoints.map(item => item.id).join(", ") || "none"}. Langfuse: ${langfuse ? "read-only" : "not connected"}. Eval budget: $${budgetUsd.toFixed(2)} per run.\nSettings: ~/.openmerit/config.json (global) and .openmerit/config.json (this repository; credential references only). Next: openmerit scout`);
+    return { harness, endpoints, langfuse: !!langfuse, budgetUsd };
+}

package/dist/state/config.d.ts

@@ -0,0 +1,116 @@
+/** Where a credential comes from: an environment variable, or a value OpenMerit stored in secrets.json. */
+export type CredentialRef = {
+    env: string;
+} | {
+    secret: string;
+};
+export type HarnessKind = "claude-code" | "codex" | "api";
+export interface HarnessSettings {
+    kind: HarnessKind;
+    /** Model the coding agent uses for harness inference; its own default when absent. */
+    model?: string;
+    /** For kind "api": a Pi provider (anthropic, openai, openrouter, ...) and model. */
+    api?: {
+        provider: string;
+        model: string;
+        apiKey: CredentialRef;
+    };
+    verifiedAt?: string;
+}
+/** How history names a harness, such as `api:openrouter/anthropic/claude-sonnet-5.5` or `codex`. */
+export declare const harnessLabel: (settings: HarnessSettings) => string;
+export interface GlobalConfig {
+    harness?: HarnessSettings;
+    [key: string]: unknown;
+}
+export type EndpointKind = "openrouter" | "openai" | "openai-compatible";
+export interface Endpoint {
+    id: string;
+    kind: EndpointKind;
+    /** OpenAI-compatible API root, such as https://openrouter.ai/api/v1. */
+    baseUrl: string;
+    apiKey: CredentialRef;
+    /** "all" for every catalogue model the endpoint serves, or explicit model IDs (catalogue or endpoint IDs). */
+    allowedModels: "all" | string[];
+    verifiedAt?: string;
+}
+/** Which Langfuse scores and traces measure a task. Read-only aggregates. */
+export interface LangfuseMapping {
+    task: string;
+    traceName: string;
+    scores: string[];
+}
+export interface LangfuseSettings {
+    baseUrl: string;
+    publicKey: CredentialRef;
+    secretKey: CredentialRef;
+    mappings: LangfuseMapping[];
+    verifiedAt?: string;
+}
+/** How OpenMerit runs a task's existing eval for one model. See README "Eval contract". */
+export interface EvalSpec {
+    command: string;
+    cwd?: string;
+    /** Extra variables for the eval; `{{model}}`, `{{items}}` and `{{experiment}}` are filled in. */
+    env?: Record<string, string>;
+    /** Helper files the harness wrote (and the user approved) to run the existing eval. */
+    helpers?: string[];
+    /** The eval's own acceptance threshold: the minimum pass rate, 0 to 1. */
+    threshold?: number;
+    /**
+     * The eval records each run as a Langfuse experiment named $OPENMERIT_EXPERIMENT on this dataset,
+     * with a per-item score. OpenMerit reads pass/fail back with the Viewer key. Without it, the eval
+     * reports its items in $OPENMERIT_RESULT.
+     */
+    langfuse?: {
+        dataset: string;
+        score: string;
+        min?: number;
+    };
+    description?: string;
+}
+export interface RepoConfig {
+    langfuse?: LangfuseSettings;
+    endpoints?: Endpoint[];
+    budget?: {
+        defaultUsd: number;
+    };
+    evals?: Record<string, EvalSpec>;
+    /** Glob patterns, relative to the repository root, of files or directories Scout's scan skips. */
+    scan?: {
+        exclude?: string[];
+    };
+    /** Runs of every item for the current model and each finalist; default 3. */
+    repeats?: number;
+    [key: string]: unknown;
+}
+export declare const DEFAULT_BUDGET_USD = 10;
+export declare const DEFAULT_REPEATS = 3;
+export declare const globalConfigPath: (env?: Record<string, string | undefined>) => string;
+export declare const repoConfigPath: (root: string) => string;
+export declare const readGlobalConfig: (env?: Record<string, string | undefined>) => GlobalConfig;
+export declare const readRepoConfig: (root: string) => RepoConfig;
+/** Merges `patch` into the stored object, keeping every other key. */
+export declare function updateGlobalConfig(patch: Partial<GlobalConfig>, env?: Record<string, string | undefined>): {
+    [x: string]: unknown;
+    harness?: HarnessSettings;
+};
+export declare function updateRepoConfig(root: string, patch: Partial<RepoConfig>): {
+    [x: string]: unknown;
+    langfuse?: LangfuseSettings;
+    endpoints?: Endpoint[];
+    budget?: {
+        defaultUsd: number;
+    };
+    evals?: Record<string, EvalSpec>;
+    scan?: {
+        exclude?: string[];
+    };
+    repeats?: number;
+};
+export declare function storeSecret(name: string, value: string, env?: Record<string, string | undefined>): CredentialRef;
+export declare function resolveCredential(ref: CredentialRef | undefined, env?: Record<string, string | undefined>): string | null;
+export declare const describeCredential: (ref: CredentialRef) => string;
+/** Default candidate pool: OpenRouter with every model eligible. */
+export declare const OPENROUTER_ENDPOINT: Omit<Endpoint, "apiKey">;
+export declare const OPENAI_ENDPOINT: Omit<Endpoint, "apiKey">;

package/dist/state/config.js

@@ -0,0 +1,60 @@
+// Settings. Global (~/.openmerit/config.json): how OpenMerit Harness is powered. Per repository
+// (<repo>/.openmerit/config.json): Langfuse, candidate endpoints, allowed models, evals and the eval
+// budget, with credential references only. Secret values live in ~/.openmerit/secrets.json (0600),
+// outside every repository. Writes keep keys OpenMerit does not know about.
+import { randomUUID } from "node:crypto";
+import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { openmeritHome } from "./paths.js";
+/** How history names a harness, such as `api:openrouter/anthropic/claude-sonnet-5.5` or `codex`. */
+export const harnessLabel = (settings) => `${settings.kind}${settings.kind === "api" ? `:${settings.api?.provider}/${settings.api?.model}` : settings.model ? `:${settings.model}` : ""}`;
+export const DEFAULT_BUDGET_USD = 10;
+export const DEFAULT_REPEATS = 3;
+function readJson(path) {
+    if (!existsSync(path))
+        return null;
+    const value = JSON.parse(readFileSync(path, "utf8"));
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new Error(`${path} must hold a JSON object`);
+    return value;
+}
+function writeJson(path, value, mode = 0o644) {
+    mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+    const temp = `${path}.${randomUUID()}.tmp`;
+    writeFileSync(temp, `${JSON.stringify(value, null, 2)}\n`, { mode });
+    renameSync(temp, path);
+    chmodSync(path, mode);
+}
+export const globalConfigPath = (env) => join(openmeritHome(env), "config.json");
+export const repoConfigPath = (root) => join(root, ".openmerit", "config.json");
+const secretsPath = (env) => join(openmeritHome(env), "secrets.json");
+export const readGlobalConfig = (env) => readJson(globalConfigPath(env)) ?? {};
+export const readRepoConfig = (root) => readJson(repoConfigPath(root)) ?? {};
+/** Merges `patch` into the stored object, keeping every other key. */
+export function updateGlobalConfig(patch, env) {
+    const next = { ...readGlobalConfig(env), ...patch };
+    writeJson(globalConfigPath(env), next);
+    return next;
+}
+export function updateRepoConfig(root, patch) {
+    const next = { ...readRepoConfig(root), ...patch };
+    writeJson(repoConfigPath(root), next);
+    return next;
+}
+export function storeSecret(name, value, env) {
+    const secrets = readJson(secretsPath(env)) ?? {};
+    secrets[name] = value;
+    writeJson(secretsPath(env), secrets, 0o600);
+    return { secret: name };
+}
+export function resolveCredential(ref, env = process.env) {
+    if (!ref)
+        return null;
+    if ("env" in ref)
+        return env[ref.env]?.trim() || null;
+    return readJson(secretsPath(env))?.[ref.secret] ?? null;
+}
+export const describeCredential = (ref) => ("env" in ref ? `$${ref.env}` : "stored in ~/.openmerit/secrets.json");
+/** Default candidate pool: OpenRouter with every model eligible. */
+export const OPENROUTER_ENDPOINT = { id: "openrouter", kind: "openrouter", baseUrl: "https://openrouter.ai/api/v1", allowedModels: "all" };
+export const OPENAI_ENDPOINT = { id: "openai", kind: "openai", baseUrl: "https://api.openai.com/v1", allowedModels: "all" };

package/dist/state/db.d.ts

@@ -0,0 +1,159 @@
+import type { DatabaseSync as Database } from "node:sqlite";
+import { type Repo } from "./paths.js";
+export type RunStatus = "running" | "needs_input" | "budget_paused" | "interrupted" | "failed" | "done" | "cancelled";
+export type UnitStatus = "pending" | "running" | "done" | "failed" | "unavailable" | "budget_paused" | "interrupted";
+export interface RunRow {
+    id: string;
+    repoId: string;
+    repoRoot: string;
+    revision: string | null;
+    status: RunStatus;
+    phase: string;
+    createdAt: string;
+    updatedAt: string;
+    packageVersion: string;
+    catalogueVersion: string;
+    harness: string;
+    budgetUsd: number;
+    state: Record<string, unknown>;
+    error: string | null;
+    lockPid: number | null;
+}
+export interface UnitRow {
+    id: string;
+    runId: string;
+    task: string;
+    model: string;
+    endpoint: string;
+    /** baseline: the current model on every item; screen: a candidate on the sampled items; final: a candidate on every item. */
+    role: "baseline" | "screen" | "final";
+    repeat: number;
+    status: UnitStatus;
+    attempts: number;
+    result: Record<string, unknown> | null;
+    error: string | null;
+}
+export interface Spend {
+    /** Settled requests: provider-reported cost, or list-price estimates where the provider reports none. */
+    settledUsd: number;
+    /** Part of settledUsd that is a list-price estimate. */
+    estimatedUsd: number;
+    /** Reserved for requests still in flight (or cut off by an interruption): cost unknown, counted at the reservation. */
+    inFlightUsd: number;
+    inFlight: number;
+    /** Requests with no price at all, counted at their reservation. */
+    unknown: number;
+    requests: number;
+    totalUsd: number;
+}
+export declare class State {
+    readonly db: Database;
+    readonly path: string;
+    constructor(env?: Record<string, string | undefined>);
+    /** Applies schema upgrades in order; older history stays readable. */
+    private migrate;
+    get schemaVersion(): number;
+    /** BEGIN IMMEDIATE: one writer at a time across CLI and MCP processes. */
+    transaction<T>(work: () => T): T;
+    close(): void;
+    createRun(input: {
+        id: string;
+        repo: Repo;
+        packageVersion: string;
+        catalogueVersion: string;
+        harness: string;
+        budgetUsd: number;
+        state: Record<string, unknown>;
+    }): RunRow;
+    getRun(id: string): RunRow | null;
+    listRuns(repoId: string, limit?: number): RunRow[];
+    private toRun;
+    updateRun(id: string, patch: {
+        status?: RunStatus;
+        phase?: string;
+        state?: Record<string, unknown>;
+        error?: string | null;
+        budgetUsd?: number;
+        harness?: string;
+    }): void;
+    /** Takes the run for this process. Fails while another live process works on it. */
+    lock(id: string): boolean;
+    unlock(id: string): void;
+    /**
+     * Runs whose process died mid-work become `interrupted`: running units are interrupted, and
+     * reservations of requests that never settled stay counted as in-flight spend of unknown cost.
+     */
+    reapInterrupted(repoId: string): string[];
+    /** Marks a run interrupted (Ctrl-C, or a dead process). Unsettled reservations stay counted as in-flight spend. */
+    markInterrupted(runId: string, message?: string): void;
+    private interrupted;
+    event(runId: string, kind: string, message: string): void;
+    events(runId: string): {
+        at: string;
+        kind: string;
+        message: string;
+    }[];
+    ensureUnit(unit: Omit<UnitRow, "status" | "attempts" | "result" | "error">): UnitRow;
+    units(runId: string): UnitRow[];
+    setUnit(id: string, patch: {
+        status: UnitStatus;
+        result?: Record<string, unknown> | null;
+        error?: string | null;
+        attempt?: boolean;
+    }): void;
+    spend(runId: string, unitId?: string): Spend;
+    /**
+     * Reserves a paid request's expected cost against the run budget, atomically across processes.
+     * Refused when settled plus reserved spend plus this request would pass the run's budget.
+     */
+    reserve(input: {
+        runId: string;
+        unitId: string;
+        model: string;
+        endpoint: string;
+        expectedUsd: number;
+    }): {
+        ok: true;
+        id: number;
+    } | {
+        ok: false;
+        spend: Spend;
+        budgetUsd: number;
+    };
+    /** Records what a reserved request cost: provider-reported (`actualUsd`), list-price estimate, or unknown (both null). */
+    settle(id: number, cost: {
+        actualUsd: number | null;
+        estimatedUsd: number | null;
+        latencyMs: number;
+        failed?: boolean;
+    }): void;
+    usage(runId: string, unitId: string): {
+        requests: number;
+        latencyMs: number;
+        actualUsd: number;
+        estimatedUsd: number;
+        unknown: number;
+    };
+    budgetEvent(runId: string, event: {
+        kind: "alert" | "raise" | "decline";
+        fromUsd: number;
+        toUsd?: number;
+        scope?: "run" | "default";
+        spentUsd: number;
+    }): void;
+    budgetEvents(runId: string): {
+        at: string;
+        kind: string;
+        fromUsd: number;
+        toUsd: number | null;
+        scope: string | null;
+        spentUsd: number;
+    }[];
+    decision(runId: string, kind: string, decision: string, detail: unknown): void;
+    decisions(runId: string): {
+        at: string;
+        kind: string;
+        decision: string;
+        detail: unknown;
+    }[];
+}