openmerit 0.4.0 → 0.4.2

package/dist/catalogue/catalogue.d.ts

@@ -0,0 +1,41 @@
+export declare const CATALOGUE_FORMAT = 1;
+export interface CatalogueModel {
+    /** OpenRouter-style ID, such as `openai/gpt-6.1-sol`. */
+    id: string;
+    name: string;
+    /** Other spellings code uses for the same model: the bare vendor ID, dated slugs. */
+    aliases: string[];
+    /** Epoch Capabilities Index: published, or ScoutBM's estimate. Null when neither exists. */
+    eci: number | null;
+    eciSource: "published" | "estimated" | "calibrated" | null;
+    eciError: number | null;
+    eciModel: string | null;
+    /** List prices in USD per million tokens; null when unknown. */
+    inputPerM: number | null;
+    outputPerM: number | null;
+    contextLength: number | null;
+    tools: boolean;
+    structuredOutputs: boolean;
+    vision: boolean;
+    released: string | null;
+}
+export interface Catalogue {
+    format: number;
+    /** Date of the snapshot, recorded with every run. */
+    version: string;
+    generatedAt: string;
+    sources: {
+        name: string;
+        url: string;
+        note: string;
+    }[];
+    models: CatalogueModel[];
+}
+export declare function loadCatalogue(path?: URL): Catalogue;
+/** Finds a catalogue model by ID or alias, ignoring case and an `openrouter/` prefix. */
+export declare function catalogueIndex(catalogue: Catalogue): {
+    find: (value: string) => CatalogueModel | null;
+    size: number;
+};
+/** List price of one request, or null when a price is unknown. */
+export declare function requestPrice(model: Pick<CatalogueModel, "inputPerM" | "outputPerM"> | null, inputTokens: number, outputTokens: number): number | null;

package/dist/catalogue/catalogue.js

@@ -0,0 +1,33 @@
+// The model/evidence catalogue bundled with the npm package (data/catalogue.json). Scout reads only
+// this file: no hosted catalogue, no download at run time. Package updates refresh it.
+import { readFileSync } from "node:fs";
+export const CATALOGUE_FORMAT = 1;
+let cached = null;
+export function loadCatalogue(path = new URL("../../data/catalogue.json", import.meta.url)) {
+    if (cached && path.href.endsWith("/data/catalogue.json"))
+        return cached;
+    const catalogue = JSON.parse(readFileSync(path, "utf8"));
+    if (catalogue.format !== CATALOGUE_FORMAT || !Array.isArray(catalogue.models))
+        throw new Error(`Unsupported catalogue format in ${path.pathname}`);
+    cached = catalogue;
+    return catalogue;
+}
+/** Anthropic's API writes `claude-sonnet-5-5` where OpenRouter writes `claude-sonnet-5.5`. */
+const key = (value) => value.toLowerCase().replace(/^openrouter\//, "").replace(/(\d)\.(\d)/g, "$1-$2");
+/** Finds a catalogue model by ID or alias, ignoring case and an `openrouter/` prefix. */
+export function catalogueIndex(catalogue) {
+    const byKey = new Map();
+    for (const model of catalogue.models)
+        byKey.set(key(model.id), model);
+    for (const model of catalogue.models)
+        for (const alias of model.aliases)
+            if (!byKey.has(key(alias)))
+                byKey.set(key(alias), model);
+    return { find: (value) => byKey.get(key(value)) ?? null, size: byKey.size };
+}
+/** List price of one request, or null when a price is unknown. */
+export function requestPrice(model, inputTokens, outputTokens) {
+    if (!model || model.inputPerM === null || model.outputPerM === null)
+        return null;
+    return (inputTokens * model.inputPerM + outputTokens * model.outputPerM) / 1e6;
+}

package/dist/cli/prompt.d.ts

@@ -0,0 +1,26 @@
+import type { Answer, Question } from "../scout/workflow.js";
+export interface Prompter {
+    /** `signal` cancels a pending question, which then rejects. */
+    ask(question: string, fallback?: string, signal?: AbortSignal): Promise<string>;
+    secret(question: string): Promise<string>;
+    confirm(question: string, fallback?: boolean): Promise<boolean>;
+    choose<T extends string>(question: string, options: {
+        value: T;
+        label: string;
+    }[], fallback?: T): Promise<T>;
+    /** Releases the terminal to `run`, such as a child process with inherited stdin, then takes it back. */
+    handOver<T>(run: () => Promise<T>): Promise<T>;
+    close(): void;
+}
+export declare function terminalPrompter(input?: NodeJS.ReadableStream, output?: NodeJS.WritableStream): Prompter;
+/**
+ * On a terminal, shows that Scout is still working when nothing has been printed for `quietMs`: one
+ * line with a spinner, a rotating word and the seconds since the last output. `log` clears it before
+ * printing. Without a terminal it only prints.
+ */
+export declare function workingTicker(output?: NodeJS.WriteStream, quietMs?: number): {
+    log(line: string): void;
+    stop(): void;
+};
+/** Asks one workflow question in the terminal. */
+export declare function askQuestion(prompter: Prompter, question: Question, log: (line: string) => void): Promise<Answer>;

package/dist/cli/prompt.js

@@ -0,0 +1,114 @@
+// Terminal prompts on stderr, so stdout stays clean for --json output.
+import { createInterface } from "node:readline/promises";
+export function terminalPrompter(input = process.stdin, output = process.stderr) {
+    let muted = false;
+    const write = output.write.bind(output);
+    const open = () => {
+        const next = createInterface({ input, output, terminal: input.isTTY === true });
+        next._writeToOutput = (text) => { if (!muted || text.includes("\n"))
+            write(muted ? "\n" : text); };
+        return next;
+    };
+    let rl = open();
+    const ask = async (question, fallback, signal) => {
+        const answer = (await rl.question(`${question}${fallback !== undefined && fallback !== "" ? ` [${fallback}]` : ""} `, signal ? { signal } : {})).trim();
+        return answer || fallback || "";
+    };
+    return {
+        ask,
+        async secret(question) {
+            write(`${question} `);
+            muted = true;
+            try {
+                return (await rl.question("")).trim();
+            }
+            finally {
+                muted = false;
+            }
+        },
+        async confirm(question, fallback = false) {
+            for (;;) {
+                const answer = (await ask(`${question} ${fallback ? "[Y/n]" : "[y/N]"}`)).toLowerCase();
+                if (!answer)
+                    return fallback;
+                if (/^(y|yes)$/.test(answer))
+                    return true;
+                if (/^(n|no)$/.test(answer))
+                    return false;
+            }
+        },
+        async choose(question, options, fallback) {
+            write(`${question}\n${options.map((option, index) => `  ${index + 1}. ${option.label}`).join("\n")}\n`);
+            for (;;) {
+                const fallbackIndex = fallback ? options.findIndex(option => option.value === fallback) + 1 : 0;
+                const answer = await ask("Choose a number", fallbackIndex ? String(fallbackIndex) : undefined);
+                const picked = options[Number(answer) - 1] ?? options.find(option => option.value === answer);
+                if (picked)
+                    return picked.value;
+            }
+        },
+        // An open readline keeps reading the terminal, so a child sharing it would lose keystrokes to it.
+        async handOver(run) {
+            rl.close();
+            try {
+                return await run();
+            }
+            finally {
+                rl = open();
+            }
+        },
+        close() { rl.close(); },
+    };
+}
+const WORKING = ["eval-ing", "checking", "understanding", "proof-reading", "comparing", "tallying", "weighing", "cross-checking", "pondering", "measuring"];
+const SPINNER = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏";
+/**
+ * On a terminal, shows that Scout is still working when nothing has been printed for `quietMs`: one
+ * line with a spinner, a rotating word and the seconds since the last output. `log` clears it before
+ * printing. Without a terminal it only prints.
+ */
+export function workingTicker(output = process.stderr, quietMs = 3000) {
+    let last = Date.now(), shown = false, frame = 0;
+    const first = Math.floor(Math.random() * WORKING.length);
+    const clear = () => { if (shown) {
+        output.write("\r\x1b[2K");
+        shown = false;
+    } };
+    const timer = output.isTTY ? setInterval(() => {
+        const quiet = Date.now() - last;
+        if (quiet < quietMs)
+            return;
+        output.write(`\r\x1b[2K${SPINNER[frame++ % SPINNER.length]} ${WORKING[(first + Math.floor(quiet / 4000)) % WORKING.length]}… ${Math.round(quiet / 1000)}s`);
+        shown = true;
+    }, 120) : null;
+    timer?.unref();
+    return {
+        log(line) { clear(); output.write(`${line}\n`); last = Date.now(); },
+        stop() { if (timer)
+            clearInterval(timer); clear(); },
+    };
+}
+/** Asks one workflow question in the terminal. */
+export async function askQuestion(prompter, question, log) {
+    if (question.detail)
+        log(`\n${question.detail}\n`);
+    switch (question.kind) {
+        case "confirm": return prompter.confirm(question.prompt, question.default === true);
+        case "choice": return prompter.choose(question.prompt, question.options ?? [], typeof question.default === "string" ? question.default : undefined);
+        case "multi": {
+            const options = question.options ?? [];
+            log(`${question.prompt}\n${options.map((option, index) => `  ${index + 1}. ${option.label}${option.detail ? `\n     ${option.detail}` : ""}`).join("\n")}`);
+            for (;;) {
+                const answer = await prompter.ask(`Numbers or "all"${question.hint ? `, ${question.hint}` : ""}`, options.length === 1 ? "1" : undefined);
+                // Entries with "=" (such as a mapping correction) pass through as typed; the rest are option numbers.
+                const items = answer.split(",").map(item => item.trim()).filter(Boolean);
+                const typed = question.hint ? items.filter(item => item.includes("=")) : [];
+                const numbers = items.filter(item => !typed.includes(item)).flatMap(item => item.split(/\s+/));
+                const picked = numbers.length === 1 && /^all$/i.test(numbers[0]) ? options : numbers.map(item => options[Number(item) - 1]);
+                if ((picked.length || typed.length) && picked.every(Boolean))
+                    return [...typed, ...picked.map(option => option.value)];
+            }
+        }
+        case "text": return prompter.ask(question.prompt, typeof question.default === "string" ? question.default : undefined);
+    }
+}

package/dist/cli/render.d.ts

@@ -0,0 +1,12 @@
+import type { RunRow, State } from "../state/db.js";
+import type { SearchSummary, Step, TaskComparison } from "../scout/workflow.js";
+export declare function renderComparison(task: TaskComparison, runId?: string): string;
+/** The search path: each screen, in order, with its result on the sampled items. */
+export declare function renderSearch(search: SearchSummary): string[];
+/** The run's outcome; `comparisons: false` leaves out the task comparisons, such as when they were shown already. */
+export declare function renderStep(step: Step, options?: {
+    comparisons?: boolean;
+}): string;
+export declare function spendLine(step: Pick<Step, "spend" | "budgetUsd" | "harnessUsage">): string;
+export declare function renderHistory(state: State, runs: RunRow[]): string;
+export declare function renderRun(state: State, run: RunRow, step: Step): string;

package/dist/cli/render.js

@@ -0,0 +1,104 @@
+// Two significant digits below a cent, so per-item costs such as $0.000023 and $0.0000088 stay apart.
+const money = (value, estimate = false) => (value === null ? "unknown" : `${estimate ? "~" : ""}$${value.toFixed(value > 0 && value < 0.01 ? Math.min(10, 1 - Math.floor(Math.log10(value))) : 4)}`);
+const pad = (rows) => {
+    const widths = rows[0].map((_, column) => Math.max(...rows.map(row => (row[column] ?? "").length)));
+    return rows.map(row => row.map((cell, column) => cell.padEnd(widths[column])).join("  ").trimEnd()).join("\n");
+};
+function row(label, result) {
+    return [label, result.model, result.endpoint, result.total ? `${result.passes}/${result.total}` : "—", money(result.costPerItemUsd, result.costIsEstimate),
+        result.meanLatencyMs === null ? "—" : `${Math.round(result.meanLatencyMs)} ms`, result.status === "measured" ? (result.role === "baseline" ? "baseline" : result.qualified ? "qualifies" : result.note || "below baseline") : result.note];
+}
+export function renderComparison(task, runId) {
+    const lines = [`Task "${task.task}"`, `  Eval: ${task.eval.command || "none"} (${task.eval.source}; per-item results ${task.eval.langfuse ? `from Langfuse, ${task.eval.langfuse}` : "reported by the eval"}${task.eval.threshold !== null ? `; required pass rate ${task.eval.threshold}` : ""})`];
+    if (task.status === "no_eval")
+        return [...lines, `  Not reviewed: ${task.note}`].join("\n");
+    if (task.search)
+        lines.push(...renderSearch(task.search));
+    const header = ["", "Model", "Endpoint", "Passed", "Cost/item", "Latency/request", "Status"];
+    const table = [header, ...(task.baseline ? [row("Current", task.baseline)] : []), ...task.top.map((result, index) => row(`#${index + 1}`, result))];
+    lines.push(pad(table).split("\n").map(line => `  ${line}`).join("\n"));
+    const others = task.candidates.filter(item => !task.top.includes(item) && !task.top.some(top => top.model === item.model));
+    if (others.length)
+        lines.push("  Other candidates:", pad([header, ...others.map(item => row("", item))]).split("\n").slice(1).map(line => `  ${line}`).join("\n"));
+    lines.push(`  ${task.status === "ranked" ? "Top 3 cheapest models at least as good as the current model, measured on the same eval." : task.status === "no_qualified" ? "No finalist performed at least as well as the current model." : task.status === "non_discriminating" ? "The eval did not separate the models." : ""} ${task.note}`.trimEnd());
+    const experiments = [task.baseline, ...task.candidates].flatMap(item => item?.source === "langfuse" ? item.experiments : []);
+    if (experiments.length)
+        lines.push(`  Read back from ${experiments.length} Langfuse experiments your eval recorded, named ${experiments.reduce((prefix, name) => { let at = 0; while (at < prefix.length && prefix[at] === name[at])
+            at++; return prefix.slice(0, at); }).replace(/[^-]*$/, "")}…; \`openmerit history\` lists each eval run's experiment.`);
+    if (task.top.some(item => item.costIsEstimate) || task.baseline?.costIsEstimate)
+        lines.push("  ~ cost estimated from list prices and reported tokens; the endpoint does not report cost.");
+    if (task.langfuse) {
+        if ("error" in task.langfuse)
+            lines.push(`  Langfuse (read-only): not available (${task.langfuse.error})`);
+        else
+            lines.push(`  Langfuse production evidence, ${task.langfuse.traceName} since ${task.langfuse.from.slice(0, 10)}: ${task.langfuse.models.map(model => `${model.model} ${model.generations} generations, ${model.costUsd === null ? "cost unknown" : `$${model.costUsd.toFixed(4)}`}`).join("; ") || "no generations"}${task.langfuse.scores.length ? `; scores ${task.langfuse.scores.map(score => `${score.name} ${score.average.toFixed(3)} (n=${score.count})`).join(", ")}` : ""}`);
+    }
+    lines.push(`  Evidence: openmerit history ${runId ?? "<run id>"} lists every eval run (${[task.baseline, ...task.candidates].filter(Boolean).reduce((sum, item) => sum + item.evidence.length, 0)} recorded).`);
+    return lines.join("\n");
+}
+/** The search path: each screen, in order, with its result on the sampled items. */
+export function renderSearch(search) {
+    const lines = [`  Scout Model Binary Search over ${search.pool} ECI-ranked candidates${search.unranked ? ` (${search.unranked} without an ECI left out)` : ""}, screening on ${search.sampleIds.length || 5} items the current model passes:`];
+    for (const trial of search.trials)
+        lines.push(`    ${trial.stage.padEnd(10)} ${trial.model} (ECI ${trial.eci.toFixed(1)}): ${trial.passes}/${trial.total} ${trial.accepted ? "pass" : "fail"}`);
+    for (const item of search.skipped)
+        lines.push(`    skipped    ${item.model} at ${item.stage}: ${item.reason.split("\n")[0]}`);
+    if (search.pick)
+        lines.push(`    Weakest passing model: ${search.pick}. Finalists: ${search.finalists.join(", ") || "none"}${search.challengers.length ? `. Challengers: ${search.challengers.join(", ")}` : ""}.`);
+    if (search.shortfall)
+        lines.push(`    ${search.shortfall}`);
+    return lines;
+}
+/** The run's outcome; `comparisons: false` leaves out the task comparisons, such as when they were shown already. */
+export function renderStep(step, options = {}) {
+    const lines = [];
+    if (step.result && options.comparisons !== false)
+        for (const task of step.result.tasks)
+            lines.push(renderComparison(task, step.runId), "");
+    for (const [task, configure] of Object.entries(step.result?.configuration ?? {})) {
+        if (configure.decision === "no")
+            lines.push(`"${task}": not configured; your application configuration is unchanged.`);
+        else if (configure.applied)
+            lines.push(`"${task}": configured (${configure.edits.map(edit => `${edit.file}:${edit.line}`).join(", ")}).`);
+        for (const item of configure.manual ?? [])
+            lines.push(`"${task}" manual step: ${item}`);
+    }
+    lines.push(spendLine(step));
+    if (step.status !== "done")
+        lines.push(`Run ${step.runId}: ${step.status} at ${step.phase}${step.error ? ` — ${step.error}` : ""}. Resume with: openmerit scout --resume ${step.runId}`);
+    else
+        lines.push(`Run ${step.runId} saved. See it again with: openmerit history ${step.runId}`);
+    return lines.join("\n");
+}
+export function spendLine(step) {
+    const spend = step.spend;
+    const harness = step.harnessUsage;
+    return `Eval spend: $${spend.totalUsd.toFixed(4)} of $${step.budgetUsd.toFixed(2)} over ${spend.requests} requests${spend.estimatedUsd ? ` ($${spend.estimatedUsd.toFixed(4)} estimated from list prices)` : ""}${spend.inFlight ? `, $${spend.inFlightUsd.toFixed(4)} reserved for ${spend.inFlight} request(s) of unknown cost` : ""}${spend.unknown ? `, ${spend.unknown} unpriced` : ""}.`
+        + (harness ? ` Harness inference (separate from the eval budget): ${harness.calls} calls, ${harness.inputTokens + harness.outputTokens} tokens${harness.costUsd ? `, $${harness.costUsd.toFixed(4)}` : ""}${harness.unknownCost ? ` (${harness.unknownCost} calls billed by your coding-agent plan)` : ""}.` : "");
+}
+/** A run's selected tasks, cut to one short cell; `openmerit history <run>` lists them all. */
+function tasksCell(selected) {
+    const text = selected.join(", ");
+    return !text ? "—" : text.length <= 40 ? text : `${text.slice(0, 39)}… (${selected.length})`;
+}
+export function renderHistory(state, runs) {
+    if (!runs.length)
+        return "No Scout runs in this repository yet. Start one with: openmerit scout";
+    return pad([["Run", "Started", "Status", "Phase", "Model", "Tasks", "Eval spend", "openmerit / catalogue"], ...runs.map(run => {
+            const data = run.state;
+            return [run.id, run.createdAt.slice(0, 16).replace("T", " "), run.status, run.phase, data.model ?? "—", tasksCell(data.selected ?? []), `$${state.spend(run.id).totalUsd.toFixed(4)} / $${run.budgetUsd.toFixed(2)}`, `${run.packageVersion} / ${run.catalogueVersion}`];
+        })]);
+}
+export function renderRun(state, run, step) {
+    const section = (title, lines, none) => ["", title, ...(lines.length ? lines : [`  ${none}`])];
+    const lines = [
+        `Run ${run.id} (${run.status === "done" ? "done" : `${run.status} at ${run.phase}`})`, `  Repository ${run.repoRoot} at ${run.revision ?? "no commit"}`,
+        `  openmerit ${run.packageVersion}, catalogue ${run.catalogueVersion}, harness ${run.harness}`, "",
+        renderStep(step),
+        ...section("Eval runs:", state.units(run.id).map(unit => `  ${unit.status.padEnd(13)} ${unit.task} · ${unit.role} ${unit.model} on ${unit.endpoint} · run ${unit.repeat} · ${unit.attempts} attempt(s) · ${money(state.spend(run.id, unit.id).totalUsd)}${typeof unit.result?.total === "number" ? ` · ${String(unit.result.passes)}/${String(unit.result.total)} passed${unit.result.source === "langfuse" ? ` (Langfuse ${String(unit.result.experiment)})` : ""}` : ""}${unit.error ? ` · ${unit.error.split("\n")[0]}` : ""}`), "none"),
+        ...section("Budget:", state.budgetEvents(run.id).map(event => `  ${event.at.slice(0, 19)} ${event.kind}${event.toUsd !== null ? ` $${event.fromUsd.toFixed(2)} → $${event.toUsd.toFixed(2)} (${event.scope})` : ` at $${event.spentUsd.toFixed(4)} of $${event.fromUsd.toFixed(2)}`}`), `$${run.budgetUsd.toFixed(2)} for this run; not reached or changed.`),
+        ...section("Decisions:", state.decisions(run.id).map(decision => `  ${decision.at.slice(0, 19)} ${decision.kind}: ${decision.decision}${decision.kind === "configure" && decision.detail.diff ? `\n${decision.detail.diff.split("\n").map(line => `    ${line}`).join("\n")}` : ""}`), "none"),
+        ...section("Events:", state.events(run.id).map(event => `  ${event.at.slice(0, 19)} ${event.kind}: ${event.message}`), "none"),
+    ];
+    return lines.join("\n");
+}