openmerit 0.3.10 → 0.4.0

package/dist/scout/evaluators/pi.js

@@ -0,0 +1,117 @@
+import { setTimeout as delay } from "node:timers/promises";
+import { CandidateUnavailable } from "../search.js";
+const text = (message) => message.content.filter(part => part.type === "text").map(part => part.text).join("").trim();
+const asText = (value) => (typeof value === "string" ? value : JSON.stringify(value, null, 2));
+const normalize = (value) => value.toLowerCase().replace(/\s+/g, " ").replace(/[.!\s]+$/, "").trim();
+/** The first JSON object or array in a reply, with code fences removed. */
+export function extractJson(reply) {
+    const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(reply)?.[1] ?? reply;
+    const start = fenced.search(/[[{]/);
+    if (start < 0)
+        return undefined;
+    for (let end = fenced.length; end > start; end--) {
+        try {
+            return JSON.parse(fenced.slice(start, end));
+        }
+        catch { /* shorten */ }
+    }
+    return undefined;
+}
+/** Every expected field matches: objects by key, arrays as sets, strings case-insensitively. */
+export function jsonMatches(expected, actual) {
+    if (expected === null || typeof expected !== "object") {
+        if (typeof expected === "string" && typeof actual === "string")
+            return normalize(expected) === normalize(actual);
+        return expected === actual;
+    }
+    if (Array.isArray(expected)) {
+        return Array.isArray(actual) && expected.length === actual.length && expected.every(item => actual.some(other => jsonMatches(item, other)));
+    }
+    if (!actual || typeof actual !== "object" || Array.isArray(actual))
+        return false;
+    return Object.entries(expected).every(([key, value]) => jsonMatches(value, actual[key]));
+}
+const field = (expected, name) => asText(name && expected && typeof expected === "object" ? expected[name] : expected);
+/** Messages with a provider error are infrastructure failures, not quality verdicts. */
+function ensureOk(message, what) {
+    if (message.stopReason === "error" || message.stopReason === "aborted")
+        throw new Error(`${what}: ${message.errorMessage ?? message.stopReason}`);
+}
+const status = (message) => Number(/^(\d{3})\b/.exec(message.errorMessage ?? "")?.[1] ?? 0);
+/**
+ * One call with retries for transient provider errors (429, 5xx, network), waiting 5, 20 and 60 s.
+ * A model the provider rejects (400, 404) or the balance cannot cover (402) is unavailable, not failed.
+ */
+async function call(caller, model, context, options, what, unavailable, waits) {
+    for (const wait of [...waits, -1]) {
+        const reply = await caller.complete(model, context, options);
+        if (reply.stopReason !== "error")
+            return reply;
+        const code = status(reply);
+        if (unavailable && [400, 402, 404, 413, 422].includes(code))
+            throw new CandidateUnavailable(`${what}: ${(reply.errorMessage ?? "").slice(0, 240)}`);
+        if (wait < 0 || (code !== 0 && code !== 429 && code < 500))
+            return reply;
+        await delay(wait, undefined, options.signal ? { signal: options.signal } : undefined);
+    }
+    throw new Error("unreachable");
+}
+function resolve(caller, ref) {
+    const slash = ref.indexOf("/");
+    const model = slash > 0 ? caller.getModel(ref.slice(0, slash), ref.slice(slash + 1)) : undefined;
+    if (!model)
+        throw new Error(`Model ${ref} is not available with the connected providers`);
+    return model;
+}
+/**
+ * Runs each item as one model call through Pi, so any provider Pi connects (API key or
+ * subscription) can be a candidate, then grades the reply. Tool-using agents need an
+ * evaluator module that runs the real harness instead.
+ */
+export function piEvaluator(caller, spec, options = {}) {
+    const waits = options.retryDelaysMs ?? [5_000, 20_000, 60_000];
+    return async (candidate, item, context) => {
+        const model = resolve(caller, candidate.model);
+        const input = asText(item.input);
+        const user = spec.inputTemplate ? spec.inputTemplate.replaceAll("{{input}}", input) : input;
+        const reply = await call(caller, model, { systemPrompt: spec.systemPrompt, messages: [{ role: "user", content: user, timestamp: Date.now() }] }, { ...(context.signal ? { signal: context.signal } : {}), maxTokens: spec.maxTokens ?? 4096 }, candidate.model, true, waits);
+        ensureOk(reply, candidate.model);
+        const output = text(reply);
+        const usage = { input: reply.usage.input, output: reply.usage.output, total: reply.usage.totalTokens };
+        const graded = await grade(spec.grader, item, output, caller, options.judgeModel, waits, context.signal);
+        return { pass: graded.pass, comment: graded.comment, output, costUsd: reply.usage.cost.total, usage, ...(reply.durationMs ? { latencyMs: reply.durationMs } : {}) };
+    };
+}
+async function grade(grader, item, output, caller, judgeModel, waits, signal) {
+    switch (grader.type) {
+        case "exact": {
+            const expected = field(item.expectedOutput, grader.field);
+            return { pass: normalize(output) === normalize(expected), comment: `expected "${expected.slice(0, 80)}"` };
+        }
+        case "contains": {
+            const expected = field(item.expectedOutput, grader.field);
+            return { pass: normalize(output).includes(normalize(expected)), comment: `expected to contain "${expected.slice(0, 80)}"` };
+        }
+        case "json": {
+            const parsed = extractJson(output);
+            if (parsed === undefined)
+                return { pass: false, comment: "reply has no JSON" };
+            const pass = jsonMatches(item.expectedOutput, parsed);
+            return { pass, comment: pass ? "every expected field matches" : `expected ${JSON.stringify(item.expectedOutput).slice(0, 160)}` };
+        }
+        case "judge": {
+            const ref = grader.model ?? judgeModel;
+            if (!ref)
+                throw new Error("The judge grader needs a judge model (--judge-model or the Scout model)");
+            const verdict = await call(caller, resolve(caller, ref), {
+                systemPrompt: "You grade one model output against a labelled expected output. Reply with JSON only: {\"pass\": true|false, \"reason\": \"<one sentence>\"}. Pass only if the output satisfies the rubric and agrees with the expected output on every point the rubric names.",
+                messages: [{ role: "user", content: `Rubric:\n${grader.rubric}\n\nInput:\n${asText(item.input)}\n\nExpected output:\n${asText(item.expectedOutput)}\n\nModel output:\n${output}`, timestamp: Date.now() }],
+            }, { ...(signal ? { signal } : {}), maxTokens: 1024 }, `judge ${ref}`, false, waits);
+            ensureOk(verdict, `judge ${ref}`);
+            const parsed = extractJson(text(verdict));
+            if (typeof parsed?.pass !== "boolean")
+                throw new Error(`Judge ${ref} returned no verdict`);
+            return { pass: parsed.pass, comment: `judge: ${typeof parsed.reason === "string" ? parsed.reason : ""}` };
+        }
+    }
+}

package/dist/scout/pipeline.d.ts

@@ -0,0 +1,111 @@
+import type { LangfuseConnector } from "../connectors/langfuse/index.ts";
+import type { Candidate } from "./candidates.ts";
+import { type EvalItem, type Evaluate, type SearchResult } from "./search.ts";
+export interface DatasetSource {
+    /** An existing Langfuse dataset, such as the customer's `sahra-enrollments`. */
+    langfuseDataset?: string;
+    /** Local labelled items; pushed to Langfuse as `name` before evaluation so every run is an experiment on it. */
+    local?: {
+        name: string;
+        description?: string;
+        items: EvalItem[];
+    };
+}
+export interface ScoutRunOptions {
+    task: string;
+    dataset: DatasetSource;
+    candidates: Candidate[];
+    evaluate: Evaluate;
+    /** Score name each recorded item carries; the pass rule reads it back from Langfuse. */
+    passScore?: string;
+    /** Experiment metadata key for the candidate: `model_<role>`. */
+    role?: string;
+    langfuse: LangfuseConnector | null;
+    sampleSize?: number;
+    finalists?: number;
+    /** See SearchOptions.challengers. */
+    challengers?: number;
+    /** See SearchOptions.repeats. */
+    repeats?: number;
+    concurrency?: number;
+    log?: (line: string) => void;
+    signal?: AbortSignal;
+    /** Injected for tests. */
+    pickIndex?: (remaining: number) => number;
+    langfuseTimeoutMs?: number;
+}
+export interface FinalVerdict {
+    model: string;
+    eci: number;
+    eciSource: string;
+    listPrice: number | null;
+    /** Passes as Langfuse reports them; local passes when Langfuse is not connected. */
+    passes: number;
+    /** Item runs: dataset items × repeats. */
+    total: number;
+    /** Runs of each item. */
+    runs: number;
+    /** Passes per item over the repeats, e.g. { "APP-2005": 2 } when it passed 2 of 3 runs. */
+    passesByItem: Record<string, number>;
+    /** Mean measured cost per item run in USD: Langfuse's recorded generation cost when available. */
+    costPerItemUsd: number | null;
+    localPasses: number;
+    verifiedInLangfuse: boolean;
+    missingScores: string[];
+    /** One Langfuse experiment per repeat (Langfuse allows one result per dataset item per experiment). */
+    experimentIds: string[];
+    experimentId: string | null;
+    experimentUrl: string | null;
+    /** Items that failed at least one run. */
+    failedItems: string[];
+}
+export interface ScoutReport {
+    scout: "OpenMerit Scout";
+    runId: string;
+    task: string;
+    createdAt: string;
+    dataset: {
+        name: string;
+        source: "langfuse" | "local";
+        size: number;
+        datasetId: string | null;
+        url: string | null;
+    };
+    candidates: number;
+    search: Omit<SearchResult, "ordered" | "trials" | "finals" | "unavailable"> & {
+        unavailable: SearchResult["unavailable"];
+        trials: {
+            stage: string;
+            model: string;
+            eci: number;
+            eciSource: string;
+            passes: number;
+            total: number;
+            accepted: boolean;
+            costUsd: number | null;
+            experimentUrl: string | null;
+        }[];
+    };
+    finals: FinalVerdict[];
+    recommendation: FinalVerdict | null;
+    runnerUp: FinalVerdict | null;
+    why: string;
+    spend: {
+        measuredUsd: number;
+        unknownCostItems: number;
+        evaluations: number;
+        /** OpenRouter account credit used between the start and the end of the run (account-wide: concurrent use counts too). */
+        openRouterCreditsUsd?: number;
+    };
+    langfuse: {
+        baseUrl: string;
+        project: string | null;
+        experiments: number;
+    } | null;
+    limitations: string[];
+}
+/**
+ * Scout end to end: dataset → binary search over ECI-ranked candidates → finals on every item,
+ * each probe and final recorded as a Langfuse experiment → the winner from Langfuse's results.
+ */
+export declare function runScout(options: ScoutRunOptions): Promise<ScoutReport>;

package/dist/scout/pipeline.js

@@ -0,0 +1,157 @@
+import { randomUUID } from "node:crypto";
+import { compareFinals, meanCost, scoutSearch } from "./search.js";
+const slug = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60);
+/**
+ * Scout end to end: dataset → binary search over ECI-ranked candidates → finals on every item,
+ * each probe and final recorded as a Langfuse experiment → the winner from Langfuse's results.
+ */
+export async function runScout(options) {
+    const log = options.log ?? (() => { });
+    const runId = `scout-${new Date().toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 4)}`;
+    const passScore = options.passScore ?? "correct";
+    const role = options.role ?? "candidate";
+    const lf = options.langfuse;
+    let items;
+    let datasetName;
+    let datasetId = null;
+    if (options.dataset.langfuseDataset) {
+        if (!lf)
+            throw new Error("A Langfuse dataset needs a Langfuse connection (LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, LANGFUSE_BASE_URL)");
+        datasetName = options.dataset.langfuseDataset;
+        items = (await lf.getDatasetItems(datasetName)).map(item => ({ id: item.id, input: item.input, expectedOutput: item.expectedOutput, metadata: item.metadata }));
+        datasetId = (await lf.getDataset(datasetName)).id;
+    }
+    else if (options.dataset.local) {
+        datasetName = options.dataset.local.name;
+        items = options.dataset.local.items;
+        if (lf) {
+            const pushed = await lf.ensureDataset({
+                name: datasetName, ...(options.dataset.local.description ? { description: options.dataset.local.description } : {}),
+                metadata: { source: "openmerit-scout" },
+                items: items.map(item => ({ id: item.id, input: item.input, expectedOutput: item.expectedOutput, ...(item.metadata ? { metadata: item.metadata } : {}) })),
+            });
+            datasetId = pushed.datasetId;
+        }
+    }
+    else
+        throw new Error("Scout needs a labelled dataset: a Langfuse dataset name or local items");
+    log(`Dataset ${datasetName}: ${items.length} items${datasetId ? ` (Langfuse ${datasetId})` : " (local only)"}`);
+    log(`${options.candidates.length} ECI-ranked candidates · run ${runId}`);
+    const urls = new Map();
+    const project = lf ? await lf.check() : null;
+    const onTrial = async (trial) => {
+        if (!lf || !datasetId)
+            return;
+        const stage = trial.stage === "final" ? "final" : "screen";
+        const recordings = [];
+        for (let repeat = 1; repeat <= trial.runs; repeat++) {
+            const suffix = trial.runs > 1 ? `-r${repeat}` : "";
+            const experimentId = `${runId}-${stage}${suffix}-${slug(trial.candidate.model)}`;
+            const recorded = await lf.recordExperimentItems({
+                datasetId, experimentId, experimentName: `scout ${stage}${trial.runs > 1 ? ` run ${repeat}/${trial.runs}` : ""} ${trial.candidate.model} ${runId}`, traceName: "openmerit-scout-eval",
+                metadata: { [`model_${role}`]: trial.candidate.model, scout_run: runId, stage, ...(trial.runs > 1 ? { repeat, repeats: trial.runs } : {}), eci: trial.candidate.eci, eci_source: trial.candidate.eciSource, task: options.task.slice(0, 200) },
+                items: trial.outcomes.filter(outcome => (outcome.repeat ?? 1) === repeat).map(outcome => {
+                    const item = items.find(entry => entry.id === outcome.itemId);
+                    return {
+                        itemId: outcome.itemId, model: trial.candidate.model, input: item.input, output: outcome.output ?? null,
+                        startMs: outcome.startMs ?? Date.now(), endMs: outcome.endMs ?? Date.now(),
+                        ...(outcome.usage ? { usage: outcome.usage } : {}), ...(typeof outcome.costUsd === "number" ? { costUsd: outcome.costUsd } : {}),
+                        metadata: { stage: trial.stage, scout_run: runId, eci: trial.candidate.eci, ...(trial.runs > 1 ? { repeat } : {}) },
+                        scores: [{ name: passScore, value: outcome.pass, dataType: "BOOLEAN", ...(outcome.comment ? { comment: outcome.comment } : {}) }],
+                    };
+                }),
+            });
+            recordings.push({ experimentId, url: recorded.url });
+        }
+        urls.set(trial, recordings);
+    };
+    const search = await scoutSearch({
+        dataset: items, candidates: options.candidates, evaluate: options.evaluate, onTrial, log,
+        ...(options.sampleSize ? { sampleSize: options.sampleSize } : {}), ...(options.finalists ? { finalists: options.finalists } : {}),
+        ...(options.challengers !== undefined ? { challengers: options.challengers } : {}),
+        ...(options.repeats !== undefined ? { repeats: options.repeats } : {}),
+        ...(options.concurrency ? { concurrency: options.concurrency } : {}), ...(options.pickIndex ? { pickIndex: options.pickIndex } : {}),
+        ...(options.signal ? { signal: options.signal } : {}),
+    });
+    // The decision: read every final back from Langfuse, so the recommendation rests on the recorded experiment.
+    const rule = { score: passScore, equals: true };
+    const finals = [];
+    for (const trial of search.finals) {
+        const recorded = urls.get(trial) ?? [];
+        const readbacks = [];
+        if (lf && recorded.length) {
+            log(`Reading ${trial.candidate.model} back from Langfuse (${recorded.length} experiments)…`);
+            for (const { experimentId } of recorded) {
+                readbacks.push(await lf.waitForExperiment({ experimentId, pass: rule, includeCost: true, since: new Date(Date.now() - 86_400_000) }, { expectItems: items.length, timeoutMs: options.langfuseTimeoutMs ?? 300_000 }));
+            }
+        }
+        const verified = readbacks.length > 0 && readbacks.length === recorded.length;
+        const results = verified
+            ? readbacks.flatMap(readback => readback.items.map(item => ({ itemId: item.itemId, pass: item.passed })))
+            : trial.outcomes.map(outcome => ({ itemId: outcome.itemId, pass: outcome.pass }));
+        const passesByItem = {};
+        for (const item of items)
+            passesByItem[item.id] = 0;
+        for (const result of results)
+            if (result.pass)
+                passesByItem[result.itemId] = (passesByItem[result.itemId] ?? 0) + 1;
+        const langfuseCost = verified && readbacks.every(readback => readback.costUsd != null) ? readbacks.reduce((sum, readback) => sum + readback.costUsd, 0) : null;
+        finals.push({
+            model: trial.candidate.model, eci: trial.candidate.eci, eciSource: trial.candidate.eciSource, listPrice: trial.candidate.price,
+            passes: results.filter(result => result.pass).length, total: results.length, runs: trial.runs, passesByItem,
+            costPerItemUsd: langfuseCost !== null && results.length ? langfuseCost / results.length : meanCost(trial.outcomes),
+            localPasses: trial.passes, verifiedInLangfuse: verified, missingScores: readbacks.flatMap(readback => readback.missingScores),
+            experimentIds: recorded.map(item => item.experimentId), experimentId: recorded[0]?.experimentId ?? null, experimentUrl: recorded[0]?.url ?? null,
+            failedItems: Object.entries(passesByItem).filter(([, passes]) => passes < trial.runs).map(([id]) => id),
+        });
+    }
+    const asTrial = (verdict) => {
+        const trial = search.finals.find(item => item.candidate.model === verdict.model);
+        return { ...trial, passes: verdict.passes, outcomes: trial.outcomes.map(outcome => ({ ...outcome, costUsd: verdict.costPerItemUsd })) };
+    };
+    finals.sort((a, b) => compareFinals(asTrial(a), asTrial(b)));
+    // A stopped run has no recommendation: its finals are incomplete and listed only as evidence.
+    const [recommendation = null, runnerUp = null] = search.status === "selected" ? finals : [];
+    const all = [...search.trials, ...search.finals].flatMap(trial => trial.outcomes);
+    const limitations = [
+        search.limitation,
+        "ECI is Epoch's published capability index. Models without a published score use ScoutBM's estimated or calibrated ECI from the bundled CSV, which carries 2–4 points of error; models with neither are not candidates (see `unranked`).",
+        ...(lf ? [] : ["Langfuse is not connected: results were not recorded or verified in Langfuse."]),
+        ...(search.shortfall ? [search.shortfall] : []),
+        ...(search.status === "stopped" && finals.length ? [`The run stopped, so the ${finals.length} finals shown are partial and no model is recommended.`] : []),
+        ...(search.unavailable.length ? [`${search.unavailable.length} candidates could not run (provider rejected the model or the balance did not cover it) and are untested: ${search.unavailable.map(item => item.model).join(", ")}.`] : []),
+        ...(finals.some(item => item.localPasses !== item.passes) ? ["Langfuse and local pass counts differ for at least one finalist; Langfuse's count was used."] : []),
+    ];
+    return {
+        scout: "OpenMerit Scout", runId, task: options.task, createdAt: new Date().toISOString(),
+        dataset: { name: datasetName, source: options.dataset.langfuseDataset ? "langfuse" : "local", size: items.length, datasetId, url: project && datasetId ? `${lf.baseUrl}/project/${project.projectId}/datasets/${datasetId}` : null },
+        candidates: options.candidates.length,
+        search: {
+            status: search.status, ...(search.error ? { error: search.error } : {}), criterion: search.criterion, sampleIds: search.sampleIds, selected: search.selected, unavailable: search.unavailable,
+            finalRule: search.finalRule, limitation: search.limitation, ...(search.shortfall ? { shortfall: search.shortfall } : {}),
+            trials: search.trials.map(trial => ({
+                stage: trial.stage, model: trial.candidate.model, eci: trial.candidate.eci, eciSource: trial.candidate.eciSource, passes: trial.passes, total: trial.outcomes.length,
+                accepted: trial.accepted, costUsd: meanCost(trial.outcomes) === null ? null : meanCost(trial.outcomes) * trial.outcomes.length, experimentUrl: urls.get(trial)?.[0]?.url ?? null,
+            })),
+        },
+        finals, recommendation, runnerUp,
+        why: explain(search, recommendation, runnerUp),
+        spend: { measuredUsd: all.reduce((sum, item) => sum + (item.costUsd ?? 0), 0), unknownCostItems: all.filter(item => typeof item.costUsd !== "number").length, evaluations: all.length },
+        langfuse: lf && project ? { baseUrl: lf.baseUrl, project: project.projectName, experiments: [...urls.values()].reduce((sum, list) => sum + list.length, 0) } : null,
+        limitations,
+    };
+}
+function explain(search, best, next) {
+    if (search.status === "stopped")
+        return `The search stopped without a verdict: ${search.error}`;
+    if (search.status === "no_passing_model")
+        return `No candidate passed all ${search.sampleIds.length} sampled items, so Scout recommends none. Improve the prompt or harness, or check the labels.`;
+    if (!best)
+        return "No finalist was evaluated.";
+    const cost = (value) => (value === null ? "unknown cost" : `$${value.toFixed(5)}/item`);
+    const head = `${best.model} passed ${best.passes}/${best.total} item runs (${best.runs} per item) in Langfuse at ${cost(best.costPerItemUsd)} (ECI ${best.eci.toFixed(1)}).`;
+    if (!next)
+        return head;
+    const tie = next.passes === best.passes;
+    return `${head} ${tie ? "Tied on passes with" : "Ahead of"} ${next.model} (${next.passes}/${next.total}, ${cost(next.costPerItemUsd)})${tie ? `; ${best.costPerItemUsd !== null && next.costPerItemUsd !== null && best.costPerItemUsd !== next.costPerItemUsd ? "lower measured cost" : "lower ECI"} breaks the tie` : ""}.`;
+}

package/dist/scout/report.d.ts

@@ -0,0 +1,3 @@
+import type { ScoutReport } from "./pipeline.ts";
+/** The decision record as Markdown: no keys, no raw item contents. */
+export declare function reportMarkdown(report: ScoutReport): string;

package/dist/scout/report.js

@@ -0,0 +1,42 @@
+const usd = (value, digits = 5) => (value === null ? "unknown" : `$${value.toFixed(digits)}`);
+const link = (label, url) => (url ? `[${label}](${url})` : label);
+const eciCell = (eci, source) => `${eci.toFixed(1)}${source === "published" ? "" : ` (${source.slice(0, 4)}.)`}`;
+/** The decision record as Markdown: no keys, no raw item contents. */
+export function reportMarkdown(report) {
+    const best = report.recommendation;
+    const lines = [
+        `# Scout recommendation: ${best ? `\`${best.model}\`` : "none"}`,
+        "",
+        `- Task: ${report.task.replace(/\s+/g, " ").slice(0, 500)}`,
+        `- Run: \`${report.runId}\`, ${report.createdAt}`,
+        `- Dataset: ${link(`\`${report.dataset.name}\``, report.dataset.url)} (${report.dataset.source}, ${report.dataset.size} labelled items)`,
+        `- Candidates: ${report.candidates} runnable models with an Epoch Capabilities Index (published, or ScoutBM's estimate: "est."/"cali.")`,
+        `- Langfuse: ${report.langfuse ? `${report.langfuse.project} at ${report.langfuse.baseUrl}, ${report.langfuse.experiments} experiments recorded` : "not connected"}`,
+        `- Spend: ${usd(report.spend.measuredUsd, 4)} measured over ${report.spend.evaluations} evaluations${report.spend.unknownCostItems ? ` (${report.spend.unknownCostItems} without a cost)` : ""}${report.spend.openRouterCreditsUsd !== undefined ? `; OpenRouter account credit used during the run: ${usd(report.spend.openRouterCreditsUsd, 4)}` : ""}`,
+        "",
+        `**Why:** ${report.why}`,
+        "",
+        `## Finals (every item ${report.finals[0]?.runs ?? 1} times, one Langfuse experiment per run, read back from Langfuse)`,
+        "",
+        "| Rank | Model | Passes | Cost per item run | ECI | List price $/M in+out | Verified in Langfuse | Failed items |",
+        "|---|---|---|---|---|---|---|---|",
+        ...report.finals.map((item, index) => `| ${index + 1} | ${link(`\`${item.model}\``, item.experimentUrl)} | ${item.passes}/${item.total} | ${usd(item.costPerItemUsd)} | ${eciCell(item.eci, item.eciSource)} | ${item.listPrice === null ? "unknown" : item.listPrice.toFixed(2)} | ${item.verifiedInLangfuse ? "yes" : "no"}${item.missingScores.length ? ` (${item.missingScores.length} unscored)` : ""} | ${item.failedItems.map(id => `${id} (${item.passesByItem[id] ?? 0}/${item.runs})`).join(", ") || "none"} |`),
+        "",
+        `## Binary search (${report.search.criterion}; samples ${report.search.sampleIds.join(", ")})`,
+        "",
+        `Status: **${report.search.status}**${report.search.error ? ` — ${report.search.error}` : ""}. Search pick: ${report.search.selected ? `\`${report.search.selected.model}\`` : "none"}.`,
+        "",
+        "| # | Stage | Model | ECI | Passes | Verdict | Cost |",
+        "|---|---|---|---|---|---|---|",
+        ...report.search.trials.map((trial, index) => `| ${index + 1} | ${trial.stage} | ${link(`\`${trial.model}\``, trial.experimentUrl)} | ${eciCell(trial.eci, trial.eciSource)} | ${trial.passes}/${trial.total} | ${trial.accepted ? "pass" : "fail"} | ${usd(trial.costUsd, 4)} |`),
+        ...(report.search.unavailable.length ? ["", "Skipped (could not run, untested):", "", ...report.search.unavailable.map(item => `- \`${item.model}\` (${item.stage}): ${item.reason.slice(0, 200)}`)] : []),
+        "",
+        `Rule: ${report.search.finalRule}`,
+        "",
+        "## Limitations",
+        "",
+        ...report.limitations.map(item => `- ${item}`),
+        "",
+    ];
+    return lines.join("\n");
+}

package/dist/scout/search.d.ts

@@ -0,0 +1,110 @@
+import type { Candidate } from "./candidates.ts";
+/**
+ * Scout Model Binary Search, from the ScoutBM prototype:
+ * 1. Rank candidates by ECI, ascending, and draw `sampleSize` labelled items once.
+ * 2. Binary search: a model "performs well" only when it passes every sampled item. Passing moves
+ *    to weaker (cheaper-on-capability) models, failing moves to stronger ones. The search pick is
+ *    the weakest model that passed.
+ * 3. Screen the nearest neighbours on the same samples (weaker first at equal distance; once a
+ *    weaker one fails, only stronger ones) until there are `finalists` passing models.
+ * 4. Finals: every finalist runs every dataset item. The caller decides the winner from the
+ *    recorded results (Langfuse), falling back to `orderFinals`.
+ */
+export interface EvalItem {
+    id: string;
+    input: unknown;
+    expectedOutput: unknown;
+    metadata?: Record<string, unknown> | null;
+}
+export interface Outcome {
+    itemId: string;
+    pass: boolean;
+    /** What the candidate produced, recorded as the experiment item's output. */
+    output?: unknown;
+    /** Measured provider cost of this item, when known. */
+    costUsd?: number | null;
+    latencyMs?: number;
+    usage?: {
+        input: number;
+        output: number;
+        total: number;
+    } | null;
+    /** Why the item passed or failed, in a sentence. */
+    comment?: string;
+    startMs?: number;
+    endMs?: number;
+    /** Which repeat of a final this was, from 1. Screens run once and leave it unset. */
+    repeat?: number;
+}
+export type Stage = "search" | "neighbour" | "challenger" | "final";
+/**
+ * Thrown by an evaluator when this one candidate cannot run, such as a provider rejecting the model
+ * (400/404) or the balance not covering its request (402). The candidate is skipped and listed as
+ * untested; the search goes on without it. Any other throw stops the search.
+ */
+export declare class CandidateUnavailable extends Error {
+    readonly name = "CandidateUnavailable";
+}
+/** Runs one item on one candidate. Throw on transport, auth, billing or harness failures: they never count as quality failures. */
+export type Evaluate = (candidate: Candidate, item: EvalItem, context: {
+    stage: Stage;
+    signal?: AbortSignal;
+}) => Promise<Omit<Outcome, "itemId">>;
+export interface Trial {
+    stage: Stage;
+    candidate: Candidate;
+    outcomes: Outcome[];
+    passes: number;
+    /** Screens: every sample passed. Finals: always true. */
+    accepted: boolean;
+    /** Runs of each item: 1 for screens, `repeats` for finals. `outcomes` holds items × runs. */
+    runs: number;
+}
+export interface SearchOptions {
+    dataset: readonly EvalItem[];
+    /** Ascending by ECI, as `rankCandidates` returns them. */
+    candidates: readonly Candidate[];
+    evaluate: Evaluate;
+    sampleSize?: number;
+    finalists?: number;
+    /** Items evaluated at once within one probe. Default 5. */
+    concurrency?: number;
+    /** Runs of every item per finalist, so a model that passes by chance does not tie a reliable one (default 3). */
+    repeats?: number;
+    /** Untested stronger models no dearer than the leading finalist to screen after the finals (default 3; 0 turns it off). */
+    challengers?: number;
+    /** Called after every probe, screen and final, for example to record it in Langfuse. A throw stops the search. */
+    onTrial?: (trial: Trial) => Promise<void> | void;
+    pickIndex?: (remaining: number) => number;
+    signal?: AbortSignal;
+    log?: (line: string) => void;
+}
+export interface SearchResult {
+    status: "selected" | "no_passing_model" | "stopped";
+    error?: string;
+    criterion: string;
+    sampleIds: string[];
+    ordered: Candidate[];
+    /** The weakest model that passed every sample during the binary search. */
+    selected: Candidate | null;
+    trials: Trial[];
+    finals: Trial[];
+    /** Candidates skipped because they could not run; untested, not failed. */
+    unavailable: {
+        model: string;
+        stage: Stage;
+        reason: string;
+    }[];
+    shortfall?: string;
+    finalRule: string;
+    limitation: string;
+}
+export declare const SAMPLE_SIZE = 5;
+export declare const FINALISTS = 5;
+export declare function scoutSearch(input: SearchOptions): Promise<SearchResult>;
+/** Models other than the pick, nearest by rank first; at equal distance the weaker one comes first. */
+export declare function outward(ordered: readonly Candidate[], pick: Candidate): Candidate[];
+/** Mean measured cost per item, or null when any item's cost is unknown. */
+export declare function meanCost(outcomes: readonly Pick<Outcome, "costUsd">[]): number | null;
+/** Most passes; then lower measured cost per item (list price only when a cost is unknown); then lower ECI. */
+export declare function compareFinals(a: Trial, b: Trial): number;