openmerit 0.3.10 → 0.4.0

package/dist/scout/search.js

@@ -0,0 +1,205 @@
+import { randomInt } from "node:crypto";
+/**
+ * Thrown by an evaluator when this one candidate cannot run, such as a provider rejecting the model
+ * (400/404) or the balance not covering its request (402). The candidate is skipped and listed as
+ * untested; the search goes on without it. Any other throw stops the search.
+ */
+export class CandidateUnavailable extends Error {
+    name = "CandidateUnavailable";
+}
+/** By name, so an evaluator module can throw its own `CandidateUnavailable` without importing OpenMerit. */
+const unavailableError = (error) => error instanceof Error && error.name === "CandidateUnavailable";
+export const SAMPLE_SIZE = 5;
+export const FINALISTS = 5;
+export async function scoutSearch(input) {
+    const sampleSize = input.sampleSize ?? SAMPLE_SIZE;
+    const finalistCount = input.finalists ?? FINALISTS;
+    if (input.dataset.length < sampleSize)
+        throw new Error(`Scout needs an eval dataset with at least ${sampleSize} labelled items; it has ${input.dataset.length}`);
+    if (new Set(input.dataset.map(item => item.id)).size !== input.dataset.length)
+        throw new Error("Dataset item IDs must be unique");
+    if (!input.candidates.length)
+        throw new Error("No ECI-ranked candidate models are runnable with the connected providers");
+    if (new Set(input.candidates.map(item => item.model)).size !== input.candidates.length)
+        throw new Error("Candidate models must be unique");
+    const ordered = [...input.candidates].sort((a, b) => a.eci - b.eci || a.model.localeCompare(b.model));
+    const pool = [...input.dataset];
+    const samples = [];
+    const pick = input.pickIndex ?? randomInt;
+    for (let count = 0; count < sampleSize; count++) {
+        const index = pick(pool.length);
+        if (!Number.isInteger(index) || index < 0 || index >= pool.length)
+            throw new Error("Invalid random sample index");
+        samples.push(pool.splice(index, 1)[0]);
+    }
+    const repeats = input.repeats ?? 3;
+    if (!Number.isInteger(repeats) || repeats < 1)
+        throw new Error("repeats must be a positive integer");
+    const log = input.log ?? (() => { });
+    const trials = [];
+    const finals = [];
+    const unavailable = [];
+    let selected = null;
+    const result = (status, error) => ({
+        status, ...(error ? { error } : {}),
+        criterion: `${sampleSize}/${sampleSize} sampled items pass`, sampleIds: samples.map(item => item.id), ordered, selected, trials, finals, unavailable,
+        ...(status === "selected" && finals.length < finalistCount ? { shortfall: `Only ${finals.length} finalists: no further model passed the screen within the neighbour rule (once a weaker neighbour fails, only stronger models are tried)` } : {}),
+        finalRule: `Finalists are the search pick and the nearest models by ECI that also pass the same ${sampleSize} samples (${finalistCount} in all; once a weaker neighbour fails, only stronger ones are tried), plus up to ${input.challengers ?? 3} untested challengers with a higher ECI and no higher list price than the leading finalist that pass the samples. Each runs every dataset item ${repeats} times; the winner has the most passes over all runs in Langfuse, then the lower measured cost per run, then the lower ECI.`,
+        limitation: `Binary search assumes task quality rises with ECI; models it skips are untested. Screens run each sample once, so a model can be screened out by one unlucky run. ${repeats} runs per final item narrow chance differences but do not remove them, and the result is only as representative as the eval dataset.`,
+    });
+    const run = async (stage, candidate, items) => {
+        const runs = stage === "final" ? repeats : 1;
+        // Repeat-major order, so an interrupted final has whole repeats rather than one item run many times.
+        const tasks = Array.from({ length: runs }, (_, r) => items.map(item => ({ item, repeat: r + 1 }))).flat();
+        const outcomes = new Array(tasks.length);
+        let next = 0;
+        const worker = async () => {
+            while (next < tasks.length) {
+                const index = next++;
+                const { item, repeat } = tasks[index];
+                const startMs = Date.now();
+                const outcome = await input.evaluate(candidate, item, { stage, ...(input.signal ? { signal: input.signal } : {}) });
+                if (typeof outcome?.pass !== "boolean")
+                    throw new Error("Evaluator must return { pass: boolean } or throw on infrastructure failure");
+                outcomes[index] = { startMs, endMs: Date.now(), latencyMs: Date.now() - startMs, ...outcome, itemId: item.id, ...(runs > 1 ? { repeat } : {}) };
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(input.concurrency ?? 5, tasks.length) }, worker));
+        const passes = outcomes.filter(item => item.pass).length;
+        const trial = { stage, candidate, outcomes, passes, runs, accepted: stage === "final" || passes === tasks.length };
+        log(`${stage.padEnd(9)} ${candidate.model} (ECI ${candidate.eci.toFixed(1)}): ${passes}/${tasks.length}${runs > 1 ? ` over ${runs} runs of ${items.length} items` : ""}${stage === "final" ? "" : trial.accepted ? " pass" : " fail"}`);
+        await input.onTrial?.(trial);
+        return trial;
+    };
+    const failure = (stage, candidate, error) => result("stopped", `${stage} ${candidate.model}: ${error instanceof Error ? error.message : String(error)}`);
+    const skip = (stage, candidate, error) => {
+        unavailable.push({ model: candidate.model, stage, reason: error.message });
+        log(`${"skipped".padEnd(9)} ${candidate.model}: ${error.message.slice(0, 200)}`);
+    };
+    let lower = 0, upper = ordered.length - 1;
+    while (lower <= upper) {
+        const middle = Math.floor((lower + upper) / 2);
+        const candidate = ordered[middle];
+        try {
+            const trial = await run("search", candidate, samples);
+            trials.push(trial);
+            if (trial.accepted) {
+                selected = candidate;
+                upper = middle - 1;
+            }
+            else
+                lower = middle + 1;
+        }
+        catch (error) {
+            if (!unavailableError(error))
+                return failure("search", candidate, error);
+            // Untested, not failed: drop it from the ranking and search the same range without it.
+            skip("search", candidate, error);
+            ordered.splice(middle, 1);
+            upper -= 1;
+        }
+    }
+    if (!selected)
+        return result("no_passing_model");
+    const screened = new Map(trials.map(trial => [trial.candidate.model, trial.accepted]));
+    const finalists = [selected];
+    const pickAt = ordered.indexOf(selected);
+    const skipped = new Set();
+    let lookBelow = true;
+    for (const candidate of outward(ordered, selected)) {
+        if (finalists.length === finalistCount)
+            break;
+        const below = ordered.indexOf(candidate) < pickAt;
+        if (below && !lookBelow)
+            continue;
+        if (skipped.has(candidate.model))
+            continue;
+        let accepted = screened.get(candidate.model);
+        if (accepted === undefined) {
+            try {
+                const trial = await run("neighbour", candidate, samples);
+                trials.push(trial);
+                accepted = trial.accepted;
+            }
+            catch (error) {
+                if (!unavailableError(error))
+                    return failure("screen", candidate, error);
+                skip("neighbour", candidate, error);
+                skipped.add(candidate.model);
+                continue;
+            }
+        }
+        if (accepted)
+            finalists.push(candidate);
+        else if (below)
+            lookBelow = false;
+    }
+    for (const candidate of finalists) {
+        try {
+            finals.push(await run("final", candidate, input.dataset));
+        }
+        catch (error) {
+            if (!unavailableError(error))
+                return failure("final", candidate, error);
+            skip("final", candidate, error);
+        }
+    }
+    finals.sort(compareFinals);
+    // Challengers: a stronger model at the same or a lower list price than the leader could be the
+    // better buy, but the neighbour rule never reaches it. Screen the cheapest few on the same samples.
+    const leader = finals[0]?.candidate;
+    const tried = new Set([...trials.map(trial => trial.candidate.model), ...unavailable.map(item => item.model)]);
+    const challengers = leader?.price == null ? [] : ordered
+        .filter(item => item.eci > leader.eci && item.price !== null && item.price <= leader.price && !tried.has(item.model))
+        .sort((a, b) => a.price - b.price || b.eci - a.eci)
+        .slice(0, input.challengers ?? 3);
+    for (const candidate of challengers) {
+        try {
+            const trial = await run("challenger", candidate, samples);
+            trials.push(trial);
+            if (trial.accepted)
+                finals.push(await run("final", candidate, input.dataset));
+        }
+        catch (error) {
+            if (!unavailableError(error))
+                return failure("challenger", candidate, error);
+            skip("challenger", candidate, error);
+        }
+    }
+    finals.sort(compareFinals);
+    return result("selected");
+}
+/** Models other than the pick, nearest by rank first; at equal distance the weaker one comes first. */
+export function outward(ordered, pick) {
+    const index = ordered.findIndex(item => item.model === pick.model);
+    if (index < 0)
+        throw new Error("The pick is not in the ranking");
+    const out = [];
+    for (let distance = 1; out.length < ordered.length - 1; distance++) {
+        if (ordered[index - distance])
+            out.push(ordered[index - distance]);
+        if (ordered[index + distance])
+            out.push(ordered[index + distance]);
+    }
+    return out;
+}
+/** Mean measured cost per item, or null when any item's cost is unknown. */
+export function meanCost(outcomes) {
+    if (!outcomes.length || outcomes.some(item => typeof item.costUsd !== "number"))
+        return null;
+    return outcomes.reduce((sum, item) => sum + item.costUsd, 0) / outcomes.length;
+}
+/** Most passes; then lower measured cost per item (list price only when a cost is unknown); then lower ECI. */
+export function compareFinals(a, b) {
+    if (b.passes !== a.passes)
+        return b.passes - a.passes;
+    const ca = meanCost(a.outcomes), cb = meanCost(b.outcomes);
+    if (ca !== null && cb !== null && ca !== cb)
+        return ca - cb;
+    if (ca === null || cb === null) {
+        const pa = a.candidate.price ?? Infinity, pb = b.candidate.price ?? Infinity;
+        if (pa !== pb)
+            return pa - pb;
+    }
+    return a.candidate.eci - b.candidate.eci;
+}

package/dist/scout/service.d.ts

@@ -0,0 +1,148 @@
+import { type LangfuseConnector } from "../connectors/langfuse/index.ts";
+import { type CandidateFilter, type RunnableModel } from "./candidates.ts";
+import { type ModelCaller, type PiTaskSpec } from "./evaluators/pi.ts";
+import { type ScoutReport } from "./pipeline.ts";
+import { type EvalItem } from "./search.ts";
+/** Everything Scout's tools need from the host: Pi's models and credentials, and where to write. */
+export interface ScoutHost {
+    caller: ModelCaller;
+    models: () => Promise<RunnableModel[]>;
+    cwd: string;
+    env: Record<string, string | undefined>;
+    log: (line: string) => void;
+    /** The model Scout itself runs on, used as the default judge. */
+    scoutModel?: string;
+}
+export declare function langfuseFromEnv(env: Record<string, string | undefined>): LangfuseConnector | null;
+export declare function connections(host: ScoutHost): Promise<{
+    providers: Record<string, number>;
+    scoutModel: string | null;
+    langfuse: {
+        connected: boolean;
+        project?: string;
+        baseUrl?: string;
+        datasets?: string[];
+        error?: string;
+    };
+}>;
+export declare function candidates(host: ScoutHost, filter: CandidateFilter): Promise<{
+    eci: {
+        source: string;
+        url: string;
+        attribution: string;
+        fetchedAt: string;
+        rows: number;
+        estimates: {
+            file: string;
+            rows: number;
+            provenance: string;
+        };
+    };
+    counts: {
+        published: number;
+        estimated: number;
+        calibrated: number;
+        unranked: number;
+    };
+    candidates: import("./candidates.ts").Candidate[];
+    unranked: string[];
+    dropped: {
+        model: string;
+        reason: string;
+    }[];
+}>;
+/** JSONL or a JSON array of `{ id?, input, expectedOutput, metadata? }`. */
+export declare function readDatasetFile(path: string): Promise<EvalItem[]>;
+export declare function previewDataset(host: ScoutHost, source: {
+    langfuseDataset?: string;
+    path?: string;
+}): Promise<{
+    size: number;
+    enoughForScout: boolean;
+    items: {
+        id: string;
+        input: string;
+        expectedOutput: string;
+    }[];
+}>;
+export declare function saveDataset(host: ScoutHost, name: string, items: {
+    id?: string;
+    input: unknown;
+    expectedOutput: unknown;
+    metadata?: Record<string, unknown>;
+}[]): Promise<{
+    path: string;
+    items: number;
+}>;
+export interface RunParams {
+    task: string;
+    langfuseDataset?: string;
+    datasetPath?: string;
+    /** Langfuse dataset name for a local file; defaults to the file name. */
+    datasetName?: string;
+    evaluator: ({
+        type: "pi";
+    } & PiTaskSpec) | {
+        type: "module";
+        path: string;
+    };
+    filter?: CandidateFilter;
+    sampleSize?: number;
+    finalists?: number;
+    repeats?: number;
+    judgeModel?: string;
+    /** Without Langfuse the run is local only, which the report says. */
+    requireLangfuse?: boolean;
+}
+/** A rough count of evaluations, so a host can ask before spending. */
+export declare function estimate(candidateCount: number, datasetSize: number, sampleSize?: number, finalists?: number, repeats?: number, challengers?: number): {
+    candidates: number;
+    probes: number;
+    evaluations: number;
+    note: string;
+};
+export declare function plan(host: ScoutHost, params: RunParams): Promise<{
+    plugin: import("./evaluators/module.ts").EvaluatorPlugin | null;
+    filter: CandidateFilter;
+    ranked: {
+        eci: {
+            source: string;
+            url: string;
+            attribution: string;
+            fetchedAt: string;
+            rows: number;
+            estimates: {
+                file: string;
+                rows: number;
+                provenance: string;
+            };
+        };
+        counts: {
+            published: number;
+            estimated: number;
+            calibrated: number;
+            unranked: number;
+        };
+        candidates: import("./candidates.ts").Candidate[];
+        unranked: string[];
+        dropped: {
+            model: string;
+            reason: string;
+        }[];
+    };
+    datasetSize: number;
+    estimate: {
+        candidates: number;
+        probes: number;
+        evaluations: number;
+        note: string;
+    };
+}>;
+export declare function run(host: ScoutHost, params: RunParams, signal?: AbortSignal): Promise<{
+    report: ScoutReport;
+    markdown: string;
+    files: {
+        json: string;
+        markdown: string;
+    };
+}>;

package/dist/scout/service.js

@@ -0,0 +1,143 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { connectionFromEnv, createLangfuseConnector } from "../connectors/langfuse/index.js";
+import { openRouterToolSupport, rankCandidates } from "./candidates.js";
+import { loadEci, loadEciEstimates } from "./eci.js";
+import { loadEvaluatorPlugin } from "./evaluators/module.js";
+import { piEvaluator } from "./evaluators/pi.js";
+import { runScout } from "./pipeline.js";
+import { reportMarkdown } from "./report.js";
+import { FINALISTS, SAMPLE_SIZE } from "./search.js";
+let eciCache = null;
+let toolCache = null;
+let estimateCache = null;
+const eci = () => (eciCache ??= loadEci().catch(error => { eciCache = null; throw error; }));
+const tools = () => (toolCache ??= openRouterToolSupport().catch(error => { toolCache = null; throw error; }));
+export function langfuseFromEnv(env) {
+    if (!env.LANGFUSE_PUBLIC_KEY || !env.LANGFUSE_SECRET_KEY)
+        return null;
+    return createLangfuseConnector(connectionFromEnv(env), { environment: "openmerit" });
+}
+export async function connections(host) {
+    const models = await host.models();
+    const providers = {};
+    for (const model of models)
+        providers[model.provider] = (providers[model.provider] ?? 0) + 1;
+    const lf = langfuseFromEnv(host.env);
+    let langfuse = { connected: false };
+    if (lf) {
+        try {
+            const project = await lf.check();
+            langfuse = { connected: true, project: project.projectName, baseUrl: project.baseUrl, datasets: (await lf.listDatasets()).map(item => item.name) };
+        }
+        catch (error) {
+            langfuse = { connected: false, error: error instanceof Error ? error.message : String(error) };
+        }
+    }
+    return { providers, scoutModel: host.scoutModel ?? null, langfuse };
+}
+export async function candidates(host, filter) {
+    const [source, support, bundled] = await Promise.all([eci(), filter.tools ? tools() : Promise.resolve(undefined), estimateCache ??= loadEciEstimates()]);
+    const ranked = rankCandidates(await host.models(), source.rows, { ...filter, ...(support ? { toolSupport: support } : {}) }, bundled.estimates);
+    const bySource = (name) => ranked.candidates.filter(item => item.eciSource === name).length;
+    return {
+        ...ranked,
+        eci: { source: source.source, url: source.url, attribution: source.attribution, fetchedAt: source.fetchedAt, rows: source.rows.length, estimates: { file: "data/eci-estimates.csv", rows: bundled.estimates.length, provenance: bundled.provenance } },
+        counts: { published: bySource("published"), estimated: bySource("estimated"), calibrated: bySource("calibrated"), unranked: ranked.unranked.length },
+    };
+}
+/** A stable ID from the item's input, so re-saving a file keeps its Langfuse item IDs. */
+const itemId = (input) => `item-${createHash("sha256").update(JSON.stringify(input)).digest("hex").slice(0, 12)}`;
+/** JSONL or a JSON array of `{ id?, input, expectedOutput, metadata? }`. */
+export async function readDatasetFile(path) {
+    const text = await readFile(path, "utf8");
+    const rows = text.trim().startsWith("[") ? JSON.parse(text) : text.split("\n").filter(line => line.trim()).map(line => JSON.parse(line));
+    return rows.map((row, index) => {
+        const item = row;
+        const expectedOutput = item.expectedOutput ?? item.expected_output;
+        if (item.input === undefined || expectedOutput === undefined)
+            throw new Error(`${path} item ${index + 1} needs input and expectedOutput`);
+        return {
+            id: typeof item.id === "string" && item.id ? item.id : itemId(item.input), input: item.input, expectedOutput,
+            ...(item.metadata && typeof item.metadata === "object" ? { metadata: item.metadata } : {}),
+        };
+    });
+}
+export async function previewDataset(host, source) {
+    let items;
+    if (source.langfuseDataset) {
+        const lf = langfuseFromEnv(host.env);
+        if (!lf)
+            throw new Error("Langfuse is not connected: set LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY and LANGFUSE_BASE_URL");
+        items = await lf.getDatasetItems(source.langfuseDataset);
+    }
+    else if (source.path)
+        items = await readDatasetFile(resolve(host.cwd, source.path));
+    else
+        throw new Error("Give a Langfuse dataset name or a dataset file path");
+    const clip = (value) => { const text = JSON.stringify(value); return text.length > 600 ? `${text.slice(0, 600)}…` : text; };
+    return { size: items.length, enoughForScout: items.length >= SAMPLE_SIZE, items: items.slice(0, 3).map(item => ({ id: item.id, input: clip(item.input), expectedOutput: clip(item.expectedOutput) })) };
+}
+export async function saveDataset(host, name, items) {
+    if (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(name))
+        throw new Error("Dataset names are lowercase letters, digits and hyphens");
+    const dir = join(host.cwd, ".openmerit", "datasets");
+    await mkdir(dir, { recursive: true });
+    const path = join(dir, `${name}.jsonl`);
+    const rows = items.map(item => ({ id: item.id ?? itemId(item.input), input: item.input, expectedOutput: item.expectedOutput, metadata: { ...item.metadata, labelledBy: "openmerit-scout-draft" } }));
+    await writeFile(path, `${rows.map(row => JSON.stringify(row)).join("\n")}\n`);
+    return { path, items: rows.length };
+}
+/** A rough count of evaluations, so a host can ask before spending. */
+export function estimate(candidateCount, datasetSize, sampleSize = SAMPLE_SIZE, finalists = FINALISTS, repeats = 3, challengers = 3) {
+    const probes = Math.ceil(Math.log2(candidateCount + 1));
+    const screens = Math.min(finalists - 1 + 2 + challengers, Math.max(0, candidateCount - probes));
+    return { candidates: candidateCount, probes, evaluations: (probes + screens) * sampleSize + Math.min(finalists + challengers, candidateCount) * datasetSize * repeats, note: "upper estimate: challengers run only when cheaper, stronger untested models exist" };
+}
+export async function plan(host, params) {
+    const plugin = params.evaluator.type === "module" ? await loadEvaluatorPlugin(params.evaluator.path, host.cwd) : null;
+    const filter = { ...params.filter, ...(plugin?.providers && !params.filter?.providers ? { providers: plugin.providers } : {}), ...(plugin?.tools ? { tools: true } : {}) };
+    const ranked = await candidates(host, filter);
+    const size = params.langfuseDataset || params.datasetPath ? (await previewDataset(host, { ...(params.langfuseDataset ? { langfuseDataset: params.langfuseDataset } : {}), ...(params.datasetPath ? { path: params.datasetPath } : {}) })).size : 0;
+    return { plugin, filter, ranked, datasetSize: size, estimate: estimate(ranked.candidates.length, size, params.sampleSize, params.finalists, params.repeats) };
+}
+export async function run(host, params, signal) {
+    const { plugin, ranked } = await plan(host, params);
+    const lf = langfuseFromEnv(host.env);
+    if (!lf && params.requireLangfuse !== false)
+        throw new Error("Scout picks the winner from Langfuse experiments: set LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY and LANGFUSE_BASE_URL (self-hosted works), or pass requireLangfuse: false for a local-only run");
+    const evaluate = plugin
+        ? await plugin.create({ env: host.env, log: host.log })
+        : piEvaluator(host.caller, params.evaluator, { ...(params.judgeModel ?? host.scoutModel ? { judgeModel: (params.judgeModel ?? host.scoutModel) } : {}) });
+    const dataset = params.langfuseDataset
+        ? { langfuseDataset: params.langfuseDataset }
+        : { local: { name: params.datasetName ?? params.datasetPath.split("/").at(-1).replace(/\.(jsonl|json)$/, ""), description: `Labelled items for: ${params.task.slice(0, 200)}`, items: await readDatasetFile(resolve(host.cwd, params.datasetPath)) } };
+    const credits = async () => {
+        if (!host.env.OPENROUTER_API_KEY)
+            return null;
+        try {
+            const response = await fetch("https://openrouter.ai/api/v1/credits", { headers: { authorization: `Bearer ${host.env.OPENROUTER_API_KEY}` }, signal: AbortSignal.timeout(15_000) });
+            return response.ok ? (await response.json()).data?.total_usage ?? null : null;
+        }
+        catch {
+            return null;
+        }
+    };
+    const before = await credits();
+    const report = await runScout({
+        task: params.task, dataset, candidates: ranked.candidates, evaluate, langfuse: lf, log: host.log,
+        ...(plugin?.passScore ? { passScore: plugin.passScore } : {}), ...(plugin?.role ? { role: plugin.role } : {}), ...(plugin?.concurrency ? { concurrency: plugin.concurrency } : {}),
+        ...(params.sampleSize ? { sampleSize: params.sampleSize } : {}), ...(params.finalists ? { finalists: params.finalists } : {}), ...(params.repeats ? { repeats: params.repeats } : {}), ...(signal ? { signal } : {}),
+    });
+    const after = before === null ? null : await credits();
+    if (before !== null && after !== null)
+        report.spend.openRouterCreditsUsd = after - before;
+    const dir = join(host.cwd, ".openmerit", "runs", report.runId);
+    await mkdir(dir, { recursive: true });
+    const markdown = reportMarkdown(report);
+    const files = { json: join(dir, "report.json"), markdown: join(dir, "report.md") };
+    await writeFile(files.json, `${JSON.stringify({ ...report, unranked: ranked.unranked, dropped: ranked.dropped }, null, 2)}\n`);
+    await writeFile(files.markdown, markdown);
+    return { report, markdown, files };
+}

package/examples/health-declaration-codes/dataset.jsonl

@@ -0,0 +1,14 @@
+{"id":"HD-01","input":"I have high blood pressure and take amlodipine every day.","expectedOutput":{"conditions":["I10"]}}
+{"id":"HD-02","input":"Type 2 diabetes for six years, on metformin. Also high cholesterol, on atorvastatin.","expectedOutput":{"conditions":["E11","E78"]}}
+{"id":"HD-03","input":"No medical conditions.","expectedOutput":{"conditions":[]}}
+{"id":"HD-04","input":"Asthma; I use a salbutamol inhaler a few times a month.","expectedOutput":{"conditions":["J45"]}}
+{"id":"HD-05","input":"Underactive thyroid, treated with levothyroxine.","expectedOutput":{"conditions":["E03"]}}
+{"id":"HD-06","input":"I broke my arm in 2019 and it healed completely. Nothing else.","expectedOutput":{"conditions":[]}}
+{"id":"HD-07","input":"Migraines about twice a month; I take sumatriptan when they start.","expectedOutput":{"conditions":["G43"]}}
+{"id":"HD-08","input":"Atrial fibrillation, on apixaban. I also have hypertension.","expectedOutput":{"conditions":["I48","I10"]}}
+{"id":"HD-09","input":"Psoriasis on both elbows; I use a steroid cream.","expectedOutput":{"conditions":["L40"]}}
+{"id":"HD-10","input":"COPD diagnosed last year. Ex-smoker. I use a tiotropium inhaler daily.","expectedOutput":{"conditions":["J44"]}}
+{"id":"HD-11","input":"Epilepsy. Seizure-free for three years on levetiracetam, which I still take.","expectedOutput":{"conditions":["G40"]}}
+{"id":"HD-12","input":"Acid reflux (GERD); I take omeprazole daily.","expectedOutput":{"conditions":["K21"]}}
+{"id":"HD-13","input":"أعاني من ارتفاع ضغط الدم ومرض السكري من النوع الثاني وأتناول الأدوية بانتظام.","expectedOutput":{"conditions":["I10","E11"]}}
+{"id":"HD-14","input":"I had gestational diabetes in 2021; it resolved after my baby was born and my sugar tests have been normal since. No other conditions.","expectedOutput":{"conditions":[]}}

package/examples/sahra-enrollment/evaluator.ts

@@ -0,0 +1,151 @@
+// Scout evaluator for the Sahra Health playground's enrollment agent (fictional data only).
+// It uses only what a customer exposes: the playground Worker's HTTP API and Sahra's own Langfuse
+// dataset `sahra-enrollments` (item input `{ application }`, expected output `{ decision,
+// declaredConditions, notDeclared? }`). Each item runs in a fresh, isolated ledger.
+//
+//   PLAYGROUND_URL (default http://localhost:8787), PLAYGROUND_TOKEN, LANGFUSE_* for measured cost.
+import { createHash, randomUUID } from "node:crypto";
+import { setTimeout as delay } from "node:timers/promises";
+// Copy this file into your project to adapt it; it has no runtime import from OpenMerit.
+// In your project, import the type from the package instead: `import type { EvaluatorPlugin } from "openmerit"`.
+import type { EvaluatorPlugin } from "../../src/scout/evaluators/module.ts";
+
+/** Scout skips a candidate that throws this (matched by name) as untested instead of stopping. */
+class CandidateUnavailable extends Error {
+  override readonly name = "CandidateUnavailable";
+}
+
+interface Expected { decision: "enrolled" | "declined"; declaredConditions: string[]; notDeclared?: string[] }
+interface State {
+  enrollment: { decision: "enrolled" | "declined" | "pending" } | null;
+  member: { declaredConditions: string[] } | null;
+  workflow: { status?: string; error?: { message?: string } } | null;
+}
+
+/** The agent ended without recording a decision. Under `wrangler dev` the workflow reports only the NonRetryableError. */
+const NO_DECISION = /ended without a decision|threw an? NonRetryableError/;
+
+interface Observation { type: string; level?: string; statusMessage?: string | null; endTime?: string | null; totalCost?: number | null; metadata?: { generations?: string | number } }
+
+/**
+ * Why a run ended without a decision, from the playground's own trace: a provider error on a model
+ * call (`402: …`, `404: …`) is not a quality verdict. Returns null when every model call succeeded,
+ * which makes it the agent's failure.
+ */
+export function providerError(observations: readonly Observation[]): { status: number; message: string } | null {
+  for (const row of observations) {
+    if (row.type !== "GENERATION" || row.level !== "ERROR") continue;
+    const message = row.statusMessage ?? "";
+    const status = Number(/^(\d{3})\b/.exec(message)?.[1] ?? /\b(4\d\d|5\d\d)\b/.exec(message)?.[1] ?? 0);
+    return { status, message: message.slice(0, 240) };
+  }
+  return null;
+}
+
+/** The playground scenario runner's checks: the decision, then declared condition prefixes. */
+export function enrollmentPass(expected: Expected, state: State): { pass: boolean; comment: string } {
+  const decision = state.enrollment?.decision ?? "none";
+  if (decision !== expected.decision) return { pass: false, comment: `decision ${decision}, expected ${expected.decision}` };
+  if (expected.decision !== "enrolled") return { pass: true, comment: `decision ${decision}` };
+  const declared = state.member?.declaredConditions ?? [];
+  const missing = expected.declaredConditions.filter(code => !declared.some(item => item.startsWith(code)));
+  const wrong = (expected.notDeclared ?? []).filter(code => declared.some(item => item.startsWith(code)));
+  return {
+    pass: !missing.length && !wrong.length,
+    comment: `declared ${declared.join(", ") || "none"}${missing.length ? `; missing ${missing.join(", ")}` : ""}${wrong.length ? `; should not declare ${wrong.join(", ")}` : ""}`,
+  };
+}
+
+const plugin: EvaluatorPlugin = {
+  name: "sahra-enrollment",
+  providers: ["openrouter"],
+  tools: true,
+  passScore: "enrollment-correct",
+  role: "enrollment",
+  concurrency: 6,
+  create({ env, log }) {
+    const url = (env.PLAYGROUND_URL ?? "http://localhost:8787").replace(/\/+$/, "");
+    const token = env.PLAYGROUND_TOKEN;
+    if (!token) throw new Error("Set PLAYGROUND_TOKEN to the playground Worker's token");
+    const langfuse = env.LANGFUSE_PUBLIC_KEY && env.LANGFUSE_SECRET_KEY
+      ? { base: (env.LANGFUSE_BASE_URL ?? "https://us.cloud.langfuse.com").replace(/\/+$/, ""), auth: `Basic ${btoa(`${env.LANGFUSE_PUBLIC_KEY}:${env.LANGFUSE_SECRET_KEY}`)}` }
+      : null;
+
+    /** The playground's own trace once its agent root has ended and every generation is ingested; null on timeout. */
+    async function trace(traceId: string, priced: boolean): Promise<Observation[] | null> {
+      if (!langfuse) return null;
+      const deadline = Date.now() + 180_000;
+      await delay(8_000);
+      while (Date.now() < deadline) {
+        const response = await fetch(`${langfuse.base}/api/public/v2/observations?traceId=${traceId}&fields=core,basic,usage,metadata&limit=100`, { headers: { authorization: langfuse.auth } });
+        if (response.ok) {
+          const { data = [] } = await response.json() as { data?: Observation[] };
+          const root = data.find(row => row.type === "AGENT");
+          const generations = data.filter(row => row.type === "GENERATION");
+          const expected = Number(root?.metadata?.generations ?? NaN);
+          if (root?.endTime && (generations.length === expected || root.level === "ERROR")
+            && (!priced || generations.every(row => typeof row.totalCost === "number"))) return data;
+        }
+        await delay(response.status === 429 ? 30_000 : 8_000);
+      }
+      return null;
+    }
+    const traceCost = async (traceId: string) => {
+      const data = await trace(traceId, true);
+      return data ? data.filter(row => row.type === "GENERATION").reduce((sum, row) => sum + (row.totalCost ?? 0), 0) : null;
+    };
+
+    return async (candidate, item, context) => {
+      if (candidate.provider !== "openrouter") throw new Error("The enrollment agent runs OpenRouter models only");
+      const input = item.input as { application?: { applicationId?: string } };
+      const application = input.application;
+      if (!application?.applicationId) throw new Error(`Dataset item ${item.id} has no application`);
+      const expected = item.expectedOutput as Expected;
+      // Outlast EnrollmentFlow's own 10-minute limit and one retry, so the workflow decides the outcome.
+      const signal = AbortSignal.any([AbortSignal.timeout(25 * 60_000), ...(context.signal ? [context.signal] : [])]);
+      const call = async <T>(method: string, path: string, body?: unknown): Promise<T> => {
+        const response = await fetch(`${url}${path}`, { method, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: body === undefined ? null : JSON.stringify(body), signal });
+        if (!response.ok) throw new Error(`Playground ${method} ${path.split("?")[0]} HTTP ${response.status}`);
+        return await response.json() as T;
+      };
+      const traceIdOf = (runId: string) => createHash("sha256").update(`trace:${application.applicationId}--${runId}/enrollment`).digest("hex").slice(0, 32);
+      for (let attempt = 1; ; attempt++) {
+        const runId = `scout-${randomUUID()}`;
+        log(`  ${context.stage} ${candidate.id} ${item.id} run=${runId}`);
+        await call("POST", "/admin/seed", { runId });
+        await call("POST", "/enrollments", { runId, application, models: { enrollment: { provider: "openrouter", id: candidate.id } } });
+        const path = `/enrollments/${encodeURIComponent(application.applicationId)}?runId=${encodeURIComponent(runId)}`;
+        let verdict: { pass: boolean; comment: string; output: unknown } | null = null;
+        while (!verdict) {
+          const state = await call<State>("GET", path);
+          const decision = state.enrollment?.decision;
+          if (["errored", "terminated"].includes(state.workflow?.status ?? "")) {
+            if (!NO_DECISION.test(state.workflow?.error?.message ?? "")) {
+              throw new Error(`Enrollment workflow ${state.workflow?.status} (run ${runId}): ${state.workflow?.error?.message ?? "no message"}; no quality verdict`);
+            }
+            // Only the trace can say whether a model call failed (provider error) or the agent gave up.
+            const observations = await trace(traceIdOf(runId), false);
+            if (!observations) throw new Error(`Cannot tell a provider error from an agent failure for run ${runId}: its Langfuse trace did not arrive`);
+            const failed = providerError(observations);
+            if (failed && [400, 402, 404, 413, 422].includes(failed.status)) throw new CandidateUnavailable(`${failed.status} from the provider (run ${runId}): ${failed.message}`);
+            if (failed && (failed.status === 429 || failed.status >= 500 || failed.status === 0) && attempt < 2) { log(`  retrying ${candidate.id} ${item.id} after ${failed.status || "an unclassified provider error"}`); break; }
+            if (failed) throw new Error(`Provider error ${failed.status} for ${candidate.model} (run ${runId}): ${failed.message}`);
+            verdict = { pass: false, comment: "the agent ended without recording a decision", output: { decision: null } };
+          } else if (decision && decision !== "pending") {
+            verdict = { ...enrollmentPass(expected, state), output: { decision, declaredConditions: state.member?.declaredConditions ?? [] } };
+          } else if (state.workflow?.status === "complete") {
+            throw new Error(`Enrollment workflow completed without a decision (run ${runId})`);
+          } else await delay(1500, undefined, { signal });
+        }
+        if (!verdict) continue;
+        // The playground's trace ID for this agent run (see its tracing.ts): sha256("trace:<app>--<runId>/enrollment").
+        const traceId = traceIdOf(runId);
+        // Cost decides ties between finalists, so wait for it there; screens skip the ingestion wait.
+        const costUsd = context.stage === "final" ? await traceCost(traceId) : null;
+        return { ...verdict, output: { ...verdict.output as object, playgroundRunId: runId, playgroundTraceId: traceId }, costUsd };
+      }
+    };
+  },
+};
+
+export default plugin;