openmerit 0.4.1 → 0.4.2

package/dist/scout/pipeline.js

@@ -1,157 +0,0 @@
-import { randomUUID } from "node:crypto";
-import { compareFinals, meanCost, scoutSearch } from "./search.js";
-const slug = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60);
-/**
- * Scout end to end: dataset → binary search over ECI-ranked candidates → finals on every item,
- * each probe and final recorded as a Langfuse experiment → the winner from Langfuse's results.
- */
-export async function runScout(options) {
-    const log = options.log ?? (() => { });
-    const runId = `scout-${new Date().toISOString().replace(/[:.]/g, "-")}-${randomUUID().slice(0, 4)}`;
-    const passScore = options.passScore ?? "correct";
-    const role = options.role ?? "candidate";
-    const lf = options.langfuse;
-    let items;
-    let datasetName;
-    let datasetId = null;
-    if (options.dataset.langfuseDataset) {
-        if (!lf)
-            throw new Error("A Langfuse dataset needs a Langfuse connection (LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, LANGFUSE_BASE_URL)");
-        datasetName = options.dataset.langfuseDataset;
-        items = (await lf.getDatasetItems(datasetName)).map(item => ({ id: item.id, input: item.input, expectedOutput: item.expectedOutput, metadata: item.metadata }));
-        datasetId = (await lf.getDataset(datasetName)).id;
-    }
-    else if (options.dataset.local) {
-        datasetName = options.dataset.local.name;
-        items = options.dataset.local.items;
-        if (lf) {
-            const pushed = await lf.ensureDataset({
-                name: datasetName, ...(options.dataset.local.description ? { description: options.dataset.local.description } : {}),
-                metadata: { source: "openmerit-scout" },
-                items: items.map(item => ({ id: item.id, input: item.input, expectedOutput: item.expectedOutput, ...(item.metadata ? { metadata: item.metadata } : {}) })),
-            });
-            datasetId = pushed.datasetId;
-        }
-    }
-    else
-        throw new Error("Scout needs a labelled dataset: a Langfuse dataset name or local items");
-    log(`Dataset ${datasetName}: ${items.length} items${datasetId ? ` (Langfuse ${datasetId})` : " (local only)"}`);
-    log(`${options.candidates.length} ECI-ranked candidates · run ${runId}`);
-    const urls = new Map();
-    const project = lf ? await lf.check() : null;
-    const onTrial = async (trial) => {
-        if (!lf || !datasetId)
-            return;
-        const stage = trial.stage === "final" ? "final" : "screen";
-        const recordings = [];
-        for (let repeat = 1; repeat <= trial.runs; repeat++) {
-            const suffix = trial.runs > 1 ? `-r${repeat}` : "";
-            const experimentId = `${runId}-${stage}${suffix}-${slug(trial.candidate.model)}`;
-            const recorded = await lf.recordExperimentItems({
-                datasetId, experimentId, experimentName: `scout ${stage}${trial.runs > 1 ? ` run ${repeat}/${trial.runs}` : ""} ${trial.candidate.model} ${runId}`, traceName: "openmerit-scout-eval",
-                metadata: { [`model_${role}`]: trial.candidate.model, scout_run: runId, stage, ...(trial.runs > 1 ? { repeat, repeats: trial.runs } : {}), eci: trial.candidate.eci, eci_source: trial.candidate.eciSource, task: options.task.slice(0, 200) },
-                items: trial.outcomes.filter(outcome => (outcome.repeat ?? 1) === repeat).map(outcome => {
-                    const item = items.find(entry => entry.id === outcome.itemId);
-                    return {
-                        itemId: outcome.itemId, model: trial.candidate.model, input: item.input, output: outcome.output ?? null,
-                        startMs: outcome.startMs ?? Date.now(), endMs: outcome.endMs ?? Date.now(),
-                        ...(outcome.usage ? { usage: outcome.usage } : {}), ...(typeof outcome.costUsd === "number" ? { costUsd: outcome.costUsd } : {}),
-                        metadata: { stage: trial.stage, scout_run: runId, eci: trial.candidate.eci, ...(trial.runs > 1 ? { repeat } : {}) },
-                        scores: [{ name: passScore, value: outcome.pass, dataType: "BOOLEAN", ...(outcome.comment ? { comment: outcome.comment } : {}) }],
-                    };
-                }),
-            });
-            recordings.push({ experimentId, url: recorded.url });
-        }
-        urls.set(trial, recordings);
-    };
-    const search = await scoutSearch({
-        dataset: items, candidates: options.candidates, evaluate: options.evaluate, onTrial, log,
-        ...(options.sampleSize ? { sampleSize: options.sampleSize } : {}), ...(options.finalists ? { finalists: options.finalists } : {}),
-        ...(options.challengers !== undefined ? { challengers: options.challengers } : {}),
-        ...(options.repeats !== undefined ? { repeats: options.repeats } : {}),
-        ...(options.concurrency ? { concurrency: options.concurrency } : {}), ...(options.pickIndex ? { pickIndex: options.pickIndex } : {}),
-        ...(options.signal ? { signal: options.signal } : {}),
-    });
-    // The decision: read every final back from Langfuse, so the recommendation rests on the recorded experiment.
-    const rule = { score: passScore, equals: true };
-    const finals = [];
-    for (const trial of search.finals) {
-        const recorded = urls.get(trial) ?? [];
-        const readbacks = [];
-        if (lf && recorded.length) {
-            log(`Reading ${trial.candidate.model} back from Langfuse (${recorded.length} experiments)…`);
-            for (const { experimentId } of recorded) {
-                readbacks.push(await lf.waitForExperiment({ experimentId, pass: rule, includeCost: true, since: new Date(Date.now() - 86_400_000) }, { expectItems: items.length, timeoutMs: options.langfuseTimeoutMs ?? 300_000 }));
-            }
-        }
-        const verified = readbacks.length > 0 && readbacks.length === recorded.length;
-        const results = verified
-            ? readbacks.flatMap(readback => readback.items.map(item => ({ itemId: item.itemId, pass: item.passed })))
-            : trial.outcomes.map(outcome => ({ itemId: outcome.itemId, pass: outcome.pass }));
-        const passesByItem = {};
-        for (const item of items)
-            passesByItem[item.id] = 0;
-        for (const result of results)
-            if (result.pass)
-                passesByItem[result.itemId] = (passesByItem[result.itemId] ?? 0) + 1;
-        const langfuseCost = verified && readbacks.every(readback => readback.costUsd != null) ? readbacks.reduce((sum, readback) => sum + readback.costUsd, 0) : null;
-        finals.push({
-            model: trial.candidate.model, eci: trial.candidate.eci, eciSource: trial.candidate.eciSource, listPrice: trial.candidate.price,
-            passes: results.filter(result => result.pass).length, total: results.length, runs: trial.runs, passesByItem,
-            costPerItemUsd: langfuseCost !== null && results.length ? langfuseCost / results.length : meanCost(trial.outcomes),
-            localPasses: trial.passes, verifiedInLangfuse: verified, missingScores: readbacks.flatMap(readback => readback.missingScores),
-            experimentIds: recorded.map(item => item.experimentId), experimentId: recorded[0]?.experimentId ?? null, experimentUrl: recorded[0]?.url ?? null,
-            failedItems: Object.entries(passesByItem).filter(([, passes]) => passes < trial.runs).map(([id]) => id),
-        });
-    }
-    const asTrial = (verdict) => {
-        const trial = search.finals.find(item => item.candidate.model === verdict.model);
-        return { ...trial, passes: verdict.passes, outcomes: trial.outcomes.map(outcome => ({ ...outcome, costUsd: verdict.costPerItemUsd })) };
-    };
-    finals.sort((a, b) => compareFinals(asTrial(a), asTrial(b)));
-    // A stopped run has no recommendation: its finals are incomplete and listed only as evidence.
-    const [recommendation = null, runnerUp = null] = search.status === "selected" ? finals : [];
-    const all = [...search.trials, ...search.finals].flatMap(trial => trial.outcomes);
-    const limitations = [
-        search.limitation,
-        "ECI is Epoch's published capability index. Models without a published score use ScoutBM's estimated or calibrated ECI from the bundled CSV, which carries 2–4 points of error; models with neither are not candidates (see `unranked`).",
-        ...(lf ? [] : ["Langfuse is not connected: results were not recorded or verified in Langfuse."]),
-        ...(search.shortfall ? [search.shortfall] : []),
-        ...(search.status === "stopped" && finals.length ? [`The run stopped, so the ${finals.length} finals shown are partial and no model is recommended.`] : []),
-        ...(search.unavailable.length ? [`${search.unavailable.length} candidates could not run (provider rejected the model or the balance did not cover it) and are untested: ${search.unavailable.map(item => item.model).join(", ")}.`] : []),
-        ...(finals.some(item => item.localPasses !== item.passes) ? ["Langfuse and local pass counts differ for at least one finalist; Langfuse's count was used."] : []),
-    ];
-    return {
-        scout: "OpenMerit Scout", runId, task: options.task, createdAt: new Date().toISOString(),
-        dataset: { name: datasetName, source: options.dataset.langfuseDataset ? "langfuse" : "local", size: items.length, datasetId, url: project && datasetId ? `${lf.baseUrl}/project/${project.projectId}/datasets/${datasetId}` : null },
-        candidates: options.candidates.length,
-        search: {
-            status: search.status, ...(search.error ? { error: search.error } : {}), criterion: search.criterion, sampleIds: search.sampleIds, selected: search.selected, unavailable: search.unavailable,
-            finalRule: search.finalRule, limitation: search.limitation, ...(search.shortfall ? { shortfall: search.shortfall } : {}),
-            trials: search.trials.map(trial => ({
-                stage: trial.stage, model: trial.candidate.model, eci: trial.candidate.eci, eciSource: trial.candidate.eciSource, passes: trial.passes, total: trial.outcomes.length,
-                accepted: trial.accepted, costUsd: meanCost(trial.outcomes) === null ? null : meanCost(trial.outcomes) * trial.outcomes.length, experimentUrl: urls.get(trial)?.[0]?.url ?? null,
-            })),
-        },
-        finals, recommendation, runnerUp,
-        why: explain(search, recommendation, runnerUp),
-        spend: { measuredUsd: all.reduce((sum, item) => sum + (item.costUsd ?? 0), 0), unknownCostItems: all.filter(item => typeof item.costUsd !== "number").length, evaluations: all.length },
-        langfuse: lf && project ? { baseUrl: lf.baseUrl, project: project.projectName, experiments: [...urls.values()].reduce((sum, list) => sum + list.length, 0) } : null,
-        limitations,
-    };
-}
-function explain(search, best, next) {
-    if (search.status === "stopped")
-        return `The search stopped without a verdict: ${search.error}`;
-    if (search.status === "no_passing_model")
-        return `No candidate passed all ${search.sampleIds.length} sampled items, so Scout recommends none. Improve the prompt or harness, or check the labels.`;
-    if (!best)
-        return "No finalist was evaluated.";
-    const cost = (value) => (value === null ? "unknown cost" : `$${value.toFixed(5)}/item`);
-    const head = `${best.model} passed ${best.passes}/${best.total} item runs (${best.runs} per item) in Langfuse at ${cost(best.costPerItemUsd)} (ECI ${best.eci.toFixed(1)}).`;
-    if (!next)
-        return head;
-    const tie = next.passes === best.passes;
-    return `${head} ${tie ? "Tied on passes with" : "Ahead of"} ${next.model} (${next.passes}/${next.total}, ${cost(next.costPerItemUsd)})${tie ? `; ${best.costPerItemUsd !== null && next.costPerItemUsd !== null && best.costPerItemUsd !== next.costPerItemUsd ? "lower measured cost" : "lower ECI"} breaks the tie` : ""}.`;
-}

package/dist/scout/report.d.ts

@@ -1,3 +0,0 @@
-import type { ScoutReport } from "./pipeline.js";
-/** The decision record as Markdown: no keys, no raw item contents. */
-export declare function reportMarkdown(report: ScoutReport): string;

package/dist/scout/report.js

@@ -1,42 +0,0 @@
-const usd = (value, digits = 5) => (value === null ? "unknown" : `$${value.toFixed(digits)}`);
-const link = (label, url) => (url ? `[${label}](${url})` : label);
-const eciCell = (eci, source) => `${eci.toFixed(1)}${source === "published" ? "" : ` (${source.slice(0, 4)}.)`}`;
-/** The decision record as Markdown: no keys, no raw item contents. */
-export function reportMarkdown(report) {
-    const best = report.recommendation;
-    const lines = [
-        `# Scout recommendation: ${best ? `\`${best.model}\`` : "none"}`,
-        "",
-        `- Task: ${report.task.replace(/\s+/g, " ").slice(0, 500)}`,
-        `- Run: \`${report.runId}\`, ${report.createdAt}`,
-        `- Dataset: ${link(`\`${report.dataset.name}\``, report.dataset.url)} (${report.dataset.source}, ${report.dataset.size} labelled items)`,
-        `- Candidates: ${report.candidates} runnable models with an Epoch Capabilities Index (published, or ScoutBM's estimate: "est."/"cali.")`,
-        `- Langfuse: ${report.langfuse ? `${report.langfuse.project} at ${report.langfuse.baseUrl}, ${report.langfuse.experiments} experiments recorded` : "not connected"}`,
-        `- Spend: ${usd(report.spend.measuredUsd, 4)} measured over ${report.spend.evaluations} evaluations${report.spend.unknownCostItems ? ` (${report.spend.unknownCostItems} without a cost)` : ""}${report.spend.openRouterCreditsUsd !== undefined ? `; OpenRouter account credit used during the run: ${usd(report.spend.openRouterCreditsUsd, 4)}` : ""}`,
-        "",
-        `**Why:** ${report.why}`,
-        "",
-        `## Finals (every item ${report.finals[0]?.runs ?? 1} times, one Langfuse experiment per run, read back from Langfuse)`,
-        "",
-        "| Rank | Model | Passes | Cost per item run | ECI | List price $/M in+out | Verified in Langfuse | Failed items |",
-        "|---|---|---|---|---|---|---|---|",
-        ...report.finals.map((item, index) => `| ${index + 1} | ${link(`\`${item.model}\``, item.experimentUrl)} | ${item.passes}/${item.total} | ${usd(item.costPerItemUsd)} | ${eciCell(item.eci, item.eciSource)} | ${item.listPrice === null ? "unknown" : item.listPrice.toFixed(2)} | ${item.verifiedInLangfuse ? "yes" : "no"}${item.missingScores.length ? ` (${item.missingScores.length} unscored)` : ""} | ${item.failedItems.map(id => `${id} (${item.passesByItem[id] ?? 0}/${item.runs})`).join(", ") || "none"} |`),
-        "",
-        `## Binary search (${report.search.criterion}; samples ${report.search.sampleIds.join(", ")})`,
-        "",
-        `Status: **${report.search.status}**${report.search.error ? ` — ${report.search.error}` : ""}. Search pick: ${report.search.selected ? `\`${report.search.selected.model}\`` : "none"}.`,
-        "",
-        "| # | Stage | Model | ECI | Passes | Verdict | Cost |",
-        "|---|---|---|---|---|---|---|",
-        ...report.search.trials.map((trial, index) => `| ${index + 1} | ${trial.stage} | ${link(`\`${trial.model}\``, trial.experimentUrl)} | ${eciCell(trial.eci, trial.eciSource)} | ${trial.passes}/${trial.total} | ${trial.accepted ? "pass" : "fail"} | ${usd(trial.costUsd, 4)} |`),
-        ...(report.search.unavailable.length ? ["", "Skipped (could not run, untested):", "", ...report.search.unavailable.map(item => `- \`${item.model}\` (${item.stage}): ${item.reason.slice(0, 200)}`)] : []),
-        "",
-        `Rule: ${report.search.finalRule}`,
-        "",
-        "## Limitations",
-        "",
-        ...report.limitations.map(item => `- ${item}`),
-        "",
-    ];
-    return lines.join("\n");
-}

package/dist/scout/service.d.ts

@@ -1,148 +0,0 @@
-import { type LangfuseConnector } from "../connectors/langfuse/index.js";
-import { type CandidateFilter, type RunnableModel } from "./candidates.js";
-import { type ModelCaller, type PiTaskSpec } from "./evaluators/pi.js";
-import { type ScoutReport } from "./pipeline.js";
-import { type EvalItem } from "./search.js";
-/** Everything Scout's tools need from the host: Pi's models and credentials, and where to write. */
-export interface ScoutHost {
-    caller: ModelCaller;
-    models: () => Promise<RunnableModel[]>;
-    cwd: string;
-    env: Record<string, string | undefined>;
-    log: (line: string) => void;
-    /** The model Scout itself runs on, used as the default judge. */
-    scoutModel?: string;
-}
-export declare function langfuseFromEnv(env: Record<string, string | undefined>): LangfuseConnector | null;
-export declare function connections(host: ScoutHost): Promise<{
-    providers: Record<string, number>;
-    scoutModel: string | null;
-    langfuse: {
-        connected: boolean;
-        project?: string;
-        baseUrl?: string;
-        datasets?: string[];
-        error?: string;
-    };
-}>;
-export declare function candidates(host: ScoutHost, filter: CandidateFilter): Promise<{
-    eci: {
-        source: string;
-        url: string;
-        attribution: string;
-        fetchedAt: string;
-        rows: number;
-        estimates: {
-            file: string;
-            rows: number;
-            provenance: string;
-        };
-    };
-    counts: {
-        published: number;
-        estimated: number;
-        calibrated: number;
-        unranked: number;
-    };
-    candidates: import("./candidates.js").Candidate[];
-    unranked: string[];
-    dropped: {
-        model: string;
-        reason: string;
-    }[];
-}>;
-/** JSONL or a JSON array of `{ id?, input, expectedOutput, metadata? }`. */
-export declare function readDatasetFile(path: string): Promise<EvalItem[]>;
-export declare function previewDataset(host: ScoutHost, source: {
-    langfuseDataset?: string;
-    path?: string;
-}): Promise<{
-    size: number;
-    enoughForScout: boolean;
-    items: {
-        id: string;
-        input: string;
-        expectedOutput: string;
-    }[];
-}>;
-export declare function saveDataset(host: ScoutHost, name: string, items: {
-    id?: string;
-    input: unknown;
-    expectedOutput: unknown;
-    metadata?: Record<string, unknown>;
-}[]): Promise<{
-    path: string;
-    items: number;
-}>;
-export interface RunParams {
-    task: string;
-    langfuseDataset?: string;
-    datasetPath?: string;
-    /** Langfuse dataset name for a local file; defaults to the file name. */
-    datasetName?: string;
-    evaluator: ({
-        type: "pi";
-    } & PiTaskSpec) | {
-        type: "module";
-        path: string;
-    };
-    filter?: CandidateFilter;
-    sampleSize?: number;
-    finalists?: number;
-    repeats?: number;
-    judgeModel?: string;
-    /** Without Langfuse the run is local only, which the report says. */
-    requireLangfuse?: boolean;
-}
-/** A rough count of evaluations, so a host can ask before spending. */
-export declare function estimate(candidateCount: number, datasetSize: number, sampleSize?: number, finalists?: number, repeats?: number, challengers?: number): {
-    candidates: number;
-    probes: number;
-    evaluations: number;
-    note: string;
-};
-export declare function plan(host: ScoutHost, params: RunParams): Promise<{
-    plugin: import("./evaluators/module.js").EvaluatorPlugin | null;
-    filter: CandidateFilter;
-    ranked: {
-        eci: {
-            source: string;
-            url: string;
-            attribution: string;
-            fetchedAt: string;
-            rows: number;
-            estimates: {
-                file: string;
-                rows: number;
-                provenance: string;
-            };
-        };
-        counts: {
-            published: number;
-            estimated: number;
-            calibrated: number;
-            unranked: number;
-        };
-        candidates: import("./candidates.js").Candidate[];
-        unranked: string[];
-        dropped: {
-            model: string;
-            reason: string;
-        }[];
-    };
-    datasetSize: number;
-    estimate: {
-        candidates: number;
-        probes: number;
-        evaluations: number;
-        note: string;
-    };
-}>;
-export declare function run(host: ScoutHost, params: RunParams, signal?: AbortSignal): Promise<{
-    report: ScoutReport;
-    markdown: string;
-    files: {
-        json: string;
-        markdown: string;
-    };
-}>;

package/dist/scout/service.js

@@ -1,143 +0,0 @@
-import { createHash } from "node:crypto";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
-import { join, resolve } from "node:path";
-import { connectionFromEnv, createLangfuseConnector } from "../connectors/langfuse/index.js";
-import { openRouterToolSupport, rankCandidates } from "./candidates.js";
-import { loadEci, loadEciEstimates } from "./eci.js";
-import { loadEvaluatorPlugin } from "./evaluators/module.js";
-import { piEvaluator } from "./evaluators/pi.js";
-import { runScout } from "./pipeline.js";
-import { reportMarkdown } from "./report.js";
-import { FINALISTS, SAMPLE_SIZE } from "./search.js";
-let eciCache = null;
-let toolCache = null;
-let estimateCache = null;
-const eci = () => (eciCache ??= loadEci().catch(error => { eciCache = null; throw error; }));
-const tools = () => (toolCache ??= openRouterToolSupport().catch(error => { toolCache = null; throw error; }));
-export function langfuseFromEnv(env) {
-    if (!env.LANGFUSE_PUBLIC_KEY || !env.LANGFUSE_SECRET_KEY)
-        return null;
-    return createLangfuseConnector(connectionFromEnv(env), { environment: "openmerit" });
-}
-export async function connections(host) {
-    const models = await host.models();
-    const providers = {};
-    for (const model of models)
-        providers[model.provider] = (providers[model.provider] ?? 0) + 1;
-    const lf = langfuseFromEnv(host.env);
-    let langfuse = { connected: false };
-    if (lf) {
-        try {
-            const project = await lf.check();
-            langfuse = { connected: true, project: project.projectName, baseUrl: project.baseUrl, datasets: (await lf.listDatasets()).map(item => item.name) };
-        }
-        catch (error) {
-            langfuse = { connected: false, error: error instanceof Error ? error.message : String(error) };
-        }
-    }
-    return { providers, scoutModel: host.scoutModel ?? null, langfuse };
-}
-export async function candidates(host, filter) {
-    const [source, support, bundled] = await Promise.all([eci(), filter.tools ? tools() : Promise.resolve(undefined), estimateCache ??= loadEciEstimates()]);
-    const ranked = rankCandidates(await host.models(), source.rows, { ...filter, ...(support ? { toolSupport: support } : {}) }, bundled.estimates);
-    const bySource = (name) => ranked.candidates.filter(item => item.eciSource === name).length;
-    return {
-        ...ranked,
-        eci: { source: source.source, url: source.url, attribution: source.attribution, fetchedAt: source.fetchedAt, rows: source.rows.length, estimates: { file: "data/eci-estimates.csv", rows: bundled.estimates.length, provenance: bundled.provenance } },
-        counts: { published: bySource("published"), estimated: bySource("estimated"), calibrated: bySource("calibrated"), unranked: ranked.unranked.length },
-    };
-}
-/** A stable ID from the item's input, so re-saving a file keeps its Langfuse item IDs. */
-const itemId = (input) => `item-${createHash("sha256").update(JSON.stringify(input)).digest("hex").slice(0, 12)}`;
-/** JSONL or a JSON array of `{ id?, input, expectedOutput, metadata? }`. */
-export async function readDatasetFile(path) {
-    const text = await readFile(path, "utf8");
-    const rows = text.trim().startsWith("[") ? JSON.parse(text) : text.split("\n").filter(line => line.trim()).map(line => JSON.parse(line));
-    return rows.map((row, index) => {
-        const item = row;
-        const expectedOutput = item.expectedOutput ?? item.expected_output;
-        if (item.input === undefined || expectedOutput === undefined)
-            throw new Error(`${path} item ${index + 1} needs input and expectedOutput`);
-        return {
-            id: typeof item.id === "string" && item.id ? item.id : itemId(item.input), input: item.input, expectedOutput,
-            ...(item.metadata && typeof item.metadata === "object" ? { metadata: item.metadata } : {}),
-        };
-    });
-}
-export async function previewDataset(host, source) {
-    let items;
-    if (source.langfuseDataset) {
-        const lf = langfuseFromEnv(host.env);
-        if (!lf)
-            throw new Error("Langfuse is not connected: set LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY and LANGFUSE_BASE_URL");
-        items = await lf.getDatasetItems(source.langfuseDataset);
-    }
-    else if (source.path)
-        items = await readDatasetFile(resolve(host.cwd, source.path));
-    else
-        throw new Error("Give a Langfuse dataset name or a dataset file path");
-    const clip = (value) => { const text = JSON.stringify(value); return text.length > 600 ? `${text.slice(0, 600)}…` : text; };
-    return { size: items.length, enoughForScout: items.length >= SAMPLE_SIZE, items: items.slice(0, 3).map(item => ({ id: item.id, input: clip(item.input), expectedOutput: clip(item.expectedOutput) })) };
-}
-export async function saveDataset(host, name, items) {
-    if (!/^[a-z0-9][a-z0-9-]{0,63}$/.test(name))
-        throw new Error("Dataset names are lowercase letters, digits and hyphens");
-    const dir = join(host.cwd, ".openmerit", "datasets");
-    await mkdir(dir, { recursive: true });
-    const path = join(dir, `${name}.jsonl`);
-    const rows = items.map(item => ({ id: item.id ?? itemId(item.input), input: item.input, expectedOutput: item.expectedOutput, metadata: { ...item.metadata, labelledBy: "openmerit-scout-draft" } }));
-    await writeFile(path, `${rows.map(row => JSON.stringify(row)).join("\n")}\n`);
-    return { path, items: rows.length };
-}
-/** A rough count of evaluations, so a host can ask before spending. */
-export function estimate(candidateCount, datasetSize, sampleSize = SAMPLE_SIZE, finalists = FINALISTS, repeats = 3, challengers = 3) {
-    const probes = Math.ceil(Math.log2(candidateCount + 1));
-    const screens = Math.min(finalists - 1 + 2 + challengers, Math.max(0, candidateCount - probes));
-    return { candidates: candidateCount, probes, evaluations: (probes + screens) * sampleSize + Math.min(finalists + challengers, candidateCount) * datasetSize * repeats, note: "upper estimate: challengers run only when cheaper, stronger untested models exist" };
-}
-export async function plan(host, params) {
-    const plugin = params.evaluator.type === "module" ? await loadEvaluatorPlugin(params.evaluator.path, host.cwd) : null;
-    const filter = { ...params.filter, ...(plugin?.providers && !params.filter?.providers ? { providers: plugin.providers } : {}), ...(plugin?.tools ? { tools: true } : {}) };
-    const ranked = await candidates(host, filter);
-    const size = params.langfuseDataset || params.datasetPath ? (await previewDataset(host, { ...(params.langfuseDataset ? { langfuseDataset: params.langfuseDataset } : {}), ...(params.datasetPath ? { path: params.datasetPath } : {}) })).size : 0;
-    return { plugin, filter, ranked, datasetSize: size, estimate: estimate(ranked.candidates.length, size, params.sampleSize, params.finalists, params.repeats) };
-}
-export async function run(host, params, signal) {
-    const { plugin, ranked } = await plan(host, params);
-    const lf = langfuseFromEnv(host.env);
-    if (!lf && params.requireLangfuse !== false)
-        throw new Error("Scout picks the winner from Langfuse experiments: set LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY and LANGFUSE_BASE_URL (self-hosted works), or pass requireLangfuse: false for a local-only run");
-    const evaluate = plugin
-        ? await plugin.create({ env: host.env, log: host.log })
-        : piEvaluator(host.caller, params.evaluator, { ...(params.judgeModel ?? host.scoutModel ? { judgeModel: (params.judgeModel ?? host.scoutModel) } : {}) });
-    const dataset = params.langfuseDataset
-        ? { langfuseDataset: params.langfuseDataset }
-        : { local: { name: params.datasetName ?? params.datasetPath.split("/").at(-1).replace(/\.(jsonl|json)$/, ""), description: `Labelled items for: ${params.task.slice(0, 200)}`, items: await readDatasetFile(resolve(host.cwd, params.datasetPath)) } };
-    const credits = async () => {
-        if (!host.env.OPENROUTER_API_KEY)
-            return null;
-        try {
-            const response = await fetch("https://openrouter.ai/api/v1/credits", { headers: { authorization: `Bearer ${host.env.OPENROUTER_API_KEY}` }, signal: AbortSignal.timeout(15_000) });
-            return response.ok ? (await response.json()).data?.total_usage ?? null : null;
-        }
-        catch {
-            return null;
-        }
-    };
-    const before = await credits();
-    const report = await runScout({
-        task: params.task, dataset, candidates: ranked.candidates, evaluate, langfuse: lf, log: host.log,
-        ...(plugin?.passScore ? { passScore: plugin.passScore } : {}), ...(plugin?.role ? { role: plugin.role } : {}), ...(plugin?.concurrency ? { concurrency: plugin.concurrency } : {}),
-        ...(params.sampleSize ? { sampleSize: params.sampleSize } : {}), ...(params.finalists ? { finalists: params.finalists } : {}), ...(params.repeats ? { repeats: params.repeats } : {}), ...(signal ? { signal } : {}),
-    });
-    const after = before === null ? null : await credits();
-    if (before !== null && after !== null)
-        report.spend.openRouterCreditsUsd = after - before;
-    const dir = join(host.cwd, ".openmerit", "runs", report.runId);
-    await mkdir(dir, { recursive: true });
-    const markdown = reportMarkdown(report);
-    const files = { json: join(dir, "report.json"), markdown: join(dir, "report.md") };
-    await writeFile(files.json, `${JSON.stringify({ ...report, unranked: ranked.unranked, dropped: ranked.dropped }, null, 2)}\n`);
-    await writeFile(files.markdown, markdown);
-    return { report, markdown, files };
-}

package/examples/health-declaration-codes/dataset.jsonl

@@ -1,14 +0,0 @@
-{"id":"HD-01","input":"I have high blood pressure and take amlodipine every day.","expectedOutput":{"conditions":["I10"]}}
-{"id":"HD-02","input":"Type 2 diabetes for six years, on metformin. Also high cholesterol, on atorvastatin.","expectedOutput":{"conditions":["E11","E78"]}}
-{"id":"HD-03","input":"No medical conditions.","expectedOutput":{"conditions":[]}}
-{"id":"HD-04","input":"Asthma; I use a salbutamol inhaler a few times a month.","expectedOutput":{"conditions":["J45"]}}
-{"id":"HD-05","input":"Underactive thyroid, treated with levothyroxine.","expectedOutput":{"conditions":["E03"]}}
-{"id":"HD-06","input":"I broke my arm in 2019 and it healed completely. Nothing else.","expectedOutput":{"conditions":[]}}
-{"id":"HD-07","input":"Migraines about twice a month; I take sumatriptan when they start.","expectedOutput":{"conditions":["G43"]}}
-{"id":"HD-08","input":"Atrial fibrillation, on apixaban. I also have hypertension.","expectedOutput":{"conditions":["I48","I10"]}}
-{"id":"HD-09","input":"Psoriasis on both elbows; I use a steroid cream.","expectedOutput":{"conditions":["L40"]}}
-{"id":"HD-10","input":"COPD diagnosed last year. Ex-smoker. I use a tiotropium inhaler daily.","expectedOutput":{"conditions":["J44"]}}
-{"id":"HD-11","input":"Epilepsy. Seizure-free for three years on levetiracetam, which I still take.","expectedOutput":{"conditions":["G40"]}}
-{"id":"HD-12","input":"Acid reflux (GERD); I take omeprazole daily.","expectedOutput":{"conditions":["K21"]}}
-{"id":"HD-13","input":"أعاني من ارتفاع ضغط الدم ومرض السكري من النوع الثاني وأتناول الأدوية بانتظام.","expectedOutput":{"conditions":["I10","E11"]}}
-{"id":"HD-14","input":"I had gestational diabetes in 2021; it resolved after my baby was born and my sugar tests have been normal since. No other conditions.","expectedOutput":{"conditions":[]}}