openmerit 0.3.11 → 0.4.0

package/dist/pi/extension.js

@@ -0,0 +1,148 @@
+import { StringEnum } from "@earendil-works/pi-ai";
+import { Type } from "typebox";
+import { candidates, connections, plan, previewDataset, run, saveDataset } from "../scout/service.js";
+import { SCOUT_INSTRUCTIONS } from "./instructions.js";
+const json = (value) => ({ content: [{ type: "text", text: JSON.stringify(value, null, 2) }], details: undefined });
+const filterSchema = Type.Object({
+    providers: Type.Optional(Type.Array(Type.String(), { description: "Connected Pi providers to draw candidates from, e.g. [\"openrouter\"] or [\"anthropic\"]" })),
+    tools: Type.Optional(Type.Boolean({ description: "The production call uses tool calls" })),
+    maxPrice: Type.Optional(Type.Number({ description: "Max list price, USD per million input+output tokens" })),
+    include: Type.Optional(Type.Array(Type.String(), { description: "Keep only models whose provider/id contains one of these" })),
+    exclude: Type.Optional(Type.Array(Type.String(), { description: "Drop models whose provider/id contains one of these" })),
+});
+const runSchema = Type.Object({
+    task: Type.String({ description: "One paragraph: what the model does in production, its inputs and outputs, and what counts as correct" }),
+    langfuseDataset: Type.Optional(Type.String({ description: "Existing Langfuse dataset of labelled items" })),
+    datasetPath: Type.Optional(Type.String({ description: "Local JSONL of {id?, input, expectedOutput}, pushed to Langfuse before the run" })),
+    evaluator: Type.Object({
+        type: StringEnum(["pi", "module"], { description: "pi: one model call per item through Pi; module: the project's own harness evaluator" }),
+        systemPrompt: Type.Optional(Type.String({ description: "pi: the production system prompt the candidate receives" })),
+        inputTemplate: Type.Optional(Type.String({ description: "pi: user message template; {{input}} is the item input" })),
+        grader: Type.Optional(Type.Object({
+            type: StringEnum(["exact", "contains", "json", "judge"], { description: "json: every expected field must match, strings case-insensitively, arrays as sets (order ignored, same length); exact/contains: normalized text; judge: Scout's model applies the rubric" }),
+            field: Type.Optional(Type.String()),
+            rubric: Type.Optional(Type.String({ description: "judge: what a correct output must do" })),
+        })),
+        path: Type.Optional(Type.String({ description: "module: path to an evaluator module, relative to the project" })),
+    }),
+    filter: Type.Optional(filterSchema),
+    sampleSize: Type.Optional(Type.Integer({ minimum: 2, maximum: 20 })),
+    finalists: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })),
+    repeats: Type.Optional(Type.Integer({ minimum: 1, maximum: 10, description: "Runs of every item per finalist (default 3)" })),
+});
+function host(ctx, options) {
+    const registry = ctx.modelRegistry;
+    const caller = { getModel: (provider, id) => registry.find(provider, id), complete: (model, context, extra) => registry.complete(model, context, extra) };
+    const scoutModel = options.scoutModel ?? (ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : undefined);
+    return {
+        caller, cwd: ctx.cwd, env: process.env, log: options.log ?? (() => { }),
+        models: async () => registry.getAvailable().map(model => ({ provider: model.provider, id: model.id, name: model.name, cost: model.cost })),
+        ...(scoutModel ? { scoutModel } : {}),
+    };
+}
+function toRunParams(params) {
+    const { evaluator } = params;
+    let spec;
+    if (evaluator.type === "module") {
+        if (!evaluator.path)
+            throw new Error("evaluator.path is required for a module evaluator");
+        spec = { type: "module", path: evaluator.path };
+    }
+    else {
+        if (!evaluator.systemPrompt || !evaluator.grader)
+            throw new Error("A pi evaluator needs systemPrompt and grader");
+        const { type, field, rubric } = evaluator.grader;
+        const grader = type === "judge" ? { type: "judge", rubric: rubric ?? "The output must agree with the expected output." }
+            : type === "exact" ? { type: "exact", ...(field ? { field } : {}) }
+                : type === "contains" ? { type: "contains", ...(field ? { field } : {}) } : { type: "json" };
+        spec = { type: "pi", systemPrompt: evaluator.systemPrompt, grader, ...(evaluator.inputTemplate ? { inputTemplate: evaluator.inputTemplate } : {}) };
+    }
+    if (!params.langfuseDataset && !params.datasetPath)
+        throw new Error("Give langfuseDataset or datasetPath");
+    return {
+        task: params.task, evaluator: spec,
+        ...(params.langfuseDataset ? { langfuseDataset: params.langfuseDataset } : {}), ...(params.datasetPath ? { datasetPath: params.datasetPath } : {}),
+        ...(params.filter ? { filter: Object.fromEntries(Object.entries(params.filter).filter(([, value]) => value !== undefined)) } : {}),
+        ...(params.sampleSize ? { sampleSize: params.sampleSize } : {}), ...(params.finalists ? { finalists: params.finalists } : {}), ...(params.repeats ? { repeats: params.repeats } : {}),
+    };
+}
+/** OpenMerit Scout inside Pi: model-recommendation tools, a `/scout` command and Scout's instructions. */
+export function createScoutExtension(options = {}) {
+    const allowSpend = options.allowSpend ?? "ask";
+    return (pi) => {
+        pi.registerTool({
+            name: "scout_connections",
+            label: "Scout connections",
+            description: "Lists the model providers OpenMerit can run (API keys and subscriptions connected in Pi) and the Langfuse project and datasets.",
+            promptSnippet: "See which model providers and Langfuse datasets Scout can use",
+            parameters: Type.Object({}),
+            async execute(_id, _params, _signal, _update, ctx) { return json(await connections(host(ctx, options))); },
+        });
+        pi.registerTool({
+            name: "scout_candidates",
+            label: "Scout candidates",
+            description: "Ranks the runnable models by the Epoch Capabilities Index (ECI), weakest first, after the filter: Epoch's published score, or ScoutBM's estimated/calibrated ECI bundled with the package. Shows how many candidates the binary search will cover.",
+            promptSnippet: "Rank runnable candidate models by public capability (ECI)",
+            parameters: filterSchema,
+            async execute(_id, params, _signal, _update, ctx) {
+                const filter = Object.fromEntries(Object.entries(params).filter(([, value]) => value !== undefined));
+                const ranked = await candidates(host(ctx, options), filter);
+                const short = (list) => list.map(item => `${item.model} ECI ${item.eci.toFixed(1)}${item.eciSource === "published" ? "" : ` (${item.eciSource} ±${item.eciError?.toFixed(1)})`} $${item.price?.toFixed(2) ?? "?"}/M`);
+                return json({ count: ranked.candidates.length, counts: ranked.counts, weakest: short(ranked.candidates.slice(0, 8)), strongest: short(ranked.candidates.slice(-8)), unranked: ranked.unranked.length, dropped: ranked.dropped.length, eci: ranked.eci });
+            },
+        });
+        pi.registerTool({
+            name: "scout_dataset",
+            label: "Scout dataset",
+            description: "Previews a labelled eval dataset: a Langfuse dataset by name or a local JSONL of {id?, input, expectedOutput}. Scout needs at least 5 labelled items.",
+            promptSnippet: "Preview a labelled eval dataset (Langfuse or local JSONL)",
+            parameters: Type.Object({ langfuseDataset: Type.Optional(Type.String()), path: Type.Optional(Type.String()) }),
+            async execute(_id, params, _signal, _update, ctx) {
+                return json(await previewDataset(host(ctx, options), { ...(params.langfuseDataset ? { langfuseDataset: params.langfuseDataset } : {}), ...(params.path ? { path: params.path } : {}) }));
+            },
+        });
+        pi.registerTool({
+            name: "scout_save_dataset",
+            label: "Scout save dataset",
+            description: "Saves labelled items Scout drafted from the codebase (fixtures, tests, examples) to .openmerit/datasets/<name>.jsonl. Each item is marked as a Scout draft; label only what the code or fixtures state.",
+            promptSnippet: "Save drafted labelled items as a local eval dataset",
+            parameters: Type.Object({
+                name: Type.String(),
+                items: Type.Array(Type.Object({ id: Type.Optional(Type.String()), input: Type.Unknown(), expectedOutput: Type.Unknown() }), { minItems: 1 }),
+            }),
+            async execute(_id, params, _signal, _update, ctx) {
+                return json(await saveDataset(host(ctx, options), params.name, params.items.map(item => ({ ...(item.id ? { id: item.id } : {}), input: item.input, expectedOutput: item.expectedOutput }))));
+            },
+        });
+        pi.registerTool({
+            name: "scout_run",
+            label: "Scout run",
+            description: "Runs Scout Model Binary Search: ECI-ranked candidates, 5 sampled items per probe (pass = 5/5), 5 finalists on every item 3 times, each recorded as a Langfuse experiment; the winner is read back from Langfuse. Spends provider credit; without approval it returns the plan only.",
+            promptSnippet: "Binary-search the candidates on labelled items and pick the winner from Langfuse",
+            executionMode: "sequential",
+            parameters: runSchema,
+            async execute(_id, params, signal, onUpdate, ctx) {
+                const scoutHost = host(ctx, options);
+                const runParams = toRunParams(params);
+                const planned = await plan(scoutHost, runParams);
+                const summary = { candidates: planned.ranked.candidates.length, datasetSize: planned.datasetSize, estimatedEvaluations: planned.estimate.evaluations, filter: planned.filter, evaluator: planned.plugin?.name ?? "pi" };
+                let approved = allowSpend === true;
+                if (allowSpend === "ask" && ctx.hasUI)
+                    approved = await ctx.ui.confirm("Run Scout?", `About ${summary.estimatedEvaluations} evaluations over ${summary.candidates} candidates on ${summary.datasetSize} items. This spends provider credit.`);
+                if (!approved)
+                    return json({ status: "plan_only", ...summary, note: "Not run: spending was not approved. Rerun headless with --execute, or approve in the dialog." });
+                const progress = (line) => { scoutHost.log(line); onUpdate?.({ content: [{ type: "text", text: line }], details: undefined }); };
+                const result = await run({ ...scoutHost, log: progress }, runParams, signal);
+                return { content: [{ type: "text", text: `${result.markdown}\nReport files: ${result.files.markdown}, ${result.files.json}` }], details: undefined };
+            },
+        });
+        pi.registerCommand("scout", {
+            description: "Ask OpenMerit Scout which model to use for a task",
+            handler: async (args) => { pi.sendUserMessage(`Use OpenMerit Scout to recommend a model for this task: ${args || "the model calls in this codebase"}`); },
+        });
+        pi.on("before_agent_start", (event) => {
+            event.systemPromptOptions.sections.openmerit_scout = SCOUT_INSTRUCTIONS;
+        });
+    };
+}
+export default createScoutExtension();

package/dist/pi/harness.d.ts

@@ -0,0 +1,20 @@
+import { ModelRuntime } from "@earendil-works/pi-coding-agent";
+import type { RunnableModel } from "../scout/candidates.ts";
+import type { ModelCaller } from "../scout/evaluators/pi.ts";
+/**
+ * OpenMerit's changes to a stock Pi process, applied before Pi reads its environment:
+ * - its own agent directory (~/.openmerit/agent, or $OPENMERIT_HOME/agent), so OpenMerit's
+ *   logins and settings never touch a separate Pi installation; PI_CODING_AGENT_DIR still wins;
+ * - a Claude Code subscription token (CLAUDE_CODE_OAUTH_TOKEN) works as the Anthropic
+ *   subscription credential when no Anthropic key or token is set;
+ * - Pi's own update check is off, because this package pins Pi.
+ */
+export declare function configureHarness(env?: Record<string, string | undefined>): {
+    agentDir: string;
+};
+export declare function openRuntime(): Promise<{
+    runtime: ModelRuntime;
+    caller: ModelCaller;
+    models: () => Promise<RunnableModel[]>;
+}>;
+export declare function pickScoutModel(runtime: ModelRuntime, requested?: string): Promise<import("@earendil-works/pi-ai").Model<import("@earendil-works/pi-ai").Api>>;

package/dist/pi/harness.js

@@ -0,0 +1,49 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { ModelRuntime } from "@earendil-works/pi-coding-agent";
+/**
+ * OpenMerit's changes to a stock Pi process, applied before Pi reads its environment:
+ * - its own agent directory (~/.openmerit/agent, or $OPENMERIT_HOME/agent), so OpenMerit's
+ *   logins and settings never touch a separate Pi installation; PI_CODING_AGENT_DIR still wins;
+ * - a Claude Code subscription token (CLAUDE_CODE_OAUTH_TOKEN) works as the Anthropic
+ *   subscription credential when no Anthropic key or token is set;
+ * - Pi's own update check is off, because this package pins Pi.
+ */
+export function configureHarness(env = process.env) {
+    env.PI_CODING_AGENT_DIR ??= join(env.OPENMERIT_HOME ?? join(homedir(), ".openmerit"), "agent");
+    if (env.CLAUDE_CODE_OAUTH_TOKEN && !env.ANTHROPIC_API_KEY && !env.ANTHROPIC_OAUTH_TOKEN && !env.ANTHROPIC_AUTH_TOKEN) {
+        env.ANTHROPIC_OAUTH_TOKEN = env.CLAUDE_CODE_OAUTH_TOKEN;
+    }
+    env.PI_SKIP_VERSION_CHECK ??= "1";
+    return { agentDir: env.PI_CODING_AGENT_DIR };
+}
+/** Scout runs on the strongest connected general model unless the user names one. */
+const SCOUT_MODEL_PREFERENCE = [
+    "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "openai/gpt-6.1-sol", "openai/gpt-6-sol",
+    "openrouter/anthropic/claude-opus-5.5", "openrouter/anthropic/claude-sonnet-5.5", "openrouter/openai/gpt-6.1-sol",
+];
+export async function openRuntime() {
+    const runtime = await ModelRuntime.create({ allowModelNetwork: true, modelRefreshTimeoutMs: 20_000 });
+    const caller = { getModel: (provider, id) => runtime.getModel(provider, id), complete: (model, context, extra) => runtime.complete(model, context, extra) };
+    const models = async () => (await runtime.getAvailable()).map(model => ({ provider: model.provider, id: model.id, name: model.name, cost: model.cost }));
+    return { runtime, caller, models };
+}
+export async function pickScoutModel(runtime, requested) {
+    const available = await runtime.getAvailable();
+    const ref = (model) => `${model.provider}/${model.id}`;
+    if (requested) {
+        const model = available.find(item => ref(item) === requested);
+        if (!model)
+            throw new Error(`Scout model ${requested} is not available with the connected providers. Run \`openmerit status\`.`);
+        return model;
+    }
+    for (const wanted of SCOUT_MODEL_PREFERENCE) {
+        const model = available.find(item => ref(item) === wanted);
+        if (model)
+            return model;
+    }
+    const model = available[0];
+    if (!model)
+        throw new Error("No model provider is connected. Set an API key (for example OPENROUTER_API_KEY or ANTHROPIC_API_KEY) or run `openmerit login` for a subscription.");
+    return model;
+}

package/dist/pi/instructions.d.ts

@@ -0,0 +1,2 @@
+/** Scout's workflow, added to Pi's system prompt as the `openmerit_scout` section. */
+export declare const SCOUT_INSTRUCTIONS = "You are OpenMerit Scout. You recommend the model a user should run for a task, from measured results on the user's own labelled examples, not from memory.\n\nWorkflow when asked which model to use:\n1. Understand the task. If a codebase is present, find where the model is called (grep for model IDs, SDK calls, \"systemPrompt\", \"model:\"), the production prompt, whether it calls tools, its output format, and existing evals: fixtures, tests with expected outputs, Langfuse datasets (scout_connections lists them), or .openmerit/datasets/*.jsonl.\n2. Find labelled data. Prefer an existing Langfuse dataset or fixture set with expected outputs. Preview it with scout_dataset; Scout needs at least 5 items. If none exists, draft at least 6 items only from what the code, fixtures or the user state, save them with scout_save_dataset, and say they are drafts the user should check. Never invent labels you cannot justify.\n3. Choose the evaluator. If the project ships an evaluator module that runs its real harness (for example .openmerit/evaluator.ts or examples/*/evaluator.ts), use evaluator type \"module\". Otherwise use type \"pi\" with the production system prompt and the strictest grader that fits: \"json\" for structured outputs, \"exact\"/\"contains\" for short answers, \"judge\" with a precise rubric only for free text.\n4. Check candidates with scout_candidates. Filter only for real constraints: providers the user can run, tools when the call uses tools, a price ceiling the user gave, data-residency or vendor exclusions the user named. Do not narrow the pool from your own preferences.\n5. Run scout_run with the task, dataset and evaluator. It binary-searches the ECI-ranked candidates (pass = every sampled item), screens neighbours, runs 5 finalists (plus cheaper, stronger challengers) on every item 3 times, records each as a Langfuse experiment, and picks the winner from Langfuse: most passes, then lower measured cost per item, then lower ECI.\n6. Answer with the recommended model ID, its passes and cost per item, the runner-up, the Langfuse experiment links, the report path, and the limitations the report lists (sample size, untested skipped models, drafted labels, residency or policy gaps). If scout_run returns plan_only, report the plan and the command to run it.\n\nRules: measured results on the user's data outrank public benchmarks. Provider, auth or billing errors are not quality failures; report them. Never print API keys or tokens.";

package/dist/pi/instructions.js

@@ -0,0 +1,12 @@
+/** Scout's workflow, added to Pi's system prompt as the `openmerit_scout` section. */
+export const SCOUT_INSTRUCTIONS = `You are OpenMerit Scout. You recommend the model a user should run for a task, from measured results on the user's own labelled examples, not from memory.
+
+Workflow when asked which model to use:
+1. Understand the task. If a codebase is present, find where the model is called (grep for model IDs, SDK calls, "systemPrompt", "model:"), the production prompt, whether it calls tools, its output format, and existing evals: fixtures, tests with expected outputs, Langfuse datasets (scout_connections lists them), or .openmerit/datasets/*.jsonl.
+2. Find labelled data. Prefer an existing Langfuse dataset or fixture set with expected outputs. Preview it with scout_dataset; Scout needs at least 5 items. If none exists, draft at least 6 items only from what the code, fixtures or the user state, save them with scout_save_dataset, and say they are drafts the user should check. Never invent labels you cannot justify.
+3. Choose the evaluator. If the project ships an evaluator module that runs its real harness (for example .openmerit/evaluator.ts or examples/*/evaluator.ts), use evaluator type "module". Otherwise use type "pi" with the production system prompt and the strictest grader that fits: "json" for structured outputs, "exact"/"contains" for short answers, "judge" with a precise rubric only for free text.
+4. Check candidates with scout_candidates. Filter only for real constraints: providers the user can run, tools when the call uses tools, a price ceiling the user gave, data-residency or vendor exclusions the user named. Do not narrow the pool from your own preferences.
+5. Run scout_run with the task, dataset and evaluator. It binary-searches the ECI-ranked candidates (pass = every sampled item), screens neighbours, runs 5 finalists (plus cheaper, stronger challengers) on every item 3 times, records each as a Langfuse experiment, and picks the winner from Langfuse: most passes, then lower measured cost per item, then lower ECI.
+6. Answer with the recommended model ID, its passes and cost per item, the runner-up, the Langfuse experiment links, the report path, and the limitations the report lists (sample size, untested skipped models, drafted labels, residency or policy gaps). If scout_run returns plan_only, report the plan and the command to run it.
+
+Rules: measured results on the user's data outrank public benchmarks. Provider, auth or billing errors are not quality failures; report them. Never print API keys or tokens.`;

package/dist/scout/candidates.d.ts

@@ -0,0 +1,62 @@
+import type { EciEstimate, EciRow } from "./eci.ts";
+/** A model the user's connections can run, placed on the ECI scale. */
+export interface Candidate {
+    /** `provider/id` as Pi names it, such as `openrouter/deepseek/deepseek-v4-pro` or `anthropic/claude-sonnet-5-5`. */
+    model: string;
+    provider: string;
+    id: string;
+    name: string;
+    /** Epoch Capabilities Index: published, or ScoutBM's estimate from the bundled CSV. */
+    eci: number;
+    /** The Epoch row, or the catalogue model the estimate belongs to. */
+    eciModel: string;
+    eciSource: "published" | "estimated" | "calibrated";
+    /** Standard error of an estimated or calibrated ECI. */
+    eciError?: number;
+    /** List price in USD per million input plus output tokens; null when unknown. Measured cost decides ties, not this. */
+    price: number | null;
+}
+/** The parts of a Pi model this module reads. */
+export interface RunnableModel {
+    provider: string;
+    id: string;
+    name: string;
+    cost?: {
+        input: number;
+        output: number;
+    };
+}
+export interface CandidateFilter {
+    /** Connected providers to draw candidates from, in preference order when two serve the same model. */
+    providers?: string[];
+    /** The task calls tools: drop models known not to support tool calls. */
+    tools?: boolean;
+    /** `provider/id` → supports tool calls. Models missing from the map are kept. */
+    toolSupport?: ReadonlyMap<string, boolean>;
+    /** Drop models whose list price (input + output per million tokens) is above this. */
+    maxPrice?: number;
+    /** Case-insensitive substrings; a model whose `provider/id` contains any is dropped. */
+    exclude?: string[];
+    /** Keep only models whose `provider/id` contains one of these. */
+    include?: string[];
+}
+/** Exact names only, ignoring case and punctuation: "GPT-6.1 Sol" = "gpt-6.1-sol", never "GPT-6.1". */
+export declare const nameKey: (name: string) => string;
+/**
+ * Every runnable model with an ECI, ascending. A published Epoch row wins; otherwise ScoutBM's
+ * estimated or calibrated ECI from the bundled CSV, matched by OpenRouter ID or exact name. Each
+ * Epoch row or estimate is used by at most one model: the first connected provider, then the
+ * cheapest listing. Models with neither are reported as unranked, never guessed.
+ */
+export declare function rankCandidates(models: readonly RunnableModel[], rows: readonly EciRow[], filter?: CandidateFilter, estimates?: readonly EciEstimate[]): {
+    candidates: Candidate[];
+    unranked: string[];
+    dropped: {
+        model: string;
+        reason: string;
+    }[];
+};
+/** Ascending ECI; equal scores fall back to model ID purely for repeatability. */
+export declare function rank(candidates: readonly Candidate[]): Candidate[];
+/** OpenRouter's public model list says which models accept tool calls. No key is needed. */
+export declare function openRouterToolSupport(request?: typeof fetch): Promise<Map<string, boolean>>;

package/dist/scout/candidates.js

@@ -0,0 +1,90 @@
+/** Exact names only, ignoring case and punctuation: "GPT-6.1 Sol" = "gpt-6.1-sol", never "GPT-6.1". */
+export const nameKey = (name) => name.toLowerCase().replace(/[^a-z0-9]/g, "");
+/**
+ * Aliases (`~…`, `-latest`), batch-only and free-tier variants are not separate models,
+ * and free tiers are rate limited and may train on prompts.
+ */
+const ALIAS = /(^~|-latest$|:batch$|:free$|:extended$)/;
+/**
+ * Every runnable model with an ECI, ascending. A published Epoch row wins; otherwise ScoutBM's
+ * estimated or calibrated ECI from the bundled CSV, matched by OpenRouter ID or exact name. Each
+ * Epoch row or estimate is used by at most one model: the first connected provider, then the
+ * cheapest listing. Models with neither are reported as unranked, never guessed.
+ */
+export function rankCandidates(models, rows, filter = {}, estimates = []) {
+    const byName = new Map();
+    for (const row of rows)
+        for (const name of row.names)
+            byName.set(nameKey(name), (byName.get(nameKey(name)) ?? new Set()).add(row));
+    const providers = filter.providers ?? [...new Set(models.map(model => model.provider))];
+    const lower = (list) => list?.map(item => item.toLowerCase());
+    const include = lower(filter.include), exclude = lower(filter.exclude);
+    const unranked = [];
+    const dropped = [];
+    const best = new Map();
+    const estimateById = new Map(estimates.flatMap(item => [[item.catalogueModel.toLowerCase(), item], ...(item.openRouterId ? [[item.openRouterId.toLowerCase(), item]] : [])]));
+    const estimateByName = new Map();
+    for (const item of estimates)
+        for (const name of [item.name, item.catalogueModel.split("/").at(-1)])
+            estimateByName.set(nameKey(name), (estimateByName.get(nameKey(name)) ?? new Set()).add(item));
+    for (const model of models) {
+        const full = `${model.provider}/${model.id}`;
+        if (!providers.includes(model.provider) || ALIAS.test(model.id))
+            continue;
+        const id = full.toLowerCase();
+        if (include && !include.some(item => id.includes(item)))
+            continue;
+        if (exclude?.some(item => id.includes(item))) {
+            dropped.push({ model: full, reason: "excluded" });
+            continue;
+        }
+        if (filter.tools && filter.toolSupport?.get(full) === false) {
+            dropped.push({ model: full, reason: "no tool calls" });
+            continue;
+        }
+        const price = model.cost && Number.isFinite(model.cost.input + model.cost.output) ? model.cost.input + model.cost.output : null;
+        if (filter.maxPrice !== undefined && price !== null && price > filter.maxPrice) {
+            dropped.push({ model: full, reason: `price ${price} > ${filter.maxPrice}` });
+            continue;
+        }
+        const names = [model.name.replace(/^[^:]+:\s*/, ""), model.name, model.id.split("/").at(-1)];
+        const matches = new Set(names.flatMap(name => [...byName.get(nameKey(name)) ?? []]));
+        let row;
+        let candidate;
+        if (matches.size === 1) {
+            const published = [...matches][0];
+            row = published;
+            candidate = { model: full, provider: model.provider, id: model.id, name: model.name, eci: published.eci, eciModel: published.model, eciSource: "published", price };
+        }
+        else {
+            const byId = model.provider === "openrouter" ? estimateById.get(model.id.toLowerCase()) : undefined;
+            const byNames = new Set(names.flatMap(name => [...estimateByName.get(nameKey(name)) ?? []]));
+            const estimate = byId ?? (byNames.size === 1 ? [...byNames][0] : undefined);
+            if (!estimate) {
+                unranked.push(full);
+                continue;
+            }
+            row = estimate;
+            candidate = { model: full, provider: model.provider, id: model.id, name: model.name, eci: estimate.eci, eciModel: estimate.catalogueModel, eciSource: estimate.source, eciError: estimate.standardError, price };
+        }
+        const current = best.get(row);
+        const better = !current
+            || providers.indexOf(candidate.provider) < providers.indexOf(current.provider)
+            || (candidate.provider === current.provider && (candidate.price ?? Infinity) < (current.price ?? Infinity));
+        if (better)
+            best.set(row, candidate);
+    }
+    return { candidates: rank([...best.values()]), unranked: unranked.sort(), dropped };
+}
+/** Ascending ECI; equal scores fall back to model ID purely for repeatability. */
+export function rank(candidates) {
+    return [...candidates].sort((a, b) => a.eci - b.eci || a.model.localeCompare(b.model));
+}
+/** OpenRouter's public model list says which models accept tool calls. No key is needed. */
+export async function openRouterToolSupport(request = fetch) {
+    const response = await request("https://openrouter.ai/api/v1/models", { signal: AbortSignal.timeout(30_000) });
+    if (!response.ok)
+        throw new Error(`OpenRouter models HTTP ${response.status}`);
+    const body = await response.json();
+    return new Map((body.data ?? []).map(model => [`openrouter/${model.id}`, (model.supported_parameters ?? []).includes("tools")]));
+}

package/dist/scout/eci.d.ts

@@ -0,0 +1,42 @@
+export declare const ECI_SCORES_URL = "https://epoch.ai/data/eci_scores.csv";
+export declare const ECI_ATTRIBUTION = "Epoch AI, 'Epoch Capabilities Index'. Published online at epoch.ai. CC BY 4.0.";
+export interface EciRow {
+    /** Epoch's model key. */
+    model: string;
+    /** Every name Epoch lists for the model: key, display name and model versions. */
+    names: string[];
+    eci: number;
+    ciLow: number | null;
+    ciHigh: number | null;
+    date: string | null;
+}
+/** RFC 4180 fields: commas, quotes and newlines inside quoted values. */
+export declare function parseCSV(text: string): Record<string, string>[];
+export declare function parseEciScores(text: string): EciRow[];
+export interface EciSource {
+    source: string;
+    url: string;
+    attribution: string;
+    fetchedAt: string;
+    rows: EciRow[];
+}
+export declare function loadEci(request?: typeof fetch): Promise<EciSource>;
+/** ScoutBM's estimated or calibrated ECI for a model Epoch does not score, from the bundled CSV. */
+export interface EciEstimate {
+    catalogueModel: string;
+    name: string;
+    openRouterId: string | null;
+    eci: number;
+    source: "estimated" | "calibrated";
+    standardError: number;
+    benchmarks: string[];
+}
+export declare function parseEciEstimates(text: string): {
+    provenance: string;
+    estimates: EciEstimate[];
+};
+/** The CSV shipped in the package (data/eci-estimates.csv): no network, no Cloudflare. */
+export declare function loadEciEstimates(): Promise<{
+    provenance: string;
+    estimates: EciEstimate[];
+}>;

package/dist/scout/eci.js

@@ -0,0 +1,84 @@
+// Epoch Capabilities Index (ECI): the public, openly licensed capability ranking Scout's binary
+// search orders candidates by. Ported from the ScoutBM prototype (published scores only).
+export const ECI_SCORES_URL = "https://epoch.ai/data/eci_scores.csv";
+export const ECI_ATTRIBUTION = "Epoch AI, 'Epoch Capabilities Index'. Published online at epoch.ai. CC BY 4.0.";
+/** RFC 4180 fields: commas, quotes and newlines inside quoted values. */
+export function parseCSV(text) {
+    const rows = [];
+    let row = [], field = "", quoted = false;
+    for (let index = 0; index < text.length; index++) {
+        const char = text[index];
+        if (quoted) {
+            if (char === '"' && text[index + 1] === '"') {
+                field += '"';
+                index++;
+            }
+            else if (char === '"')
+                quoted = false;
+            else
+                field += char;
+        }
+        else if (char === '"')
+            quoted = true;
+        else if (char === ",") {
+            row.push(field);
+            field = "";
+        }
+        else if (char === "\n" || char === "\r") {
+            if (char === "\r" && text[index + 1] === "\n")
+                index++;
+            row.push(field);
+            rows.push(row);
+            row = [];
+            field = "";
+        }
+        else
+            field += char;
+    }
+    if (field || row.length) {
+        row.push(field);
+        rows.push(row);
+    }
+    const [header = [], ...body] = rows.filter(item => item.some(Boolean));
+    return body.map(cells => Object.fromEntries(header.map((name, index) => [name, (cells[index] ?? "").trim()])));
+}
+const num = (value) => (value && Number.isFinite(Number(value)) ? Number(value) : null);
+export function parseEciScores(text) {
+    const rows = parseCSV(text);
+    const missing = ["Model", "Display name", "eci", "model_versions"].filter(name => rows[0] && !(name in rows[0]));
+    if (!rows.length || missing.length)
+        throw new Error(`Epoch eci_scores.csv is empty or missing columns: ${missing.join(", ")}`);
+    const scored = rows.filter(row => row.Model && num(row.eci) !== null).map(row => ({
+        model: row.Model, eci: num(row.eci), ciLow: num(row.eci_ci_low), ciHigh: num(row.eci_ci_high), date: row.date || null,
+        names: [...new Set([row.Model, row["Display name"], ...row.model_versions.split(",").map(item => item.trim())].filter(Boolean))],
+    }));
+    if (new Set(scored.map(item => item.model)).size !== scored.length)
+        throw new Error("Duplicate Epoch ECI model rows; retry the download");
+    return scored;
+}
+export async function loadEci(request = fetch) {
+    const response = await request(ECI_SCORES_URL, { signal: AbortSignal.timeout(30_000) });
+    if (!response.ok)
+        throw new Error(`Epoch ECI HTTP ${response.status}; no fallback ranking is used`);
+    return { source: "Epoch Capabilities Index", url: ECI_SCORES_URL, attribution: ECI_ATTRIBUTION, fetchedAt: new Date().toISOString(), rows: parseEciScores(await response.text()) };
+}
+export function parseEciEstimates(text) {
+    const lines = text.split("\n");
+    const provenance = lines.filter(line => line.startsWith("#")).map(line => line.slice(1).trim()).join(" ");
+    const rows = parseCSV(lines.filter(line => !line.startsWith("#")).join("\n"));
+    const estimates = rows.flatMap(row => {
+        const eci = num(row.eci), standardError = num(row.standard_error);
+        if (!row.catalogue_model || eci === null || (row.eci_source !== "estimated" && row.eci_source !== "calibrated"))
+            return [];
+        return [{
+                catalogueModel: row.catalogue_model, name: row.name ?? row.catalogue_model, openRouterId: row.openrouter_id || null, eci,
+                source: row.eci_source, standardError: standardError ?? 0, benchmarks: (row.benchmarks ?? "").split(";").map(item => item.trim()).filter(Boolean),
+            }];
+    });
+    return { provenance, estimates };
+}
+/** The CSV shipped in the package (data/eci-estimates.csv): no network, no Cloudflare. */
+export async function loadEciEstimates() {
+    const { readFile } = await import("node:fs/promises");
+    return parseEciEstimates(await readFile(new URL("../../data/eci-estimates.csv", import.meta.url), "utf8"));
+}

package/dist/scout/evaluators/module.d.ts

@@ -0,0 +1,24 @@
+import type { Evaluate } from "../search.ts";
+/**
+ * An evaluator that runs a candidate through the user's own harness, such as a deployed agent's
+ * HTTP API, and checks the result against the labelled item. A module's default export is one of these.
+ */
+export interface EvaluatorPlugin {
+    name: string;
+    /** Pi providers the harness can run, such as ["openrouter"]. Candidates are drawn only from these. */
+    providers?: string[];
+    /** The harness calls tools, so models without tool calls are not candidates. */
+    tools?: boolean;
+    /** Score name recorded per item in Langfuse, ideally the customer's own metric (e.g. `enrollment-correct`). */
+    passScore?: string;
+    /** Experiment metadata key `model_<role>`. */
+    role?: string;
+    /** Items evaluated at once within one probe. */
+    concurrency?: number;
+    create(context: {
+        env: Record<string, string | undefined>;
+        log: (line: string) => void;
+    }): Evaluate | Promise<Evaluate>;
+}
+/** Imports a `.ts` or `.js` evaluator module from the user's project. Node.js 22.19 strips its types. */
+export declare function loadEvaluatorPlugin(path: string, cwd?: string): Promise<EvaluatorPlugin>;

package/dist/scout/evaluators/module.js

@@ -0,0 +1,19 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+/** Imports a `.ts` or `.js` evaluator module from the user's project. Node.js 22.19 strips its types. */
+export async function loadEvaluatorPlugin(path, cwd = process.cwd()) {
+    const loaded = await import(__rewriteRelativeImportExtension(pathToFileURL(resolve(cwd, path)).href));
+    const plugin = loaded.default;
+    if (!plugin || typeof plugin !== "object" || typeof plugin.create !== "function" || typeof plugin.name !== "string") {
+        throw new Error(`${path} must default-export an evaluator: { name, create(context) => evaluate }`);
+    }
+    return plugin;
+}

package/dist/scout/evaluators/pi.d.ts

@@ -0,0 +1,53 @@
+import type { Api, AssistantMessage, Context, Model } from "@earendil-works/pi-ai";
+import { type Evaluate } from "../search.ts";
+/** What the Pi evaluator needs from Pi: a `ModelRuntime`, or an extension's `ctx.modelRegistry` through `fromRegistry`. */
+export interface ModelCaller {
+    getModel(provider: string, id: string): Model<Api> | undefined;
+    complete(model: Model<Api>, context: Context, options?: {
+        signal?: AbortSignal;
+        maxTokens?: number;
+        apiKey?: string;
+    }): Promise<AssistantMessage>;
+}
+export type Grader = 
+/** Normalized text equality with `expectedOutput` (or `expectedOutput[field]`). */
+{
+    type: "exact";
+    field?: string;
+}
+/** The output contains the expected text, case-insensitively. */
+ | {
+    type: "contains";
+    field?: string;
+}
+/** The output's JSON contains every expected field with an equal value (strings compared case-insensitively, arrays as sets). */
+ | {
+    type: "json";
+}
+/** A judge model answers PASS or FAIL for the output against the expected output and rubric. */
+ | {
+    type: "judge";
+    rubric: string;
+    model?: string;
+};
+export interface PiTaskSpec {
+    /** The candidate's system prompt: the instructions the production model receives. */
+    systemPrompt: string;
+    /** Optional template for the user message; `{{input}}` is the item's input as text or JSON. */
+    inputTemplate?: string;
+    grader: Grader;
+    maxTokens?: number;
+}
+/** The first JSON object or array in a reply, with code fences removed. */
+export declare function extractJson(reply: string): unknown;
+/** Every expected field matches: objects by key, arrays as sets, strings case-insensitively. */
+export declare function jsonMatches(expected: unknown, actual: unknown): boolean;
+/**
+ * Runs each item as one model call through Pi, so any provider Pi connects (API key or
+ * subscription) can be a candidate, then grades the reply. Tool-using agents need an
+ * evaluator module that runs the real harness instead.
+ */
+export declare function piEvaluator(caller: ModelCaller, spec: PiTaskSpec, options?: {
+    judgeModel?: string;
+    retryDelaysMs?: number[];
+}): Evaluate;