openmerit 0.4.1 → 0.4.2

package/dist/pi/extension.js

@@ -1,148 +0,0 @@
-import { StringEnum } from "@earendil-works/pi-ai";
-import { Type } from "typebox";
-import { candidates, connections, plan, previewDataset, run, saveDataset } from "../scout/service.js";
-import { SCOUT_INSTRUCTIONS } from "./instructions.js";
-const json = (value) => ({ content: [{ type: "text", text: JSON.stringify(value, null, 2) }], details: undefined });
-const filterSchema = Type.Object({
-    providers: Type.Optional(Type.Array(Type.String(), { description: "Connected Pi providers to draw candidates from, e.g. [\"openrouter\"] or [\"anthropic\"]" })),
-    tools: Type.Optional(Type.Boolean({ description: "The production call uses tool calls" })),
-    maxPrice: Type.Optional(Type.Number({ description: "Max list price, USD per million input+output tokens" })),
-    include: Type.Optional(Type.Array(Type.String(), { description: "Keep only models whose provider/id contains one of these" })),
-    exclude: Type.Optional(Type.Array(Type.String(), { description: "Drop models whose provider/id contains one of these" })),
-});
-const runSchema = Type.Object({
-    task: Type.String({ description: "One paragraph: what the model does in production, its inputs and outputs, and what counts as correct" }),
-    langfuseDataset: Type.Optional(Type.String({ description: "Existing Langfuse dataset of labelled items" })),
-    datasetPath: Type.Optional(Type.String({ description: "Local JSONL of {id?, input, expectedOutput}, pushed to Langfuse before the run" })),
-    evaluator: Type.Object({
-        type: StringEnum(["pi", "module"], { description: "pi: one model call per item through Pi; module: the project's own harness evaluator" }),
-        systemPrompt: Type.Optional(Type.String({ description: "pi: the production system prompt the candidate receives" })),
-        inputTemplate: Type.Optional(Type.String({ description: "pi: user message template; {{input}} is the item input" })),
-        grader: Type.Optional(Type.Object({
-            type: StringEnum(["exact", "contains", "json", "judge"], { description: "json: every expected field must match, strings case-insensitively, arrays as sets (order ignored, same length); exact/contains: normalized text; judge: Scout's model applies the rubric" }),
-            field: Type.Optional(Type.String()),
-            rubric: Type.Optional(Type.String({ description: "judge: what a correct output must do" })),
-        })),
-        path: Type.Optional(Type.String({ description: "module: path to an evaluator module, relative to the project" })),
-    }),
-    filter: Type.Optional(filterSchema),
-    sampleSize: Type.Optional(Type.Integer({ minimum: 2, maximum: 20 })),
-    finalists: Type.Optional(Type.Integer({ minimum: 1, maximum: 10 })),
-    repeats: Type.Optional(Type.Integer({ minimum: 1, maximum: 10, description: "Runs of every item per finalist (default 3)" })),
-});
-function host(ctx, options) {
-    const registry = ctx.modelRegistry;
-    const caller = { getModel: (provider, id) => registry.find(provider, id), complete: (model, context, extra) => registry.complete(model, context, extra) };
-    const scoutModel = options.scoutModel ?? (ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : undefined);
-    return {
-        caller, cwd: ctx.cwd, env: process.env, log: options.log ?? (() => { }),
-        models: async () => registry.getAvailable().map(model => ({ provider: model.provider, id: model.id, name: model.name, cost: model.cost })),
-        ...(scoutModel ? { scoutModel } : {}),
-    };
-}
-function toRunParams(params) {
-    const { evaluator } = params;
-    let spec;
-    if (evaluator.type === "module") {
-        if (!evaluator.path)
-            throw new Error("evaluator.path is required for a module evaluator");
-        spec = { type: "module", path: evaluator.path };
-    }
-    else {
-        if (!evaluator.systemPrompt || !evaluator.grader)
-            throw new Error("A pi evaluator needs systemPrompt and grader");
-        const { type, field, rubric } = evaluator.grader;
-        const grader = type === "judge" ? { type: "judge", rubric: rubric ?? "The output must agree with the expected output." }
-            : type === "exact" ? { type: "exact", ...(field ? { field } : {}) }
-                : type === "contains" ? { type: "contains", ...(field ? { field } : {}) } : { type: "json" };
-        spec = { type: "pi", systemPrompt: evaluator.systemPrompt, grader, ...(evaluator.inputTemplate ? { inputTemplate: evaluator.inputTemplate } : {}) };
-    }
-    if (!params.langfuseDataset && !params.datasetPath)
-        throw new Error("Give langfuseDataset or datasetPath");
-    return {
-        task: params.task, evaluator: spec,
-        ...(params.langfuseDataset ? { langfuseDataset: params.langfuseDataset } : {}), ...(params.datasetPath ? { datasetPath: params.datasetPath } : {}),
-        ...(params.filter ? { filter: Object.fromEntries(Object.entries(params.filter).filter(([, value]) => value !== undefined)) } : {}),
-        ...(params.sampleSize ? { sampleSize: params.sampleSize } : {}), ...(params.finalists ? { finalists: params.finalists } : {}), ...(params.repeats ? { repeats: params.repeats } : {}),
-    };
-}
-/** OpenMerit Scout inside Pi: model-recommendation tools, a `/scout` command and Scout's instructions. */
-export function createScoutExtension(options = {}) {
-    const allowSpend = options.allowSpend ?? "ask";
-    return (pi) => {
-        pi.registerTool({
-            name: "scout_connections",
-            label: "Scout connections",
-            description: "Lists the model providers OpenMerit can run (API keys and subscriptions connected in Pi) and the Langfuse project and datasets.",
-            promptSnippet: "See which model providers and Langfuse datasets Scout can use",
-            parameters: Type.Object({}),
-            async execute(_id, _params, _signal, _update, ctx) { return json(await connections(host(ctx, options))); },
-        });
-        pi.registerTool({
-            name: "scout_candidates",
-            label: "Scout candidates",
-            description: "Ranks the runnable models by the Epoch Capabilities Index (ECI), weakest first, after the filter: Epoch's published score, or ScoutBM's estimated/calibrated ECI bundled with the package. Shows how many candidates the binary search will cover.",
-            promptSnippet: "Rank runnable candidate models by public capability (ECI)",
-            parameters: filterSchema,
-            async execute(_id, params, _signal, _update, ctx) {
-                const filter = Object.fromEntries(Object.entries(params).filter(([, value]) => value !== undefined));
-                const ranked = await candidates(host(ctx, options), filter);
-                const short = (list) => list.map(item => `${item.model} ECI ${item.eci.toFixed(1)}${item.eciSource === "published" ? "" : ` (${item.eciSource} ±${item.eciError?.toFixed(1)})`} $${item.price?.toFixed(2) ?? "?"}/M`);
-                return json({ count: ranked.candidates.length, counts: ranked.counts, weakest: short(ranked.candidates.slice(0, 8)), strongest: short(ranked.candidates.slice(-8)), unranked: ranked.unranked.length, dropped: ranked.dropped.length, eci: ranked.eci });
-            },
-        });
-        pi.registerTool({
-            name: "scout_dataset",
-            label: "Scout dataset",
-            description: "Previews a labelled eval dataset: a Langfuse dataset by name or a local JSONL of {id?, input, expectedOutput}. Scout needs at least 5 labelled items.",
-            promptSnippet: "Preview a labelled eval dataset (Langfuse or local JSONL)",
-            parameters: Type.Object({ langfuseDataset: Type.Optional(Type.String()), path: Type.Optional(Type.String()) }),
-            async execute(_id, params, _signal, _update, ctx) {
-                return json(await previewDataset(host(ctx, options), { ...(params.langfuseDataset ? { langfuseDataset: params.langfuseDataset } : {}), ...(params.path ? { path: params.path } : {}) }));
-            },
-        });
-        pi.registerTool({
-            name: "scout_save_dataset",
-            label: "Scout save dataset",
-            description: "Saves labelled items Scout drafted from the codebase (fixtures, tests, examples) to .openmerit/datasets/<name>.jsonl. Each item is marked as a Scout draft; label only what the code or fixtures state.",
-            promptSnippet: "Save drafted labelled items as a local eval dataset",
-            parameters: Type.Object({
-                name: Type.String(),
-                items: Type.Array(Type.Object({ id: Type.Optional(Type.String()), input: Type.Unknown(), expectedOutput: Type.Unknown() }), { minItems: 1 }),
-            }),
-            async execute(_id, params, _signal, _update, ctx) {
-                return json(await saveDataset(host(ctx, options), params.name, params.items.map(item => ({ ...(item.id ? { id: item.id } : {}), input: item.input, expectedOutput: item.expectedOutput }))));
-            },
-        });
-        pi.registerTool({
-            name: "scout_run",
-            label: "Scout run",
-            description: "Runs Scout Model Binary Search: ECI-ranked candidates, 5 sampled items per probe (pass = 5/5), 5 finalists on every item 3 times, each recorded as a Langfuse experiment; the winner is read back from Langfuse. Spends provider credit; without approval it returns the plan only.",
-            promptSnippet: "Binary-search the candidates on labelled items and pick the winner from Langfuse",
-            executionMode: "sequential",
-            parameters: runSchema,
-            async execute(_id, params, signal, onUpdate, ctx) {
-                const scoutHost = host(ctx, options);
-                const runParams = toRunParams(params);
-                const planned = await plan(scoutHost, runParams);
-                const summary = { candidates: planned.ranked.candidates.length, datasetSize: planned.datasetSize, estimatedEvaluations: planned.estimate.evaluations, filter: planned.filter, evaluator: planned.plugin?.name ?? "pi" };
-                let approved = allowSpend === true;
-                if (allowSpend === "ask" && ctx.hasUI)
-                    approved = await ctx.ui.confirm("Run Scout?", `About ${summary.estimatedEvaluations} evaluations over ${summary.candidates} candidates on ${summary.datasetSize} items. This spends provider credit.`);
-                if (!approved)
-                    return json({ status: "plan_only", ...summary, note: "Not run: spending was not approved. Rerun headless with --execute, or approve in the dialog." });
-                const progress = (line) => { scoutHost.log(line); onUpdate?.({ content: [{ type: "text", text: line }], details: undefined }); };
-                const result = await run({ ...scoutHost, log: progress }, runParams, signal);
-                return { content: [{ type: "text", text: `${result.markdown}\nReport files: ${result.files.markdown}, ${result.files.json}` }], details: undefined };
-            },
-        });
-        pi.registerCommand("scout", {
-            description: "Ask OpenMerit Scout which model to use for a task",
-            handler: async (args) => { pi.sendUserMessage(`Use OpenMerit Scout to recommend a model for this task: ${args || "the model calls in this codebase"}`); },
-        });
-        pi.on("before_agent_start", (event) => {
-            event.systemPromptOptions.sections.openmerit_scout = SCOUT_INSTRUCTIONS;
-        });
-    };
-}
-export default createScoutExtension();

package/dist/pi/harness.d.ts

@@ -1,20 +0,0 @@
-import { ModelRuntime } from "@earendil-works/pi-coding-agent";
-import type { RunnableModel } from "../scout/candidates.js";
-import type { ModelCaller } from "../scout/evaluators/pi.js";
-/**
- * OpenMerit's changes to a stock Pi process, applied before Pi reads its environment:
- * - its own agent directory (~/.openmerit/agent, or $OPENMERIT_HOME/agent), so OpenMerit's
- *   logins and settings never touch a separate Pi installation; PI_CODING_AGENT_DIR still wins;
- * - a Claude Code subscription token (CLAUDE_CODE_OAUTH_TOKEN) works as the Anthropic
- *   subscription credential when no Anthropic key or token is set;
- * - Pi's own update check is off, because this package pins Pi.
- */
-export declare function configureHarness(env?: Record<string, string | undefined>): {
-    agentDir: string;
-};
-export declare function openRuntime(): Promise<{
-    runtime: ModelRuntime;
-    caller: ModelCaller;
-    models: () => Promise<RunnableModel[]>;
-}>;
-export declare function pickScoutModel(runtime: ModelRuntime, requested?: string): Promise<import("@earendil-works/pi-ai").Model<import("@earendil-works/pi-ai").Api>>;

package/dist/pi/harness.js

@@ -1,49 +0,0 @@
-import { homedir } from "node:os";
-import { join } from "node:path";
-import { ModelRuntime } from "@earendil-works/pi-coding-agent";
-/**
- * OpenMerit's changes to a stock Pi process, applied before Pi reads its environment:
- * - its own agent directory (~/.openmerit/agent, or $OPENMERIT_HOME/agent), so OpenMerit's
- *   logins and settings never touch a separate Pi installation; PI_CODING_AGENT_DIR still wins;
- * - a Claude Code subscription token (CLAUDE_CODE_OAUTH_TOKEN) works as the Anthropic
- *   subscription credential when no Anthropic key or token is set;
- * - Pi's own update check is off, because this package pins Pi.
- */
-export function configureHarness(env = process.env) {
-    env.PI_CODING_AGENT_DIR ??= join(env.OPENMERIT_HOME ?? join(homedir(), ".openmerit"), "agent");
-    if (env.CLAUDE_CODE_OAUTH_TOKEN && !env.ANTHROPIC_API_KEY && !env.ANTHROPIC_OAUTH_TOKEN && !env.ANTHROPIC_AUTH_TOKEN) {
-        env.ANTHROPIC_OAUTH_TOKEN = env.CLAUDE_CODE_OAUTH_TOKEN;
-    }
-    env.PI_SKIP_VERSION_CHECK ??= "1";
-    return { agentDir: env.PI_CODING_AGENT_DIR };
-}
-/** Scout runs on the strongest connected general model unless the user names one. */
-const SCOUT_MODEL_PREFERENCE = [
-    "anthropic/claude-opus-5-5", "anthropic/claude-sonnet-5-5", "openai/gpt-6.1-sol", "openai/gpt-6-sol",
-    "openrouter/anthropic/claude-opus-5.5", "openrouter/anthropic/claude-sonnet-5.5", "openrouter/openai/gpt-6.1-sol",
-];
-export async function openRuntime() {
-    const runtime = await ModelRuntime.create({ allowModelNetwork: true, modelRefreshTimeoutMs: 20_000 });
-    const caller = { getModel: (provider, id) => runtime.getModel(provider, id), complete: (model, context, extra) => runtime.complete(model, context, extra) };
-    const models = async () => (await runtime.getAvailable()).map(model => ({ provider: model.provider, id: model.id, name: model.name, cost: model.cost }));
-    return { runtime, caller, models };
-}
-export async function pickScoutModel(runtime, requested) {
-    const available = await runtime.getAvailable();
-    const ref = (model) => `${model.provider}/${model.id}`;
-    if (requested) {
-        const model = available.find(item => ref(item) === requested);
-        if (!model)
-            throw new Error(`Scout model ${requested} is not available with the connected providers. Run \`openmerit status\`.`);
-        return model;
-    }
-    for (const wanted of SCOUT_MODEL_PREFERENCE) {
-        const model = available.find(item => ref(item) === wanted);
-        if (model)
-            return model;
-    }
-    const model = available[0];
-    if (!model)
-        throw new Error("No model provider is connected. Set an API key (for example OPENROUTER_API_KEY or ANTHROPIC_API_KEY) or run `openmerit login` for a subscription.");
-    return model;
-}

package/dist/pi/instructions.d.ts

@@ -1,2 +0,0 @@
-/** Scout's workflow, added to Pi's system prompt as the `openmerit_scout` section. */
-export declare const SCOUT_INSTRUCTIONS = "You are OpenMerit Scout. You recommend the model a user should run for a task, from measured results on the user's own labelled examples, not from memory.\n\nWorkflow when asked which model to use:\n1. Understand the task. If a codebase is present, find where the model is called (grep for model IDs, SDK calls, \"systemPrompt\", \"model:\"), the production prompt, whether it calls tools, its output format, and existing evals: fixtures, tests with expected outputs, Langfuse datasets (scout_connections lists them), or .openmerit/datasets/*.jsonl.\n2. Find labelled data. Prefer an existing Langfuse dataset or fixture set with expected outputs. Preview it with scout_dataset; Scout needs at least 5 items. If none exists, draft at least 6 items only from what the code, fixtures or the user state, save them with scout_save_dataset, and say they are drafts the user should check. Never invent labels you cannot justify.\n3. Choose the evaluator. If the project ships an evaluator module that runs its real harness (for example .openmerit/evaluator.mts, .openmerit/evaluator.ts or examples/*/evaluator.mts), use evaluator type \"module\". Otherwise use type \"pi\" with the production system prompt and the strictest grader that fits: \"json\" for structured outputs, \"exact\"/\"contains\" for short answers, \"judge\" with a precise rubric only for free text.\n4. Check candidates with scout_candidates. Filter only for real constraints: providers the user can run, tools when the call uses tools, a price ceiling the user gave, data-residency or vendor exclusions the user named. Do not narrow the pool from your own preferences.\n5. Run scout_run with the task, dataset and evaluator. It binary-searches the ECI-ranked candidates (pass = every sampled item), screens neighbours, runs 5 finalists (plus cheaper, stronger challengers) on every item 3 times, records each as a Langfuse experiment, and picks the winner from Langfuse: most passes, then lower measured cost per item, then lower ECI.\n6. Answer with the recommended model ID, its passes and cost per item, the runner-up, the Langfuse experiment links, the report path, and the limitations the report lists (sample size, untested skipped models, drafted labels, residency or policy gaps). If scout_run returns plan_only, report the plan and the command to run it.\n\nRules: measured results on the user's data outrank public benchmarks. Provider, auth or billing errors are not quality failures; report them. Never print API keys or tokens.";

package/dist/pi/instructions.js

@@ -1,12 +0,0 @@
-/** Scout's workflow, added to Pi's system prompt as the `openmerit_scout` section. */
-export const SCOUT_INSTRUCTIONS = `You are OpenMerit Scout. You recommend the model a user should run for a task, from measured results on the user's own labelled examples, not from memory.
-
-Workflow when asked which model to use:
-1. Understand the task. If a codebase is present, find where the model is called (grep for model IDs, SDK calls, "systemPrompt", "model:"), the production prompt, whether it calls tools, its output format, and existing evals: fixtures, tests with expected outputs, Langfuse datasets (scout_connections lists them), or .openmerit/datasets/*.jsonl.
-2. Find labelled data. Prefer an existing Langfuse dataset or fixture set with expected outputs. Preview it with scout_dataset; Scout needs at least 5 items. If none exists, draft at least 6 items only from what the code, fixtures or the user state, save them with scout_save_dataset, and say they are drafts the user should check. Never invent labels you cannot justify.
-3. Choose the evaluator. If the project ships an evaluator module that runs its real harness (for example .openmerit/evaluator.mts, .openmerit/evaluator.ts or examples/*/evaluator.mts), use evaluator type "module". Otherwise use type "pi" with the production system prompt and the strictest grader that fits: "json" for structured outputs, "exact"/"contains" for short answers, "judge" with a precise rubric only for free text.
-4. Check candidates with scout_candidates. Filter only for real constraints: providers the user can run, tools when the call uses tools, a price ceiling the user gave, data-residency or vendor exclusions the user named. Do not narrow the pool from your own preferences.
-5. Run scout_run with the task, dataset and evaluator. It binary-searches the ECI-ranked candidates (pass = every sampled item), screens neighbours, runs 5 finalists (plus cheaper, stronger challengers) on every item 3 times, records each as a Langfuse experiment, and picks the winner from Langfuse: most passes, then lower measured cost per item, then lower ECI.
-6. Answer with the recommended model ID, its passes and cost per item, the runner-up, the Langfuse experiment links, the report path, and the limitations the report lists (sample size, untested skipped models, drafted labels, residency or policy gaps). If scout_run returns plan_only, report the plan and the command to run it.
-
-Rules: measured results on the user's data outrank public benchmarks. Provider, auth or billing errors are not quality failures; report them. Never print API keys or tokens.`;

package/dist/scout/evaluators/module.d.ts

@@ -1,24 +0,0 @@
-import type { Evaluate } from "../search.js";
-/**
- * An evaluator that runs a candidate through the user's own harness, such as a deployed agent's
- * HTTP API, and checks the result against the labelled item. A module's default export is one of these.
- */
-export interface EvaluatorPlugin {
-    name: string;
-    /** Pi providers the harness can run, such as ["openrouter"]. Candidates are drawn only from these. */
-    providers?: string[];
-    /** The harness calls tools, so models without tool calls are not candidates. */
-    tools?: boolean;
-    /** Score name recorded per item in Langfuse, ideally the customer's own metric (e.g. `enrollment-correct`). */
-    passScore?: string;
-    /** Experiment metadata key `model_<role>`. */
-    role?: string;
-    /** Items evaluated at once within one probe. */
-    concurrency?: number;
-    create(context: {
-        env: Record<string, string | undefined>;
-        log: (line: string) => void;
-    }): Evaluate | Promise<Evaluate>;
-}
-/** Imports an evaluator module (`.mts`, `.ts`, `.mjs` or `.js`) from the user's project. Node.js 22.19 strips its types. */
-export declare function loadEvaluatorPlugin(path: string, cwd?: string): Promise<EvaluatorPlugin>;

package/dist/scout/evaluators/module.js

@@ -1,19 +0,0 @@
-var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
-    if (typeof path === "string" && /^\.\.?\//.test(path)) {
-        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
-            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
-        });
-    }
-    return path;
-};
-import { resolve } from "node:path";
-import { pathToFileURL } from "node:url";
-/** Imports an evaluator module (`.mts`, `.ts`, `.mjs` or `.js`) from the user's project. Node.js 22.19 strips its types. */
-export async function loadEvaluatorPlugin(path, cwd = process.cwd()) {
-    const loaded = await import(__rewriteRelativeImportExtension(pathToFileURL(resolve(cwd, path)).href));
-    const plugin = loaded.default;
-    if (!plugin || typeof plugin !== "object" || typeof plugin.create !== "function" || typeof plugin.name !== "string") {
-        throw new Error(`${path} must default-export an evaluator: { name, create(context) => evaluate }`);
-    }
-    return plugin;
-}

package/dist/scout/evaluators/pi.d.ts

@@ -1,53 +0,0 @@
-import type { Api, AssistantMessage, Context, Model } from "@earendil-works/pi-ai";
-import { type Evaluate } from "../search.js";
-/** What the Pi evaluator needs from Pi: a `ModelRuntime`, or an extension's `ctx.modelRegistry` through `fromRegistry`. */
-export interface ModelCaller {
-    getModel(provider: string, id: string): Model<Api> | undefined;
-    complete(model: Model<Api>, context: Context, options?: {
-        signal?: AbortSignal;
-        maxTokens?: number;
-        apiKey?: string;
-    }): Promise<AssistantMessage>;
-}
-export type Grader = 
-/** Normalized text equality with `expectedOutput` (or `expectedOutput[field]`). */
-{
-    type: "exact";
-    field?: string;
-}
-/** The output contains the expected text, case-insensitively. */
- | {
-    type: "contains";
-    field?: string;
-}
-/** The output's JSON contains every expected field with an equal value (strings compared case-insensitively, arrays as sets). */
- | {
-    type: "json";
-}
-/** A judge model answers PASS or FAIL for the output against the expected output and rubric. */
- | {
-    type: "judge";
-    rubric: string;
-    model?: string;
-};
-export interface PiTaskSpec {
-    /** The candidate's system prompt: the instructions the production model receives. */
-    systemPrompt: string;
-    /** Optional template for the user message; `{{input}}` is the item's input as text or JSON. */
-    inputTemplate?: string;
-    grader: Grader;
-    maxTokens?: number;
-}
-/** The first JSON object or array in a reply, with code fences removed. */
-export declare function extractJson(reply: string): unknown;
-/** Every expected field matches: objects by key, arrays as sets, strings case-insensitively. */
-export declare function jsonMatches(expected: unknown, actual: unknown): boolean;
-/**
- * Runs each item as one model call through Pi, so any provider Pi connects (API key or
- * subscription) can be a candidate, then grades the reply. Tool-using agents need an
- * evaluator module that runs the real harness instead.
- */
-export declare function piEvaluator(caller: ModelCaller, spec: PiTaskSpec, options?: {
-    judgeModel?: string;
-    retryDelaysMs?: number[];
-}): Evaluate;

package/dist/scout/evaluators/pi.js

@@ -1,117 +0,0 @@
-import { setTimeout as delay } from "node:timers/promises";
-import { CandidateUnavailable } from "../search.js";
-const text = (message) => message.content.filter(part => part.type === "text").map(part => part.text).join("").trim();
-const asText = (value) => (typeof value === "string" ? value : JSON.stringify(value, null, 2));
-const normalize = (value) => value.toLowerCase().replace(/\s+/g, " ").replace(/[.!\s]+$/, "").trim();
-/** The first JSON object or array in a reply, with code fences removed. */
-export function extractJson(reply) {
-    const fenced = /```(?:json)?\s*([\s\S]*?)```/.exec(reply)?.[1] ?? reply;
-    const start = fenced.search(/[[{]/);
-    if (start < 0)
-        return undefined;
-    for (let end = fenced.length; end > start; end--) {
-        try {
-            return JSON.parse(fenced.slice(start, end));
-        }
-        catch { /* shorten */ }
-    }
-    return undefined;
-}
-/** Every expected field matches: objects by key, arrays as sets, strings case-insensitively. */
-export function jsonMatches(expected, actual) {
-    if (expected === null || typeof expected !== "object") {
-        if (typeof expected === "string" && typeof actual === "string")
-            return normalize(expected) === normalize(actual);
-        return expected === actual;
-    }
-    if (Array.isArray(expected)) {
-        return Array.isArray(actual) && expected.length === actual.length && expected.every(item => actual.some(other => jsonMatches(item, other)));
-    }
-    if (!actual || typeof actual !== "object" || Array.isArray(actual))
-        return false;
-    return Object.entries(expected).every(([key, value]) => jsonMatches(value, actual[key]));
-}
-const field = (expected, name) => asText(name && expected && typeof expected === "object" ? expected[name] : expected);
-/** Messages with a provider error are infrastructure failures, not quality verdicts. */
-function ensureOk(message, what) {
-    if (message.stopReason === "error" || message.stopReason === "aborted")
-        throw new Error(`${what}: ${message.errorMessage ?? message.stopReason}`);
-}
-const status = (message) => Number(/^(\d{3})\b/.exec(message.errorMessage ?? "")?.[1] ?? 0);
-/**
- * One call with retries for transient provider errors (429, 5xx, network), waiting 5, 20 and 60 s.
- * A model the provider rejects (400, 404) or the balance cannot cover (402) is unavailable, not failed.
- */
-async function call(caller, model, context, options, what, unavailable, waits) {
-    for (const wait of [...waits, -1]) {
-        const reply = await caller.complete(model, context, options);
-        if (reply.stopReason !== "error")
-            return reply;
-        const code = status(reply);
-        if (unavailable && [400, 402, 404, 413, 422].includes(code))
-            throw new CandidateUnavailable(`${what}: ${(reply.errorMessage ?? "").slice(0, 240)}`);
-        if (wait < 0 || (code !== 0 && code !== 429 && code < 500))
-            return reply;
-        await delay(wait, undefined, options.signal ? { signal: options.signal } : undefined);
-    }
-    throw new Error("unreachable");
-}
-function resolve(caller, ref) {
-    const slash = ref.indexOf("/");
-    const model = slash > 0 ? caller.getModel(ref.slice(0, slash), ref.slice(slash + 1)) : undefined;
-    if (!model)
-        throw new Error(`Model ${ref} is not available with the connected providers`);
-    return model;
-}
-/**
- * Runs each item as one model call through Pi, so any provider Pi connects (API key or
- * subscription) can be a candidate, then grades the reply. Tool-using agents need an
- * evaluator module that runs the real harness instead.
- */
-export function piEvaluator(caller, spec, options = {}) {
-    const waits = options.retryDelaysMs ?? [5_000, 20_000, 60_000];
-    return async (candidate, item, context) => {
-        const model = resolve(caller, candidate.model);
-        const input = asText(item.input);
-        const user = spec.inputTemplate ? spec.inputTemplate.replaceAll("{{input}}", input) : input;
-        const reply = await call(caller, model, { systemPrompt: spec.systemPrompt, messages: [{ role: "user", content: user, timestamp: Date.now() }] }, { ...(context.signal ? { signal: context.signal } : {}), maxTokens: spec.maxTokens ?? 4096 }, candidate.model, true, waits);
-        ensureOk(reply, candidate.model);
-        const output = text(reply);
-        const usage = { input: reply.usage.input, output: reply.usage.output, total: reply.usage.totalTokens };
-        const graded = await grade(spec.grader, item, output, caller, options.judgeModel, waits, context.signal);
-        return { pass: graded.pass, comment: graded.comment, output, costUsd: reply.usage.cost.total, usage, ...(reply.durationMs ? { latencyMs: reply.durationMs } : {}) };
-    };
-}
-async function grade(grader, item, output, caller, judgeModel, waits, signal) {
-    switch (grader.type) {
-        case "exact": {
-            const expected = field(item.expectedOutput, grader.field);
-            return { pass: normalize(output) === normalize(expected), comment: `expected "${expected.slice(0, 80)}"` };
-        }
-        case "contains": {
-            const expected = field(item.expectedOutput, grader.field);
-            return { pass: normalize(output).includes(normalize(expected)), comment: `expected to contain "${expected.slice(0, 80)}"` };
-        }
-        case "json": {
-            const parsed = extractJson(output);
-            if (parsed === undefined)
-                return { pass: false, comment: "reply has no JSON" };
-            const pass = jsonMatches(item.expectedOutput, parsed);
-            return { pass, comment: pass ? "every expected field matches" : `expected ${JSON.stringify(item.expectedOutput).slice(0, 160)}` };
-        }
-        case "judge": {
-            const ref = grader.model ?? judgeModel;
-            if (!ref)
-                throw new Error("The judge grader needs a judge model (--judge-model or the Scout model)");
-            const verdict = await call(caller, resolve(caller, ref), {
-                systemPrompt: "You grade one model output against a labelled expected output. Reply with JSON only: {\"pass\": true|false, \"reason\": \"<one sentence>\"}. Pass only if the output satisfies the rubric and agrees with the expected output on every point the rubric names.",
-                messages: [{ role: "user", content: `Rubric:\n${grader.rubric}\n\nInput:\n${asText(item.input)}\n\nExpected output:\n${asText(item.expectedOutput)}\n\nModel output:\n${output}`, timestamp: Date.now() }],
-            }, { ...(signal ? { signal } : {}), maxTokens: 1024 }, `judge ${ref}`, false, waits);
-            ensureOk(verdict, `judge ${ref}`);
-            const parsed = extractJson(text(verdict));
-            if (typeof parsed?.pass !== "boolean")
-                throw new Error(`Judge ${ref} returned no verdict`);
-            return { pass: parsed.pass, comment: `judge: ${typeof parsed.reason === "string" ? parsed.reason : ""}` };
-        }
-    }
-}

package/dist/scout/pipeline.d.ts

@@ -1,111 +0,0 @@
-import type { LangfuseConnector } from "../connectors/langfuse/index.js";
-import type { Candidate } from "./candidates.js";
-import { type EvalItem, type Evaluate, type SearchResult } from "./search.js";
-export interface DatasetSource {
-    /** An existing Langfuse dataset, such as the customer's `sahra-enrollments`. */
-    langfuseDataset?: string;
-    /** Local labelled items; pushed to Langfuse as `name` before evaluation so every run is an experiment on it. */
-    local?: {
-        name: string;
-        description?: string;
-        items: EvalItem[];
-    };
-}
-export interface ScoutRunOptions {
-    task: string;
-    dataset: DatasetSource;
-    candidates: Candidate[];
-    evaluate: Evaluate;
-    /** Score name each recorded item carries; the pass rule reads it back from Langfuse. */
-    passScore?: string;
-    /** Experiment metadata key for the candidate: `model_<role>`. */
-    role?: string;
-    langfuse: LangfuseConnector | null;
-    sampleSize?: number;
-    finalists?: number;
-    /** See SearchOptions.challengers. */
-    challengers?: number;
-    /** See SearchOptions.repeats. */
-    repeats?: number;
-    concurrency?: number;
-    log?: (line: string) => void;
-    signal?: AbortSignal;
-    /** Injected for tests. */
-    pickIndex?: (remaining: number) => number;
-    langfuseTimeoutMs?: number;
-}
-export interface FinalVerdict {
-    model: string;
-    eci: number;
-    eciSource: string;
-    listPrice: number | null;
-    /** Passes as Langfuse reports them; local passes when Langfuse is not connected. */
-    passes: number;
-    /** Item runs: dataset items × repeats. */
-    total: number;
-    /** Runs of each item. */
-    runs: number;
-    /** Passes per item over the repeats, e.g. { "APP-2005": 2 } when it passed 2 of 3 runs. */
-    passesByItem: Record<string, number>;
-    /** Mean measured cost per item run in USD: Langfuse's recorded generation cost when available. */
-    costPerItemUsd: number | null;
-    localPasses: number;
-    verifiedInLangfuse: boolean;
-    missingScores: string[];
-    /** One Langfuse experiment per repeat (Langfuse allows one result per dataset item per experiment). */
-    experimentIds: string[];
-    experimentId: string | null;
-    experimentUrl: string | null;
-    /** Items that failed at least one run. */
-    failedItems: string[];
-}
-export interface ScoutReport {
-    scout: "OpenMerit Scout";
-    runId: string;
-    task: string;
-    createdAt: string;
-    dataset: {
-        name: string;
-        source: "langfuse" | "local";
-        size: number;
-        datasetId: string | null;
-        url: string | null;
-    };
-    candidates: number;
-    search: Omit<SearchResult, "ordered" | "trials" | "finals" | "unavailable"> & {
-        unavailable: SearchResult["unavailable"];
-        trials: {
-            stage: string;
-            model: string;
-            eci: number;
-            eciSource: string;
-            passes: number;
-            total: number;
-            accepted: boolean;
-            costUsd: number | null;
-            experimentUrl: string | null;
-        }[];
-    };
-    finals: FinalVerdict[];
-    recommendation: FinalVerdict | null;
-    runnerUp: FinalVerdict | null;
-    why: string;
-    spend: {
-        measuredUsd: number;
-        unknownCostItems: number;
-        evaluations: number;
-        /** OpenRouter account credit used between the start and the end of the run (account-wide: concurrent use counts too). */
-        openRouterCreditsUsd?: number;
-    };
-    langfuse: {
-        baseUrl: string;
-        project: string | null;
-        experiments: number;
-    } | null;
-    limitations: string[];
-}
-/**
- * Scout end to end: dataset → binary search over ECI-ranked candidates → finals on every item,
- * each probe and final recorded as a Langfuse experiment → the winner from Langfuse's results.
- */
-export declare function runScout(options: ScoutRunOptions): Promise<ScoutReport>;