openmerit 0.4.1 → 0.4.2

package/dist/scout/scan.js

@@ -0,0 +1,169 @@
+// Step 1 of Scout: find model and endpoint usages in the repository without changing it. Reads only
+// files Git does not ignore (or, outside Git, skips dependency and build directories), minus the
+// repository's `scan.exclude` patterns. A JSON file that names many models is data (a catalogue,
+// saved results, a dataset), not configuration, and is skipped. Snippets are redacted: anything
+// shaped like a key or token is replaced before it is shown, stored or sent.
+import { execFileSync } from "node:child_process";
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { extname, join, matchesGlob, relative } from "node:path";
+import { catalogueIndex } from "../catalogue/catalogue.js";
+const TEXT = new Set([".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rb", ".java", ".kt", ".rs", ".php", ".cs", ".swift",
+    ".json", ".jsonc", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env", ".example", ".sh", ".tf"]);
+const SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", "out", ".next", ".venv", "venv", "__pycache__", "target", ".openmerit", "coverage", ".amp"]);
+const SKIP_FILES = /(^|\/)(package-lock\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?|poetry\.lock|Cargo\.lock|go\.sum)$/;
+const MAX_BYTES = 512_000;
+/** More model usages than this in one JSON file make it data rather than configuration. */
+export const MAX_DATA_FILE_MODELS = 20;
+/** True when `file` matches a pattern, or sits in a directory a pattern names. */
+const excluded = (file, patterns) => patterns.some(pattern => matchesGlob(file, pattern) || matchesGlob(file, `${pattern.replace(/\/+$/, "")}/**`));
+const SECRET_PATTERNS = [
+    /\b(sk|pk|rk)-(?:proj-|ant-|or-|lf-|live-|test-)?[A-Za-z0-9_-]{8,}/g,
+    /\bAKIA[0-9A-Z]{12,}\b/g, /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g, /\bxox[abpr]-[A-Za-z0-9-]{10,}\b/g, /\bAIza[0-9A-Za-z_-]{20,}\b/g,
+    /(Bearer\s+)[A-Za-z0-9._~+/-]{12,}=*/gi,
+    /((?:api[_-]?key|apikey|token|secret|password|authorization)\w*["']?\s*[:=]\s*["'])[^"'\s]{4,}(["'])/gi,
+];
+/** Replaces key- and token-shaped text with [redacted]. */
+export function redact(text) {
+    let out = text;
+    for (const pattern of SECRET_PATTERNS)
+        out = out.replace(pattern, (match, ...groups) => {
+            const prefix = typeof groups[0] === "string" && /[:=\s]$/.test(groups[0]) ? groups[0] : typeof groups[0] === "string" && /^Bearer/i.test(groups[0]) ? groups[0] : "";
+            const suffix = typeof groups[1] === "string" && /^["']$/.test(groups[1]) ? groups[1] : "";
+            return prefix || suffix ? `${prefix}[redacted]${suffix}` : "[redacted]";
+        });
+    return out;
+}
+export function listFiles(root) {
+    try {
+        const out = execFileSync("git", ["ls-files", "-z", "--cached", "--others", "--exclude-standard"], { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 64 * 1024 * 1024 });
+        return out.split("\0").filter(Boolean).filter(file => !file.split("/").some(part => SKIP_DIRS.has(part)));
+    }
+    catch {
+        const files = [];
+        const walk = (dir) => {
+            for (const entry of readdirSync(dir, { withFileTypes: true })) {
+                if (entry.isDirectory()) {
+                    if (!SKIP_DIRS.has(entry.name))
+                        walk(join(dir, entry.name));
+                }
+                else if (entry.isFile())
+                    files.push(relative(root, join(dir, entry.name)));
+            }
+        };
+        walk(root);
+        return files;
+    }
+}
+const isText = (file) => TEXT.has(extname(file)) || /(^|\/)\.env(\.[\w-]+)?$/.test(file) || /(^|\/)Dockerfile$/.test(file);
+const STRING = /(["'`])((?:\\.|(?!\1)[^\\\n])*)\1/g;
+const MODEL_KEY = /\b(?:\w*model\w*|engine|deployment)\b["']?\s*(?::|=|=>|,)\s*/i;
+const DYNAMIC = /\b(\w*model\w*)\b["']?\s*(?::|=(?!=))\s*([^"'`\s,;)}\]][^,;)}\]\n]*)/i;
+const ENV = /(?:process\.env\.([A-Z][A-Z0-9_]*)|process\.env\[["']([A-Z][A-Z0-9_]*)["']\]|import\.meta\.env\.([A-Z][A-Z0-9_]*)|os\.environ(?:\.get)?\(?\[?["']([A-Z][A-Z0-9_]*)["']|os\.getenv\(["']([A-Z][A-Z0-9_]*)["']|Deno\.env\.get\(["']([A-Z][A-Z0-9_]*)["']|\$\{?([A-Z][A-Z0-9_]*MODEL[A-Z0-9_]*)\}?)/;
+const URL_PATTERN = /https?:\/\/[^\s"'`<>)]+/g;
+const ENDPOINT_HOSTS = /(api\.openai\.com|openrouter\.ai|api\.anthropic\.com|generativelanguage\.googleapis\.com|openai\.azure\.com|api\.together\.xyz|api\.groq\.com|api\.mistral\.ai|api\.deepseek\.com|api\.fireworks\.ai|bedrock|localhost:\d+\/v1|\/v1\/?$)/;
+const BASE_URL_KEY = /\b(base_?url|baseURL|api_?base|endpoint)\b["']?\s*(?::|=(?!=))\s*([^\n,;]+)/i;
+/** Scans the repository. Deterministic; no model is called. */
+export function scanRepo(root, catalogue, options = {}) {
+    const index = catalogueIndex(catalogue);
+    const usages = [];
+    const skipped = [];
+    const files = listFiles(root).filter(file => isText(file) && !SKIP_FILES.test(file) && !excluded(file, options.exclude ?? []));
+    let scanned = 0;
+    for (const file of files) {
+        const path = join(root, file);
+        let text;
+        try {
+            if (statSync(path).size > MAX_BYTES)
+                continue;
+            text = readFileSync(path, "utf8");
+        }
+        catch {
+            continue;
+        }
+        if (text.includes("\0"))
+            continue;
+        scanned++;
+        const before = usages.length;
+        const lines = text.split("\n");
+        lines.forEach((raw, lineIndex) => {
+            if (raw.length > 2000)
+                return;
+            const line = lineIndex + 1;
+            const snippet = redact(raw.trim()).slice(0, 240);
+            const add = (usage) => {
+                if (usages.some(item => item.file === file && item.line === line && item.kind === usage.kind && item.value === usage.value))
+                    return;
+                usages.push({ id: `u${usages.length + 1}`, file, line, snippet, ...usage });
+            };
+            let literalModel = false;
+            for (const match of raw.matchAll(STRING)) {
+                const value = match[2];
+                // catalogue:<endpoint>:<model> is how OpenMerit's own saved results name a model, not a model an app calls.
+                if (value.length < 3 || value.length > 120 || /\s/.test(value) || value.startsWith("catalogue:"))
+                    continue;
+                const model = index.find(value);
+                const keyed = MODEL_KEY.test(raw.slice(Math.max(0, match.index - 40), match.index));
+                if (model && (keyed || value.includes("/") || /\d/.test(value))) {
+                    literalModel = true;
+                    const env = ENV.exec(raw);
+                    const envVar = env?.slice(1).find(Boolean);
+                    add({ kind: "model", value: redact(value), dynamic: !!envVar, ...(envVar ? { envVar } : {}), catalogueId: model.id });
+                }
+                else if (keyed && /^[\w.:/@-]+$/.test(value) && /[a-z]/i.test(value) && /\d|gpt|claude|gemini|llama|mistral|qwen|deepseek/i.test(value) && !/^https?:/.test(value)) {
+                    literalModel = true;
+                    add({ kind: "model", value: redact(value), dynamic: false, catalogueId: null });
+                }
+            }
+            if (!literalModel) {
+                const dynamic = DYNAMIC.exec(raw);
+                if (dynamic && !/^(true|false|null|undefined|None|\d)/.test(dynamic[2]) && !/\b(type|interface|import|from|def|class)\b/.test(raw.slice(0, dynamic.index)) && !/^\s*(\/\/|#|\*)/.test(raw)
+                    && !/[:]\s*(string|number|Model|ModelRef)\b/.test(raw)) {
+                    const env = ENV.exec(raw);
+                    const envVar = env?.slice(1).find(Boolean);
+                    let expression = dynamic[2].trim().replace(/[\s,;]+$/, "").slice(0, 120);
+                    expression += ")".repeat(Math.max(0, (expression.match(/\(/g) ?? []).length - (expression.match(/\)/g) ?? []).length));
+                    if (envVar || /^[A-Za-z_$][\w$.[\]"'()]*$/.test(expression))
+                        add({ kind: "model", value: redact(expression), dynamic: true, ...(envVar ? { envVar } : {}), catalogueId: null });
+                }
+            }
+            for (const match of raw.matchAll(URL_PATTERN)) {
+                const url = match[0].replace(/[.,]+$/, "");
+                if (ENDPOINT_HOSTS.test(url))
+                    add({ kind: "endpoint", value: redact(url), dynamic: false, catalogueId: null });
+            }
+            const base = BASE_URL_KEY.exec(raw);
+            if (base && !URL_PATTERN.test(base[2])) {
+                const env = ENV.exec(base[2]);
+                const envVar = env?.slice(1).find(Boolean);
+                if (envVar)
+                    add({ kind: "endpoint", value: redact(base[2].trim().replace(/[\s,;]+$/, "").slice(0, 120)), dynamic: true, envVar, catalogueId: null });
+            }
+            URL_PATTERN.lastIndex = 0;
+        });
+        // This file's usages are the last ones added, so dropping them keeps usage IDs sequential.
+        if (extname(file) === ".json" && usages.slice(before).filter(usage => usage.kind === "model").length > MAX_DATA_FILE_MODELS) {
+            usages.length = before;
+            skipped.push(file);
+            scanned--;
+        }
+    }
+    const groups = new Map();
+    for (const usage of usages.filter(item => item.kind === "model" && !item.dynamic || item.kind === "model" && item.catalogueId)) {
+        const key = usage.catalogueId ?? `?${usage.value}`;
+        const group = groups.get(key) ?? { catalogueId: usage.catalogueId, value: usage.catalogueId ?? usage.value, usageIds: [] };
+        group.usageIds.push(usage.id);
+        groups.set(key, group);
+    }
+    return { files: scanned, usages, models: [...groups.values()].sort((a, b) => b.usageIds.length - a.usageIds.length || a.value.localeCompare(b.value)), skipped };
+}
+/** A few lines around a usage, redacted, for the harness. */
+export function context(root, usage, radius = 6) {
+    try {
+        const lines = readFileSync(join(root, usage.file), "utf8").split("\n");
+        const start = Math.max(0, usage.line - 1 - radius);
+        return lines.slice(start, usage.line + radius).map((line, offset) => `${start + offset + 1}${start + offset + 1 === usage.line ? ">" : " "} ${redact(line).slice(0, 300)}`).join("\n");
+    }
+    catch {
+        return "";
+    }
+}

package/dist/scout/search.d.ts

@@ -1,110 +1,37 @@
-import type { Candidate } from "./candidates.js";
-/**
- * Scout Model Binary Search, from the ScoutBM prototype:
- * 1. Rank candidates by ECI, ascending, and draw `sampleSize` labelled items once.
- * 2. Binary search: a model "performs well" only when it passes every sampled item. Passing moves
- *    to weaker (cheaper-on-capability) models, failing moves to stronger ones. The search pick is
- *    the weakest model that passed.
- * 3. Screen the nearest neighbours on the same samples (weaker first at equal distance; once a
- *    weaker one fails, only stronger ones) until there are `finalists` passing models.
- * 4. Finals: every finalist runs every dataset item. The caller decides the winner from the
- *    recorded results (Langfuse), falling back to `orderFinals`.
- */
-export interface EvalItem {
-    id: string;
-    input: unknown;
-    expectedOutput: unknown;
-    metadata?: Record<string, unknown> | null;
-}
-export interface Outcome {
-    itemId: string;
-    pass: boolean;
-    /** What the candidate produced, recorded as the experiment item's output. */
-    output?: unknown;
-    /** Measured provider cost of this item, when known. */
-    costUsd?: number | null;
-    latencyMs?: number;
-    usage?: {
-        input: number;
-        output: number;
-        total: number;
-    } | null;
-    /** Why the item passed or failed, in a sentence. */
-    comment?: string;
-    startMs?: number;
-    endMs?: number;
-    /** Which repeat of a final this was, from 1. Screens run once and leave it unset. */
-    repeat?: number;
-}
-export type Stage = "search" | "neighbour" | "challenger" | "final";
-/**
- * Thrown by an evaluator when this one candidate cannot run, such as a provider rejecting the model
- * (400/404) or the balance not covering its request (402). The candidate is skipped and listed as
- * untested; the search goes on without it. Any other throw stops the search.
- */
 export declare class CandidateUnavailable extends Error {
     readonly name = "CandidateUnavailable";
 }
-/** Runs one item on one candidate. Throw on transport, auth, billing or harness failures: they never count as quality failures. */
-export type Evaluate = (candidate: Candidate, item: EvalItem, context: {
-    stage: Stage;
-    signal?: AbortSignal;
-}) => Promise<Omit<Outcome, "itemId">>;
-export interface Trial {
-    stage: Stage;
-    candidate: Candidate;
-    outcomes: Outcome[];
-    passes: number;
-    /** Screens: every sample passed. Finals: always true. */
-    accepted: boolean;
-    /** Runs of each item: 1 for screens, `repeats` for finals. `outcomes` holds items × runs. */
-    runs: number;
-}
-export interface SearchOptions {
-    dataset: readonly EvalItem[];
-    /** Ascending by ECI, as `rankCandidates` returns them. */
-    candidates: readonly Candidate[];
-    evaluate: Evaluate;
-    sampleSize?: number;
-    finalists?: number;
-    /** Items evaluated at once within one probe. Default 5. */
-    concurrency?: number;
-    /** Runs of every item per finalist, so a model that passes by chance does not tie a reliable one (default 3). */
-    repeats?: number;
-    /** Untested stronger models no dearer than the leading finalist to screen after the finals (default 3; 0 turns it off). */
-    challengers?: number;
-    /** Called after every probe, screen and final, for example to record it in Langfuse. A throw stops the search. */
-    onTrial?: (trial: Trial) => Promise<void> | void;
-    pickIndex?: (remaining: number) => number;
-    signal?: AbortSignal;
-    log?: (line: string) => void;
+export declare const SAMPLE_SIZE = 5;
+export declare const FINALISTS = 5;
+export declare const CHALLENGERS = 3;
+export interface Ranked {
+    key: string;
+    eci: number;
+    price: number | null;
 }
-export interface SearchResult {
-    status: "selected" | "no_passing_model" | "stopped";
-    error?: string;
-    criterion: string;
-    sampleIds: string[];
-    ordered: Candidate[];
+export type Stage = "search" | "neighbour" | "challenger";
+export interface SearchResult<T extends Ranked> {
     /** The weakest model that passed every sample during the binary search. */
-    selected: Candidate | null;
-    trials: Trial[];
-    finals: Trial[];
-    /** Candidates skipped because they could not run; untested, not failed. */
+    selected: T | null;
+    /** The pick and its passing neighbours, nearest first. */
+    finalists: T[];
+    trials: {
+        stage: Stage;
+        candidate: T;
+        accepted: boolean;
+    }[];
+    /** Skipped because they could not run: untested, not failed. */
     unavailable: {
-        model: string;
+        candidate: T;
         stage: Stage;
         reason: string;
     }[];
     shortfall?: string;
-    finalRule: string;
-    limitation: string;
 }
-export declare const SAMPLE_SIZE = 5;
-export declare const FINALISTS = 5;
-export declare function scoutSearch(input: SearchOptions): Promise<SearchResult>;
+/** Ascending ECI; equal scores fall back to the key purely for repeatability. */
+export declare const order: <T extends Ranked>(candidates: readonly T[]) => T[];
 /** Models other than the pick, nearest by rank first; at equal distance the weaker one comes first. */
-export declare function outward(ordered: readonly Candidate[], pick: Candidate): Candidate[];
-/** Mean measured cost per item, or null when any item's cost is unknown. */
-export declare function meanCost(outcomes: readonly Pick<Outcome, "costUsd">[]): number | null;
-/** Most passes; then lower measured cost per item (list price only when a cost is unknown); then lower ECI. */
-export declare function compareFinals(a: Trial, b: Trial): number;
+export declare function outward<T extends Ranked>(ordered: readonly T[], pick: T): T[];
+export declare function binarySearch<T extends Ranked>(candidates: readonly T[], screen: (candidate: T, stage: Stage) => Promise<boolean>, finalistCount?: number): Promise<SearchResult<T>>;
+/** Untested models stronger than the leader and no dearer by list price, cheapest first. */
+export declare function challengers<T extends Ranked>(ordered: readonly T[], leader: T, tried: ReadonlySet<string>, count?: number): T[];

package/dist/scout/search.js

@@ -1,89 +1,51 @@
-import { randomInt } from "node:crypto";
-/**
- * Thrown by an evaluator when this one candidate cannot run, such as a provider rejecting the model
- * (400/404) or the balance not covering its request (402). The candidate is skipped and listed as
- * untested; the search goes on without it. Any other throw stops the search.
- */
+// Scout Model Binary Search, from the ScoutBM prototype:
+// 1. Rank candidates by ECI, ascending. Screens all run on the same few sampled items.
+// 2. Binary search: a model "performs well" only when it passes every sampled item. Passing moves
+//    to weaker (cheaper-on-capability) models, failing moves to stronger ones. The search pick is
+//    the weakest model that passed.
+// 3. Screen the nearest neighbours on the same samples (weaker first at equal distance; once a
+//    weaker one fails, only stronger ones) until there are `finalists` passing models.
+// 4. Challengers: after the finals, a stronger model at the same or a lower list price than the
+//    leading finalist could be the better buy, but the neighbour rule never reaches it; screen the
+//    cheapest few. The caller runs finals and challengers on every item.
+// A screen throws CandidateUnavailable when the model cannot run at all (the provider rejects it):
+// it is skipped as untested and the search goes on. Any other throw stops the search.
 export class CandidateUnavailable extends Error {
     name = "CandidateUnavailable";
 }
-/** By name, so an evaluator module can throw its own `CandidateUnavailable` without importing OpenMerit. */
-const unavailableError = (error) => error instanceof Error && error.name === "CandidateUnavailable";
+const unavailable = (error) => error instanceof Error && error.name === "CandidateUnavailable";
 export const SAMPLE_SIZE = 5;
 export const FINALISTS = 5;
-export async function scoutSearch(input) {
-    const sampleSize = input.sampleSize ?? SAMPLE_SIZE;
-    const finalistCount = input.finalists ?? FINALISTS;
-    if (input.dataset.length < sampleSize)
-        throw new Error(`Scout needs an eval dataset with at least ${sampleSize} labelled items; it has ${input.dataset.length}`);
-    if (new Set(input.dataset.map(item => item.id)).size !== input.dataset.length)
-        throw new Error("Dataset item IDs must be unique");
-    if (!input.candidates.length)
-        throw new Error("No ECI-ranked candidate models are runnable with the connected providers");
-    if (new Set(input.candidates.map(item => item.model)).size !== input.candidates.length)
-        throw new Error("Candidate models must be unique");
-    const ordered = [...input.candidates].sort((a, b) => a.eci - b.eci || a.model.localeCompare(b.model));
-    const pool = [...input.dataset];
-    const samples = [];
-    const pick = input.pickIndex ?? randomInt;
-    for (let count = 0; count < sampleSize; count++) {
-        const index = pick(pool.length);
-        if (!Number.isInteger(index) || index < 0 || index >= pool.length)
-            throw new Error("Invalid random sample index");
-        samples.push(pool.splice(index, 1)[0]);
+export const CHALLENGERS = 3;
+/** Ascending ECI; equal scores fall back to the key purely for repeatability. */
+export const order = (candidates) => [...candidates].sort((a, b) => a.eci - b.eci || a.key.localeCompare(b.key));
+/** Models other than the pick, nearest by rank first; at equal distance the weaker one comes first. */
+export function outward(ordered, pick) {
+    const index = ordered.findIndex(item => item.key === pick.key);
+    if (index < 0)
+        throw new Error("The pick is not in the ranking");
+    const out = [];
+    for (let distance = 1; out.length < ordered.length - 1; distance++) {
+        if (ordered[index - distance])
+            out.push(ordered[index - distance]);
+        if (ordered[index + distance])
+            out.push(ordered[index + distance]);
     }
-    const repeats = input.repeats ?? 3;
-    if (!Number.isInteger(repeats) || repeats < 1)
-        throw new Error("repeats must be a positive integer");
-    const log = input.log ?? (() => { });
+    return out;
+}
+export async function binarySearch(candidates, screen, finalistCount = FINALISTS) {
+    const ordered = order(candidates);
     const trials = [];
-    const finals = [];
-    const unavailable = [];
+    const skipped = [];
     let selected = null;
-    const result = (status, error) => ({
-        status, ...(error ? { error } : {}),
-        criterion: `${sampleSize}/${sampleSize} sampled items pass`, sampleIds: samples.map(item => item.id), ordered, selected, trials, finals, unavailable,
-        ...(status === "selected" && finals.length < finalistCount ? { shortfall: `Only ${finals.length} finalists: no further model passed the screen within the neighbour rule (once a weaker neighbour fails, only stronger models are tried)` } : {}),
-        finalRule: `Finalists are the search pick and the nearest models by ECI that also pass the same ${sampleSize} samples (${finalistCount} in all; once a weaker neighbour fails, only stronger ones are tried), plus up to ${input.challengers ?? 3} untested challengers with a higher ECI and no higher list price than the leading finalist that pass the samples. Each runs every dataset item ${repeats} times; the winner has the most passes over all runs in Langfuse, then the lower measured cost per run, then the lower ECI.`,
-        limitation: `Binary search assumes task quality rises with ECI; models it skips are untested. Screens run each sample once, so a model can be screened out by one unlucky run. ${repeats} runs per final item narrow chance differences but do not remove them, and the result is only as representative as the eval dataset.`,
-    });
-    const run = async (stage, candidate, items) => {
-        const runs = stage === "final" ? repeats : 1;
-        // Repeat-major order, so an interrupted final has whole repeats rather than one item run many times.
-        const tasks = Array.from({ length: runs }, (_, r) => items.map(item => ({ item, repeat: r + 1 }))).flat();
-        const outcomes = new Array(tasks.length);
-        let next = 0;
-        const worker = async () => {
-            while (next < tasks.length) {
-                const index = next++;
-                const { item, repeat } = tasks[index];
-                const startMs = Date.now();
-                const outcome = await input.evaluate(candidate, item, { stage, ...(input.signal ? { signal: input.signal } : {}) });
-                if (typeof outcome?.pass !== "boolean")
-                    throw new Error("Evaluator must return { pass: boolean } or throw on infrastructure failure");
-                outcomes[index] = { startMs, endMs: Date.now(), latencyMs: Date.now() - startMs, ...outcome, itemId: item.id, ...(runs > 1 ? { repeat } : {}) };
-            }
-        };
-        await Promise.all(Array.from({ length: Math.min(input.concurrency ?? 5, tasks.length) }, worker));
-        const passes = outcomes.filter(item => item.pass).length;
-        const trial = { stage, candidate, outcomes, passes, runs, accepted: stage === "final" || passes === tasks.length };
-        log(`${stage.padEnd(9)} ${candidate.model} (ECI ${candidate.eci.toFixed(1)}): ${passes}/${tasks.length}${runs > 1 ? ` over ${runs} runs of ${items.length} items` : ""}${stage === "final" ? "" : trial.accepted ? " pass" : " fail"}`);
-        await input.onTrial?.(trial);
-        return trial;
-    };
-    const failure = (stage, candidate, error) => result("stopped", `${stage} ${candidate.model}: ${error instanceof Error ? error.message : String(error)}`);
-    const skip = (stage, candidate, error) => {
-        unavailable.push({ model: candidate.model, stage, reason: error.message });
-        log(`${"skipped".padEnd(9)} ${candidate.model}: ${error.message.slice(0, 200)}`);
-    };
     let lower = 0, upper = ordered.length - 1;
     while (lower <= upper) {
         const middle = Math.floor((lower + upper) / 2);
         const candidate = ordered[middle];
         try {
-            const trial = await run("search", candidate, samples);
-            trials.push(trial);
-            if (trial.accepted) {
+            const accepted = await screen(candidate, "search");
+            trials.push({ stage: "search", candidate, accepted });
+            if (accepted) {
                 selected = candidate;
                 upper = middle - 1;
             }
@@ -91,20 +53,19 @@ export async function scoutSearch(input) {
                 lower = middle + 1;
         }
         catch (error) {
-            if (!unavailableError(error))
-                return failure("search", candidate, error);
+            if (!unavailable(error))
+                throw error;
             // Untested, not failed: drop it from the ranking and search the same range without it.
-            skip("search", candidate, error);
+            skipped.push({ candidate, stage: "search", reason: error.message });
             ordered.splice(middle, 1);
             upper -= 1;
         }
     }
     if (!selected)
-        return result("no_passing_model");
-    const screened = new Map(trials.map(trial => [trial.candidate.model, trial.accepted]));
+        return { selected: null, finalists: [], trials, unavailable: skipped };
+    const screened = new Map(trials.map(trial => [trial.candidate.key, trial.accepted]));
     const finalists = [selected];
     const pickAt = ordered.indexOf(selected);
-    const skipped = new Set();
     let lookBelow = true;
     for (const candidate of outward(ordered, selected)) {
         if (finalists.length === finalistCount)
@@ -112,20 +73,16 @@ export async function scoutSearch(input) {
         const below = ordered.indexOf(candidate) < pickAt;
         if (below && !lookBelow)
             continue;
-        if (skipped.has(candidate.model))
-            continue;
-        let accepted = screened.get(candidate.model);
+        let accepted = screened.get(candidate.key);
         if (accepted === undefined) {
             try {
-                const trial = await run("neighbour", candidate, samples);
-                trials.push(trial);
-                accepted = trial.accepted;
+                accepted = await screen(candidate, "neighbour");
+                trials.push({ stage: "neighbour", candidate, accepted });
             }
             catch (error) {
-                if (!unavailableError(error))
-                    return failure("screen", candidate, error);
-                skip("neighbour", candidate, error);
-                skipped.add(candidate.model);
+                if (!unavailable(error))
+                    throw error;
+                skipped.push({ candidate, stage: "neighbour", reason: error.message });
                 continue;
             }
         }
@@ -134,72 +91,15 @@ export async function scoutSearch(input) {
         else if (below)
             lookBelow = false;
     }
-    for (const candidate of finalists) {
-        try {
-            finals.push(await run("final", candidate, input.dataset));
-        }
-        catch (error) {
-            if (!unavailableError(error))
-                return failure("final", candidate, error);
-            skip("final", candidate, error);
-        }
-    }
-    finals.sort(compareFinals);
-    // Challengers: a stronger model at the same or a lower list price than the leader could be the
-    // better buy, but the neighbour rule never reaches it. Screen the cheapest few on the same samples.
-    const leader = finals[0]?.candidate;
-    const tried = new Set([...trials.map(trial => trial.candidate.model), ...unavailable.map(item => item.model)]);
-    const challengers = leader?.price == null ? [] : ordered
-        .filter(item => item.eci > leader.eci && item.price !== null && item.price <= leader.price && !tried.has(item.model))
-        .sort((a, b) => a.price - b.price || b.eci - a.eci)
-        .slice(0, input.challengers ?? 3);
-    for (const candidate of challengers) {
-        try {
-            const trial = await run("challenger", candidate, samples);
-            trials.push(trial);
-            if (trial.accepted)
-                finals.push(await run("final", candidate, input.dataset));
-        }
-        catch (error) {
-            if (!unavailableError(error))
-                return failure("challenger", candidate, error);
-            skip("challenger", candidate, error);
-        }
-    }
-    finals.sort(compareFinals);
-    return result("selected");
-}
-/** Models other than the pick, nearest by rank first; at equal distance the weaker one comes first. */
-export function outward(ordered, pick) {
-    const index = ordered.findIndex(item => item.model === pick.model);
-    if (index < 0)
-        throw new Error("The pick is not in the ranking");
-    const out = [];
-    for (let distance = 1; out.length < ordered.length - 1; distance++) {
-        if (ordered[index - distance])
-            out.push(ordered[index - distance]);
-        if (ordered[index + distance])
-            out.push(ordered[index + distance]);
-    }
-    return out;
-}
-/** Mean measured cost per item, or null when any item's cost is unknown. */
-export function meanCost(outcomes) {
-    if (!outcomes.length || outcomes.some(item => typeof item.costUsd !== "number"))
-        return null;
-    return outcomes.reduce((sum, item) => sum + item.costUsd, 0) / outcomes.length;
+    return {
+        selected, finalists, trials, unavailable: skipped,
+        ...(finalists.length < finalistCount ? { shortfall: `Only ${finalists.length} finalists: no further model passed the screen within the neighbour rule (once a weaker neighbour fails, only stronger models are tried).` } : {}),
+    };
 }
-/** Most passes; then lower measured cost per item (list price only when a cost is unknown); then lower ECI. */
-export function compareFinals(a, b) {
-    if (b.passes !== a.passes)
-        return b.passes - a.passes;
-    const ca = meanCost(a.outcomes), cb = meanCost(b.outcomes);
-    if (ca !== null && cb !== null && ca !== cb)
-        return ca - cb;
-    if (ca === null || cb === null) {
-        const pa = a.candidate.price ?? Infinity, pb = b.candidate.price ?? Infinity;
-        if (pa !== pb)
-            return pa - pb;
-    }
-    return a.candidate.eci - b.candidate.eci;
+/** Untested models stronger than the leader and no dearer by list price, cheapest first. */
+export function challengers(ordered, leader, tried, count = CHALLENGERS) {
+    if (leader.price === null)
+        return [];
+    return ordered.filter(item => item.eci > leader.eci && item.price !== null && item.price <= leader.price && !tried.has(item.key))
+        .sort((a, b) => a.price - b.price || b.eci - a.eci).slice(0, count);
 }