openmerit 0.4.0 → 0.4.1

package/README.md

@@ -38,7 +38,7 @@ openmerit scout "Extract chronic conditions as ICD-10-CM categories from health
   --dataset examples/health-declaration-codes/dataset.jsonl --providers openrouter --max-price 30 --execute
 
 # The search alone, without the agent: deterministic, for CI and other coding agents (JSON output with --json).
-openmerit run --langfuse-dataset sahra-enrollments --evaluator ./evaluator.ts --execute
+openmerit run --langfuse-dataset sahra-enrollments --evaluator ./evaluator.mts --execute
 openmerit run --dataset items.jsonl --system-prompt "$(cat prompt.txt)" --grader json --execute
 
 # Interactive: Pi with Scout. Ask "which model should I use for …" or type /scout.
@@ -72,7 +72,7 @@ Every run writes `.openmerit/runs/<run-id>/report.md` and `report.json` in the p
   - `json`: every expected field must match; arrays are compared as sets.
   - `exact` and `contains`: for short answers.
   - `judge`: a rubric checked by Scout's model.
-- **Your harness** (`--evaluator module.ts`): default-export an `EvaluatorPlugin` that runs one item through your real system and returns `{ pass, output, costUsd? }`. When a model cannot run, throw an error whose `name` is `"CandidateUnavailable"`; Scout then skips it as untested. Throw any other error to stop the run. [`examples/sahra-enrollment/evaluator.ts`](examples/sahra-enrollment/evaluator.ts), which you can copy into your project, drives a deployed multi-step tool-calling agent over HTTP. It reads cost and provider errors from that agent's own Langfuse traces.
+- **Your harness** (`--evaluator module.mts`, relative to the current directory): default-export an `EvaluatorPlugin` that runs one item through your real system and returns `{ pass, output, costUsd? }`. When a model cannot run, throw an error whose `name` is `"CandidateUnavailable"`; Scout then skips it as untested. Throw any other error to stop the run. [`examples/sahra-enrollment/evaluator.mts`](examples/sahra-enrollment/evaluator.mts), which you can copy into your project (change its type import to `"openmerit"`), drives a deployed multi-step tool-calling agent over HTTP. It reads cost and provider errors from that agent's own Langfuse traces.
 
 ## Limitations
 

package/dist/cli.js

@@ -17,7 +17,7 @@ Options:
   --codebase <dir>          Project Scout reads (default: current directory)
   --langfuse-dataset <name> Labelled Langfuse dataset
   --dataset <file.jsonl>    Local labelled items {id?, input, expectedOutput}; pushed to Langfuse
-  --evaluator <module.ts>   Evaluator module that runs your real harness
+  --evaluator <module.mts>  Evaluator module that runs your real harness (relative to the current directory)
   --system-prompt <text>    run: production system prompt for the built-in Pi evaluator
   --grader <exact|contains|json|judge>  run: how the Pi evaluator checks a reply (default json)
   --rubric <text>           run: rubric for the judge grader
@@ -107,7 +107,7 @@ async function main() {
     if (command === "run") {
         const { host } = await hostFor(values);
         const evaluator = values.evaluator
-            ? { type: "module", path: values.evaluator }
+            ? { type: "module", path: resolve(values.evaluator) }
             : (() => {
                 if (!values["system-prompt"])
                     throw new Error("run needs --evaluator <module> or --system-prompt for the built-in Pi evaluator");
@@ -118,7 +118,7 @@ async function main() {
             })();
         const params = {
             task: values.task ?? positionals.join(" ") ?? "", evaluator, filter: filterFrom(values),
-            ...(values["langfuse-dataset"] ? { langfuseDataset: values["langfuse-dataset"] } : {}), ...(values.dataset ? { datasetPath: values.dataset } : {}),
+            ...(values["langfuse-dataset"] ? { langfuseDataset: values["langfuse-dataset"] } : {}), ...(values.dataset ? { datasetPath: resolve(values.dataset) } : {}),
             ...(values["sample-size"] ? { sampleSize: Number(values["sample-size"]) } : {}), ...(values.finalists ? { finalists: Number(values.finalists) } : {}),
             ...(values.repeats ? { repeats: Number(values.repeats) } : {}),
         };
@@ -157,8 +157,8 @@ async function main() {
         });
         const hints = [
             values["langfuse-dataset"] && `Use the Langfuse dataset "${values["langfuse-dataset"]}".`,
-            values.dataset && `Use the local dataset ${values.dataset}.`,
-            values.evaluator && `Use the evaluator module ${values.evaluator}.`,
+            values.dataset && `Use the local dataset ${resolve(values.dataset)}.`,
+            values.evaluator && `Use the evaluator module ${resolve(values.evaluator)}.`,
             (values.providers || values.tools || values["max-price"] || values.exclude) && `Candidate filter: ${JSON.stringify(filterFrom(values))}.`,
             values["sample-size"] && `sampleSize ${values["sample-size"]}.`, values.finalists && `finalists ${values.finalists}.`, values.repeats && `repeats ${values.repeats}.`,
         ].filter(Boolean).join(" ");

package/dist/connectors/langfuse/index.d.ts

@@ -1,5 +1,5 @@
-import { type LangfuseConnection, type TransportOptions } from "./client.ts";
-export { connectionFromEnv, type LangfuseConnection, LangfuseError, normalizeBaseUrl, type TransportOptions } from "./client.ts";
+import { type LangfuseConnection, type TransportOptions } from "./client.js";
+export { connectionFromEnv, type LangfuseConnection, LangfuseError, normalizeBaseUrl, type TransportOptions } from "./client.js";
 export type ScoreValue = number | boolean | string;
 export type Metadata = Record<string, string | number | boolean>;
 export interface EvalItem {

package/dist/index.d.ts

@@ -1,11 +1,11 @@
-export { rankCandidates, rank, openRouterToolSupport, type Candidate, type CandidateFilter, type RunnableModel } from "./scout/candidates.ts";
-export { loadEci, parseEciScores, ECI_ATTRIBUTION, type EciRow, type EciSource } from "./scout/eci.ts";
-export { scoutSearch, CandidateUnavailable, outward, compareFinals, meanCost, SAMPLE_SIZE, FINALISTS, type EvalItem, type Evaluate, type Outcome, type SearchOptions, type SearchResult, type Stage, type Trial } from "./scout/search.ts";
-export { runScout, type ScoutReport, type ScoutRunOptions, type FinalVerdict, type DatasetSource } from "./scout/pipeline.ts";
-export { reportMarkdown } from "./scout/report.ts";
-export { piEvaluator, jsonMatches, extractJson, type Grader, type ModelCaller, type PiTaskSpec } from "./scout/evaluators/pi.ts";
-export { loadEvaluatorPlugin, type EvaluatorPlugin } from "./scout/evaluators/module.ts";
-export * as scout from "./scout/service.ts";
-export { createScoutExtension, type ScoutExtensionOptions } from "./pi/extension.ts";
-export { configureHarness } from "./pi/harness.ts";
-export * as langfuse from "./connectors/langfuse/index.ts";
+export { rankCandidates, rank, openRouterToolSupport, type Candidate, type CandidateFilter, type RunnableModel } from "./scout/candidates.js";
+export { loadEci, parseEciScores, ECI_ATTRIBUTION, type EciRow, type EciSource } from "./scout/eci.js";
+export { scoutSearch, CandidateUnavailable, outward, compareFinals, meanCost, SAMPLE_SIZE, FINALISTS, type EvalItem, type Evaluate, type Outcome, type SearchOptions, type SearchResult, type Stage, type Trial } from "./scout/search.js";
+export { runScout, type ScoutReport, type ScoutRunOptions, type FinalVerdict, type DatasetSource } from "./scout/pipeline.js";
+export { reportMarkdown } from "./scout/report.js";
+export { piEvaluator, jsonMatches, extractJson, type Grader, type ModelCaller, type PiTaskSpec } from "./scout/evaluators/pi.js";
+export { loadEvaluatorPlugin, type EvaluatorPlugin } from "./scout/evaluators/module.js";
+export * as scout from "./scout/service.js";
+export { createScoutExtension, type ScoutExtensionOptions } from "./pi/extension.js";
+export { configureHarness } from "./pi/harness.js";
+export * as langfuse from "./connectors/langfuse/index.js";

package/dist/pi/harness.d.ts

@@ -1,6 +1,6 @@
 import { ModelRuntime } from "@earendil-works/pi-coding-agent";
-import type { RunnableModel } from "../scout/candidates.ts";
-import type { ModelCaller } from "../scout/evaluators/pi.ts";
+import type { RunnableModel } from "../scout/candidates.js";
+import type { ModelCaller } from "../scout/evaluators/pi.js";
 /**
  * OpenMerit's changes to a stock Pi process, applied before Pi reads its environment:
  * - its own agent directory (~/.openmerit/agent, or $OPENMERIT_HOME/agent), so OpenMerit's

package/dist/pi/instructions.d.ts

@@ -1,2 +1,2 @@
 /** Scout's workflow, added to Pi's system prompt as the `openmerit_scout` section. */
-export declare const SCOUT_INSTRUCTIONS = "You are OpenMerit Scout. You recommend the model a user should run for a task, from measured results on the user's own labelled examples, not from memory.\n\nWorkflow when asked which model to use:\n1. Understand the task. If a codebase is present, find where the model is called (grep for model IDs, SDK calls, \"systemPrompt\", \"model:\"), the production prompt, whether it calls tools, its output format, and existing evals: fixtures, tests with expected outputs, Langfuse datasets (scout_connections lists them), or .openmerit/datasets/*.jsonl.\n2. Find labelled data. Prefer an existing Langfuse dataset or fixture set with expected outputs. Preview it with scout_dataset; Scout needs at least 5 items. If none exists, draft at least 6 items only from what the code, fixtures or the user state, save them with scout_save_dataset, and say they are drafts the user should check. Never invent labels you cannot justify.\n3. Choose the evaluator. If the project ships an evaluator module that runs its real harness (for example .openmerit/evaluator.ts or examples/*/evaluator.ts), use evaluator type \"module\". Otherwise use type \"pi\" with the production system prompt and the strictest grader that fits: \"json\" for structured outputs, \"exact\"/\"contains\" for short answers, \"judge\" with a precise rubric only for free text.\n4. Check candidates with scout_candidates. Filter only for real constraints: providers the user can run, tools when the call uses tools, a price ceiling the user gave, data-residency or vendor exclusions the user named. Do not narrow the pool from your own preferences.\n5. Run scout_run with the task, dataset and evaluator. It binary-searches the ECI-ranked candidates (pass = every sampled item), screens neighbours, runs 5 finalists (plus cheaper, stronger challengers) on every item 3 times, records each as a Langfuse experiment, and picks the winner from Langfuse: most passes, then lower measured cost per item, then lower ECI.\n6. Answer with the recommended model ID, its passes and cost per item, the runner-up, the Langfuse experiment links, the report path, and the limitations the report lists (sample size, untested skipped models, drafted labels, residency or policy gaps). If scout_run returns plan_only, report the plan and the command to run it.\n\nRules: measured results on the user's data outrank public benchmarks. Provider, auth or billing errors are not quality failures; report them. Never print API keys or tokens.";
+export declare const SCOUT_INSTRUCTIONS = "You are OpenMerit Scout. You recommend the model a user should run for a task, from measured results on the user's own labelled examples, not from memory.\n\nWorkflow when asked which model to use:\n1. Understand the task. If a codebase is present, find where the model is called (grep for model IDs, SDK calls, \"systemPrompt\", \"model:\"), the production prompt, whether it calls tools, its output format, and existing evals: fixtures, tests with expected outputs, Langfuse datasets (scout_connections lists them), or .openmerit/datasets/*.jsonl.\n2. Find labelled data. Prefer an existing Langfuse dataset or fixture set with expected outputs. Preview it with scout_dataset; Scout needs at least 5 items. If none exists, draft at least 6 items only from what the code, fixtures or the user state, save them with scout_save_dataset, and say they are drafts the user should check. Never invent labels you cannot justify.\n3. Choose the evaluator. If the project ships an evaluator module that runs its real harness (for example .openmerit/evaluator.mts, .openmerit/evaluator.ts or examples/*/evaluator.mts), use evaluator type \"module\". Otherwise use type \"pi\" with the production system prompt and the strictest grader that fits: \"json\" for structured outputs, \"exact\"/\"contains\" for short answers, \"judge\" with a precise rubric only for free text.\n4. Check candidates with scout_candidates. Filter only for real constraints: providers the user can run, tools when the call uses tools, a price ceiling the user gave, data-residency or vendor exclusions the user named. Do not narrow the pool from your own preferences.\n5. Run scout_run with the task, dataset and evaluator. It binary-searches the ECI-ranked candidates (pass = every sampled item), screens neighbours, runs 5 finalists (plus cheaper, stronger challengers) on every item 3 times, records each as a Langfuse experiment, and picks the winner from Langfuse: most passes, then lower measured cost per item, then lower ECI.\n6. Answer with the recommended model ID, its passes and cost per item, the runner-up, the Langfuse experiment links, the report path, and the limitations the report lists (sample size, untested skipped models, drafted labels, residency or policy gaps). If scout_run returns plan_only, report the plan and the command to run it.\n\nRules: measured results on the user's data outrank public benchmarks. Provider, auth or billing errors are not quality failures; report them. Never print API keys or tokens.";

package/dist/pi/instructions.js

@@ -4,7 +4,7 @@ export const SCOUT_INSTRUCTIONS = `You are OpenMerit Scout. You recommend the mo
 Workflow when asked which model to use:
 1. Understand the task. If a codebase is present, find where the model is called (grep for model IDs, SDK calls, "systemPrompt", "model:"), the production prompt, whether it calls tools, its output format, and existing evals: fixtures, tests with expected outputs, Langfuse datasets (scout_connections lists them), or .openmerit/datasets/*.jsonl.
 2. Find labelled data. Prefer an existing Langfuse dataset or fixture set with expected outputs. Preview it with scout_dataset; Scout needs at least 5 items. If none exists, draft at least 6 items only from what the code, fixtures or the user state, save them with scout_save_dataset, and say they are drafts the user should check. Never invent labels you cannot justify.
-3. Choose the evaluator. If the project ships an evaluator module that runs its real harness (for example .openmerit/evaluator.ts or examples/*/evaluator.ts), use evaluator type "module". Otherwise use type "pi" with the production system prompt and the strictest grader that fits: "json" for structured outputs, "exact"/"contains" for short answers, "judge" with a precise rubric only for free text.
+3. Choose the evaluator. If the project ships an evaluator module that runs its real harness (for example .openmerit/evaluator.mts, .openmerit/evaluator.ts or examples/*/evaluator.mts), use evaluator type "module". Otherwise use type "pi" with the production system prompt and the strictest grader that fits: "json" for structured outputs, "exact"/"contains" for short answers, "judge" with a precise rubric only for free text.
 4. Check candidates with scout_candidates. Filter only for real constraints: providers the user can run, tools when the call uses tools, a price ceiling the user gave, data-residency or vendor exclusions the user named. Do not narrow the pool from your own preferences.
 5. Run scout_run with the task, dataset and evaluator. It binary-searches the ECI-ranked candidates (pass = every sampled item), screens neighbours, runs 5 finalists (plus cheaper, stronger challengers) on every item 3 times, records each as a Langfuse experiment, and picks the winner from Langfuse: most passes, then lower measured cost per item, then lower ECI.
 6. Answer with the recommended model ID, its passes and cost per item, the runner-up, the Langfuse experiment links, the report path, and the limitations the report lists (sample size, untested skipped models, drafted labels, residency or policy gaps). If scout_run returns plan_only, report the plan and the command to run it.

package/dist/scout/candidates.d.ts

@@ -1,4 +1,4 @@
-import type { EciEstimate, EciRow } from "./eci.ts";
+import type { EciEstimate, EciRow } from "./eci.js";
 /** A model the user's connections can run, placed on the ECI scale. */
 export interface Candidate {
     /** `provider/id` as Pi names it, such as `openrouter/deepseek/deepseek-v4-pro` or `anthropic/claude-sonnet-5-5`. */

package/dist/scout/evaluators/module.d.ts

@@ -1,4 +1,4 @@
-import type { Evaluate } from "../search.ts";
+import type { Evaluate } from "../search.js";
 /**
  * An evaluator that runs a candidate through the user's own harness, such as a deployed agent's
  * HTTP API, and checks the result against the labelled item. A module's default export is one of these.
@@ -20,5 +20,5 @@ export interface EvaluatorPlugin {
         log: (line: string) => void;
     }): Evaluate | Promise<Evaluate>;
 }
-/** Imports a `.ts` or `.js` evaluator module from the user's project. Node.js 22.19 strips its types. */
+/** Imports an evaluator module (`.mts`, `.ts`, `.mjs` or `.js`) from the user's project. Node.js 22.19 strips its types. */
 export declare function loadEvaluatorPlugin(path: string, cwd?: string): Promise<EvaluatorPlugin>;

package/dist/scout/evaluators/module.js

@@ -8,7 +8,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 };
 import { resolve } from "node:path";
 import { pathToFileURL } from "node:url";
-/** Imports a `.ts` or `.js` evaluator module from the user's project. Node.js 22.19 strips its types. */
+/** Imports an evaluator module (`.mts`, `.ts`, `.mjs` or `.js`) from the user's project. Node.js 22.19 strips its types. */
 export async function loadEvaluatorPlugin(path, cwd = process.cwd()) {
     const loaded = await import(__rewriteRelativeImportExtension(pathToFileURL(resolve(cwd, path)).href));
     const plugin = loaded.default;

package/dist/scout/evaluators/pi.d.ts

@@ -1,5 +1,5 @@
 import type { Api, AssistantMessage, Context, Model } from "@earendil-works/pi-ai";
-import { type Evaluate } from "../search.ts";
+import { type Evaluate } from "../search.js";
 /** What the Pi evaluator needs from Pi: a `ModelRuntime`, or an extension's `ctx.modelRegistry` through `fromRegistry`. */
 export interface ModelCaller {
     getModel(provider: string, id: string): Model<Api> | undefined;

package/dist/scout/pipeline.d.ts

@@ -1,6 +1,6 @@
-import type { LangfuseConnector } from "../connectors/langfuse/index.ts";
-import type { Candidate } from "./candidates.ts";
-import { type EvalItem, type Evaluate, type SearchResult } from "./search.ts";
+import type { LangfuseConnector } from "../connectors/langfuse/index.js";
+import type { Candidate } from "./candidates.js";
+import { type EvalItem, type Evaluate, type SearchResult } from "./search.js";
 export interface DatasetSource {
     /** An existing Langfuse dataset, such as the customer's `sahra-enrollments`. */
     langfuseDataset?: string;

package/dist/scout/report.d.ts

@@ -1,3 +1,3 @@
-import type { ScoutReport } from "./pipeline.ts";
+import type { ScoutReport } from "./pipeline.js";
 /** The decision record as Markdown: no keys, no raw item contents. */
 export declare function reportMarkdown(report: ScoutReport): string;

package/dist/scout/search.d.ts

@@ -1,4 +1,4 @@
-import type { Candidate } from "./candidates.ts";
+import type { Candidate } from "./candidates.js";
 /**
  * Scout Model Binary Search, from the ScoutBM prototype:
  * 1. Rank candidates by ECI, ascending, and draw `sampleSize` labelled items once.

package/dist/scout/service.d.ts

@@ -1,8 +1,8 @@
-import { type LangfuseConnector } from "../connectors/langfuse/index.ts";
-import { type CandidateFilter, type RunnableModel } from "./candidates.ts";
-import { type ModelCaller, type PiTaskSpec } from "./evaluators/pi.ts";
-import { type ScoutReport } from "./pipeline.ts";
-import { type EvalItem } from "./search.ts";
+import { type LangfuseConnector } from "../connectors/langfuse/index.js";
+import { type CandidateFilter, type RunnableModel } from "./candidates.js";
+import { type ModelCaller, type PiTaskSpec } from "./evaluators/pi.js";
+import { type ScoutReport } from "./pipeline.js";
+import { type EvalItem } from "./search.js";
 /** Everything Scout's tools need from the host: Pi's models and credentials, and where to write. */
 export interface ScoutHost {
     caller: ModelCaller;
@@ -44,7 +44,7 @@ export declare function candidates(host: ScoutHost, filter: CandidateFilter): Pr
         calibrated: number;
         unranked: number;
     };
-    candidates: import("./candidates.ts").Candidate[];
+    candidates: import("./candidates.js").Candidate[];
     unranked: string[];
     dropped: {
         model: string;
@@ -102,7 +102,7 @@ export declare function estimate(candidateCount: number, datasetSize: number, sa
     note: string;
 };
 export declare function plan(host: ScoutHost, params: RunParams): Promise<{
-    plugin: import("./evaluators/module.ts").EvaluatorPlugin | null;
+    plugin: import("./evaluators/module.js").EvaluatorPlugin | null;
     filter: CandidateFilter;
     ranked: {
         eci: {
@@ -123,7 +123,7 @@ export declare function plan(host: ScoutHost, params: RunParams): Promise<{
             calibrated: number;
             unranked: number;
         };
-        candidates: import("./candidates.ts").Candidate[];
+        candidates: import("./candidates.js").Candidate[];
         unranked: string[];
         dropped: {
             model: string;

package/examples/sahra-enrollment/{evaluator.ts → evaluator.mts}

@@ -6,7 +6,8 @@
 //   PLAYGROUND_URL (default http://localhost:8787), PLAYGROUND_TOKEN, LANGFUSE_* for measured cost.
 import { createHash, randomUUID } from "node:crypto";
 import { setTimeout as delay } from "node:timers/promises";
-// Copy this file into your project to adapt it; it has no runtime import from OpenMerit.
+// Copy this file into your project to adapt it; it has no runtime import from OpenMerit. The .mts
+// extension makes Node.js load it as an ES module whatever your package.json says.
 // In your project, import the type from the package instead: `import type { EvaluatorPlugin } from "openmerit"`.
 import type { EvaluatorPlugin } from "../../src/scout/evaluators/module.ts";
 
@@ -103,7 +104,7 @@ const plugin: EvaluatorPlugin = {
       const expected = item.expectedOutput as Expected;
       // Outlast EnrollmentFlow's own 10-minute limit and one retry, so the workflow decides the outcome.
       const signal = AbortSignal.any([AbortSignal.timeout(25 * 60_000), ...(context.signal ? [context.signal] : [])]);
-      const call = async <T>(method: string, path: string, body?: unknown): Promise<T> => {
+      const call = async <T,>(method: string, path: string, body?: unknown): Promise<T> => {
         const response = await fetch(`${url}${path}`, { method, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: body === undefined ? null : JSON.stringify(body), signal });
         if (!response.ok) throw new Error(`Playground ${method} ${path.split("?")[0]} HTTP ${response.status}`);
         return await response.json() as T;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmerit",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Find the right model for your task: Scout reads your task or codebase, binary-searches benchmark-ranked models on your own labelled examples, and picks the winner from Langfuse evaluations.",
   "type": "module",
   "license": "MIT",
@@ -40,7 +40,7 @@
     "node": ">=22.19.0"
   },
   "scripts": {
-    "build": "rm -rf dist && tsc -p tsconfig.build.json && chmod +x dist/cli.js",
+    "build": "rm -rf dist && tsc -p tsconfig.build.json && node scripts/rewrite-declaration-imports.ts && chmod +x dist/cli.js",
     "typecheck": "tsc -p tsconfig.json",
     "test": "node --test \"src/**/*.test.ts\" \"test/**/*.test.ts\"",
     "check": "npm run typecheck && npm test",