openmerit 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +239 -58
  2. package/data/catalogue.json +6778 -0
  3. package/dist/catalogue/catalogue.d.ts +41 -0
  4. package/dist/catalogue/catalogue.js +33 -0
  5. package/dist/cli/prompt.d.ts +26 -0
  6. package/dist/cli/prompt.js +114 -0
  7. package/dist/cli/render.d.ts +12 -0
  8. package/dist/cli/render.js +104 -0
  9. package/dist/cli.js +245 -158
  10. package/dist/connect.d.ts +34 -0
  11. package/dist/connect.js +158 -0
  12. package/dist/connectors/langfuse/index.d.ts +39 -112
  13. package/dist/connectors/langfuse/index.js +95 -246
  14. package/dist/daemon/daemon.d.ts +81 -0
  15. package/dist/daemon/daemon.js +408 -0
  16. package/dist/daemon/registry.d.ts +111 -0
  17. package/dist/daemon/registry.js +109 -0
  18. package/dist/eval/proxy.d.ts +54 -0
  19. package/dist/eval/proxy.js +174 -0
  20. package/dist/eval/runner.d.ts +29 -0
  21. package/dist/eval/runner.js +80 -0
  22. package/dist/harness/claude-bridge.d.ts +21 -0
  23. package/dist/harness/claude-bridge.js +65 -0
  24. package/dist/harness/harness.d.ts +60 -0
  25. package/dist/harness/harness.js +221 -0
  26. package/dist/index.d.ts +10 -11
  27. package/dist/index.js +9 -10
  28. package/dist/local.d.ts +12 -0
  29. package/dist/local.js +40 -0
  30. package/dist/mcp.d.ts +1 -0
  31. package/dist/mcp.js +155 -0
  32. package/dist/scout/candidates.d.ts +1 -1
  33. package/dist/scout/pool.d.ts +17 -0
  34. package/dist/scout/pool.js +56 -0
  35. package/dist/scout/scan.d.ts +42 -0
  36. package/dist/scout/scan.js +169 -0
  37. package/dist/scout/search.d.ts +25 -98
  38. package/dist/scout/search.js +56 -156
  39. package/dist/scout/workflow.d.ts +221 -0
  40. package/dist/scout/workflow.js +941 -0
  41. package/dist/setup.d.ts +44 -0
  42. package/dist/setup.js +211 -0
  43. package/dist/state/config.d.ts +116 -0
  44. package/dist/state/config.js +60 -0
  45. package/dist/state/db.d.ts +159 -0
  46. package/dist/state/db.js +260 -0
  47. package/dist/state/paths.d.ts +12 -0
  48. package/dist/state/paths.js +23 -0
  49. package/dist/update.d.ts +22 -0
  50. package/dist/update.js +58 -0
  51. package/package.json +13 -18
  52. package/dist/connectors/langfuse/otlp.d.ts +0 -60
  53. package/dist/connectors/langfuse/otlp.js +0 -40
  54. package/dist/pi/extension.d.ts +0 -13
  55. package/dist/pi/extension.js +0 -148
  56. package/dist/pi/harness.d.ts +0 -20
  57. package/dist/pi/harness.js +0 -49
  58. package/dist/pi/instructions.d.ts +0 -2
  59. package/dist/pi/instructions.js +0 -12
  60. package/dist/scout/evaluators/module.d.ts +0 -24
  61. package/dist/scout/evaluators/module.js +0 -19
  62. package/dist/scout/evaluators/pi.d.ts +0 -53
  63. package/dist/scout/evaluators/pi.js +0 -117
  64. package/dist/scout/pipeline.d.ts +0 -111
  65. package/dist/scout/pipeline.js +0 -157
  66. package/dist/scout/report.d.ts +0 -3
  67. package/dist/scout/report.js +0 -42
  68. package/dist/scout/service.d.ts +0 -148
  69. package/dist/scout/service.js +0 -143
  70. package/examples/health-declaration-codes/dataset.jsonl +0 -14
  71. package/examples/sahra-enrollment/evaluator.ts +0 -151
package/dist/cli.js CHANGED
@@ -1,178 +1,265 @@
1
1
  #!/usr/bin/env node
2
+ import { spawnSync } from "node:child_process";
3
+ import { realpathSync } from "node:fs";
4
+ import { fileURLToPath } from "node:url";
2
5
  import { parseArgs } from "node:util";
3
- import { resolve } from "node:path";
4
- import { configureHarness, openRuntime, pickScoutModel } from "./pi/harness.js";
5
- const HELP = `OpenMerit: find the right model for your task.
6
+ const HELP = `OpenMerit: find cheaper models that perform at least as well as the ones your code uses today,
7
+ measured with your own evals. Runs locally; no OpenMerit account.
6
8
 
7
9
  Usage:
8
- openmerit Pi coding agent with OpenMerit Scout (/scout, scout_* tools)
9
- openmerit login Connect a subscription or API key (Pi's /login)
10
- openmerit status Connected providers, Scout's model and Langfuse
11
- openmerit scout "<task>" [options] Scout agent: reads the task and codebase, runs the search, recommends a model
12
- openmerit run [options] The search alone, without the agent (for CI and coding agents)
13
- openmerit candidates [options] ECI-ranked candidates the connected providers can run
14
- openmerit pi [pi args] Pi itself, with the Scout extension
10
+ openmerit setup Connect OpenMerit Harness (as openmerit connect does), register the MCP
11
+ server, and configure this repository: read-only Langfuse, candidate
12
+ endpoints and allowed models, default eval budget. Rerunnable.
13
+ openmerit connect [claude-code|codex|api]
14
+ Power OpenMerit Harness with Claude Code (its own login), Codex models
15
+ (a ChatGPT Plus/Pro sign-in) or an API key. Signs in when needed.
16
+ openmerit scout Scan model usages, pick a model and tasks, binary-search cheaper models on your eval
17
+ candidates, compare with the current model, optionally configure.
18
+ openmerit scout --resume <run> Continue a paused, interrupted or failed run.
19
+ openmerit history [<run>] This repository's runs, or one run in detail.
20
+ openmerit mcp The local MCP server (stdio), for Claude Code, Codex and other clients.
21
+ openmerit daemon [status|stop] The background process that runs Scout (started by scout and mcp when needed).
22
+ Without an argument it runs in the foreground.
15
23
 
16
- Options:
17
- --codebase <dir> Project Scout reads (default: current directory)
18
- --langfuse-dataset <name> Labelled Langfuse dataset
19
- --dataset <file.jsonl> Local labelled items {id?, input, expectedOutput}; pushed to Langfuse
20
- --evaluator <module.ts> Evaluator module that runs your real harness
21
- --system-prompt <text> run: production system prompt for the built-in Pi evaluator
22
- --grader <exact|contains|json|judge> run: how the Pi evaluator checks a reply (default json)
23
- --rubric <text> run: rubric for the judge grader
24
- --task <text> run: task description recorded with the experiments
25
- --providers <a,b> Candidate providers (default: every connected provider)
26
- --tools Candidates must support tool calls
27
- --max-price <usd> Max list price per million input+output tokens
28
- --exclude <a,b> Drop candidates whose provider/id contains any of these
29
- --model <provider/id> Model Scout itself runs on
30
- --sample-size <n> Items per binary-search probe (default 5)
31
- --finalists <n> Finalists run on every item (default 5)
32
- --repeats <n> Runs of every item per finalist (default 3)
33
- --execute Allow spending provider credit (otherwise plan only)
34
- --json Machine-readable output
24
+ Scout options:
25
+ --answer <id=value> Answer a question without a prompt (repeatable), as --json returns them
26
+ --json Print the run state as JSON and stop at the first question
35
27
 
36
- Credentials come from Pi: API keys in the environment (OPENROUTER_API_KEY, ANTHROPIC_API_KEY,
37
- OPENAI_API_KEY, ...), subscriptions via \`openmerit login\`, or CLAUDE_CODE_OAUTH_TOKEN. Langfuse:
38
- LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, LANGFUSE_BASE_URL (Cloud or self-hosted). No OpenMerit account.`;
39
- const [command = "", ...rest] = process.argv.slice(2);
40
- configureHarness();
41
- async function launchPi(args) {
42
- const { main } = await import("@earendil-works/pi-coding-agent");
43
- const { createScoutExtension } = await import("./pi/extension.js");
44
- await main(args, { extensionFactories: [{ name: "openmerit-scout", factory: createScoutExtension({ allowSpend: "ask" }) }] });
28
+ Connect options (setup takes them too, with --harness <claude-code|codex|api> and --harness-model <id>):
29
+ --model <id> Model for claude-code or codex
30
+ --device-code Sign in to ChatGPT with a code entered on any device (for servers)
31
+ --api-provider <pi provider> --api-model <id> --api-key-env <VAR>
32
+ --yes Accept defaults; never prompt
33
+
34
+ Setup options (for scripts; interactive setup asks instead):
35
+ --harness <claude-code|codex|api> --harness-model <id> --device-code
36
+ --api-provider <pi provider> --api-model <id> --api-key-env <VAR>
37
+ --endpoint <openrouter:VAR | openai:VAR | id=https://host/v1:VAR> (repeatable)
38
+ --allowed-models <a,b> --budget <usd> --register-mcp | --no-register-mcp
39
+ --langfuse-url <url> --langfuse-public-key-env <VAR> --langfuse-secret-key-env <VAR>
40
+ --yes Accept defaults; never prompt`;
41
+ const [command = "help", ...rest] = process.argv.slice(2);
42
+ const log = (line) => process.stderr.write(`${line}\n`);
43
+ async function update(json) {
44
+ const { checkForUpdate, offerUpdate } = await import("./update.js");
45
+ const info = await checkForUpdate();
46
+ if (!info.available)
47
+ return null;
48
+ const interactive = process.stdin.isTTY === true && !json;
49
+ if (!interactive) {
50
+ log(`openmerit ${info.latest} is available (installed ${info.current}). Update with: npm install -g openmerit@${info.latest}`);
51
+ return null;
52
+ }
53
+ const { terminalPrompter } = await import("./cli/prompt.js");
54
+ const prompter = terminalPrompter();
55
+ try {
56
+ return await offerUpdate(info, { interactive, confirm: question => prompter.confirm(question, true), log, argv: process.argv.slice(2) });
57
+ }
58
+ finally {
59
+ prompter.close();
60
+ }
45
61
  }
46
- function options(args) {
47
- return parseArgs({
48
- args, allowPositionals: true,
49
- options: {
50
- codebase: { type: "string" }, "langfuse-dataset": { type: "string" }, dataset: { type: "string" }, evaluator: { type: "string" },
51
- "system-prompt": { type: "string" }, grader: { type: "string" }, rubric: { type: "string" }, task: { type: "string" },
52
- providers: { type: "string" }, tools: { type: "boolean" }, "max-price": { type: "string" }, exclude: { type: "string" },
53
- model: { type: "string" }, "sample-size": { type: "string" }, finalists: { type: "string" }, repeats: { type: "string" },
54
- execute: { type: "boolean", default: false }, json: { type: "boolean", default: false }, help: { type: "boolean" },
55
- },
56
- });
62
+ function mcpCommand() {
63
+ const self = realpathSync(fileURLToPath(import.meta.url));
64
+ const found = spawnSync("sh", ["-c", "command -v openmerit"], { encoding: "utf8" }).stdout.trim();
65
+ try {
66
+ if (found && realpathSync(found) === self)
67
+ return ["openmerit", "mcp"];
68
+ }
69
+ catch { /* fall through */ }
70
+ return [process.execPath, self, "mcp"];
57
71
  }
58
- const list = (value) => value?.split(",").map(item => item.trim()).filter(Boolean);
59
- const filterFrom = (values) => ({
60
- ...(values.providers ? { providers: list(values.providers) } : {}), ...(values.tools ? { tools: true } : {}),
61
- ...(values["max-price"] ? { maxPrice: Number(values["max-price"]) } : {}), ...(values.exclude ? { exclude: list(values.exclude) } : {}),
62
- });
63
- const log = (line) => process.stderr.write(`${line}\n`);
64
- async function hostFor(values) {
65
- const { runtime, caller, models } = await openRuntime();
66
- const scout = await pickScoutModel(runtime, values.model).catch(() => null);
67
- return { runtime, scout, host: { caller, models, cwd: resolve(values.codebase ?? "."), env: process.env, log, ...(scout ? { scoutModel: `${scout.provider}/${scout.id}` } : {}) } };
72
+ async function setup(args) {
73
+ const { values } = parseArgs({ args, options: {
74
+ harness: { type: "string" }, "harness-model": { type: "string" }, "api-provider": { type: "string" }, "api-model": { type: "string" }, "api-key-env": { type: "string" },
75
+ endpoint: { type: "string", multiple: true }, "allowed-models": { type: "string" }, budget: { type: "string" }, "register-mcp": { type: "boolean" }, "no-register-mcp": { type: "boolean" },
76
+ "langfuse-url": { type: "string" }, "langfuse-public-key-env": { type: "string" }, "langfuse-secret-key-env": { type: "string" }, yes: { type: "boolean" }, "device-code": { type: "boolean" },
77
+ } });
78
+ if (values.harness && !["claude-code", "codex", "api"].includes(values.harness))
79
+ throw new Error("--harness is claude-code, codex or api");
80
+ const { runSetup } = await import("./setup.js");
81
+ const { State } = await import("./state/db.js");
82
+ const { findRepo } = await import("./state/paths.js");
83
+ const { terminalPrompter } = await import("./cli/prompt.js");
84
+ const interactive = process.stdin.isTTY === true && !values.yes;
85
+ const prompter = interactive ? terminalPrompter() : null;
86
+ const state = new State();
87
+ try {
88
+ const result = await runSetup({
89
+ repo: findRepo(process.cwd()), state, prompter, log, env: process.env, mcpCommand: mcpCommand(),
90
+ flags: {
91
+ ...(values.harness ? { harness: values.harness } : {}), ...(values["harness-model"] ? { harnessModel: values["harness-model"] } : {}),
92
+ ...(values["device-code"] ? { deviceCode: true } : {}), ...(values["api-provider"] ? { apiProvider: values["api-provider"] } : {}), ...(values["api-model"] ? { apiModel: values["api-model"] } : {}), ...(values["api-key-env"] ? { apiKeyEnv: values["api-key-env"] } : {}),
93
+ ...(values.endpoint ? { endpoints: values.endpoint } : {}), ...(values["allowed-models"] ? { allowedModels: values["allowed-models"].split(",").map(item => item.trim()).filter(Boolean) } : {}),
94
+ ...(values.budget ? { budget: Number(values.budget) } : {}), ...(values["register-mcp"] ? { registerMcp: true } : values["no-register-mcp"] ? { registerMcp: false } : {}),
95
+ ...(values["langfuse-url"] ? { langfuseUrl: values["langfuse-url"] } : {}), ...(values["langfuse-public-key-env"] ? { langfusePublicKeyEnv: values["langfuse-public-key-env"] } : {}),
96
+ ...(values["langfuse-secret-key-env"] ? { langfuseSecretKeyEnv: values["langfuse-secret-key-env"] } : {}), ...(!interactive ? { nonInteractive: true } : {}),
97
+ },
98
+ });
99
+ if (!result.harness)
100
+ process.exitCode = 1;
101
+ }
102
+ finally {
103
+ prompter?.close();
104
+ state.close();
105
+ }
68
106
  }
69
- async function main() {
70
- if (command === "")
71
- return launchPi(rest);
72
- if (command === "pi")
73
- return launchPi(rest);
74
- if (command === "login")
75
- return launchPi(["/login", ...rest]);
76
- if (command === "help" || command === "--help" || command === "-h")
77
- return void process.stdout.write(`${HELP}\n`);
78
- if (command === "--version" || command === "-v") {
79
- const { readFile } = await import("node:fs/promises");
80
- const pkg = JSON.parse(await readFile(new URL("../package.json", import.meta.url), "utf8"));
81
- return void process.stdout.write(`${pkg.version}\n`);
107
+ async function connectCommand(args) {
108
+ const { values, positionals } = parseArgs({ args, allowPositionals: true, options: {
109
+ model: { type: "string" }, "device-code": { type: "boolean" }, "api-provider": { type: "string" }, "api-model": { type: "string" }, "api-key-env": { type: "string" }, yes: { type: "boolean" },
110
+ } });
111
+ const [kind] = positionals;
112
+ if (kind && !["claude-code", "codex", "api"].includes(kind))
113
+ throw new Error("Connect claude-code, codex or api.");
114
+ const { connect } = await import("./connect.js");
115
+ const { findRepo } = await import("./state/paths.js");
116
+ const { terminalPrompter } = await import("./cli/prompt.js");
117
+ const interactive = process.stdin.isTTY === true && !values.yes;
118
+ const prompter = interactive ? terminalPrompter() : null;
119
+ let root = process.cwd();
120
+ try {
121
+ root = findRepo(root).root;
82
122
  }
83
- const { values, positionals } = options(rest);
84
- if (values.help)
85
- return void process.stdout.write(`${HELP}\n`);
86
- const service = await import("./scout/service.js");
87
- if (command === "status") {
88
- const { host } = await hostFor(values);
89
- const status = await service.connections(host);
90
- if (values.json)
91
- return void process.stdout.write(`${JSON.stringify(status, null, 2)}\n`);
92
- process.stdout.write(`Providers: ${Object.entries(status.providers).map(([name, count]) => `${name} (${count} models)`).join(", ") || "none"}\n`);
93
- process.stdout.write(`Scout model: ${status.scoutModel ?? "none"}\n`);
94
- process.stdout.write(`Langfuse: ${status.langfuse.connected ? `${status.langfuse.project} at ${status.langfuse.baseUrl}; datasets ${status.langfuse.datasets?.join(", ") || "none"}` : `not connected${status.langfuse.error ? ` (${status.langfuse.error})` : ""}`}\n`);
95
- return;
123
+ catch { /* outside a repository: the check reads nothing */ }
124
+ try {
125
+ const saved = await connect({ root, prompter, log, env: process.env, flags: {
126
+ ...(values.model ? { model: values.model } : {}), ...(values["device-code"] ? { deviceCode: true } : {}), ...(values["api-provider"] ? { apiProvider: values["api-provider"] } : {}),
127
+ ...(values["api-model"] ? { apiModel: values["api-model"] } : {}), ...(values["api-key-env"] ? { apiKeyEnv: values["api-key-env"] } : {}), ...(!interactive ? { nonInteractive: true } : {}),
128
+ } }, kind);
129
+ if (saved)
130
+ log("Saved to ~/.openmerit/config.json. Next: openmerit setup in a repository, or openmerit scout.");
131
+ else
132
+ process.exitCode = 1;
96
133
  }
97
- if (command === "candidates") {
98
- const { host } = await hostFor(values);
99
- const ranked = await service.candidates(host, filterFrom(values));
100
- if (values.json)
101
- return void process.stdout.write(`${JSON.stringify(ranked, null, 2)}\n`);
102
- for (const item of ranked.candidates)
103
- process.stdout.write(`${item.eci.toFixed(1).padStart(6)} ${item.eciSource === "published" ? " " : `${item.eciSource.slice(0, 4)} ±${(item.eciError ?? 0).toFixed(1)}`.padEnd(9)} ${item.model} ${item.price === null ? "?" : `$${item.price.toFixed(2)}/M`}\n`);
104
- process.stdout.write(`${ranked.candidates.length} candidates (${ranked.counts.published} published ECI, ${ranked.counts.estimated} estimated, ${ranked.counts.calibrated} calibrated from data/eci-estimates.csv), ${ranked.unranked.length} unranked, ${ranked.dropped.length} filtered. ${ranked.eci.attribution}\n`);
105
- return;
134
+ finally {
135
+ prompter?.close();
106
136
  }
107
- if (command === "run") {
108
- const { host } = await hostFor(values);
109
- const evaluator = values.evaluator
110
- ? { type: "module", path: values.evaluator }
111
- : (() => {
112
- if (!values["system-prompt"])
113
- throw new Error("run needs --evaluator <module> or --system-prompt for the built-in Pi evaluator");
114
- const kind = values.grader ?? "json";
115
- const grader = kind === "judge" ? { type: "judge", rubric: values.rubric ?? "The output must agree with the expected output." }
116
- : kind === "exact" ? { type: "exact" } : kind === "contains" ? { type: "contains" } : { type: "json" };
117
- return { type: "pi", systemPrompt: values["system-prompt"], grader };
118
- })();
119
- const params = {
120
- task: values.task ?? positionals.join(" ") ?? "", evaluator, filter: filterFrom(values),
121
- ...(values["langfuse-dataset"] ? { langfuseDataset: values["langfuse-dataset"] } : {}), ...(values.dataset ? { datasetPath: values.dataset } : {}),
122
- ...(values["sample-size"] ? { sampleSize: Number(values["sample-size"]) } : {}), ...(values.finalists ? { finalists: Number(values.finalists) } : {}),
123
- ...(values.repeats ? { repeats: Number(values.repeats) } : {}),
124
- };
125
- if (!values.execute) {
126
- const planned = await service.plan(host, params);
127
- const out = { status: "plan_only", candidates: planned.ranked.candidates.length, datasetSize: planned.datasetSize, estimate: planned.estimate, evaluator: planned.plugin?.name ?? "pi", note: "Add --execute to run it." };
128
- return void process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
137
+ }
138
+ async function scout(args) {
139
+ const { values } = parseArgs({ args, options: { resume: { type: "string" }, answer: { type: "string", multiple: true }, json: { type: "boolean", default: false } } });
140
+ const { localContext, missingSetup, startRun } = await import("./local.js");
141
+ const { advanceInDaemon } = await import("./daemon/daemon.js");
142
+ const { renderComparison, renderStep } = await import("./cli/render.js");
143
+ const { askQuestion, terminalPrompter, workingTicker } = await import("./cli/prompt.js");
144
+ const controller = new AbortController();
145
+ let interrupts = 0;
146
+ process.on("SIGINT", () => { if (++interrupts > 1)
147
+ process.exit(130); log("\nStopping; the run is saved. Ctrl-C again to quit now."); controller.abort(); });
148
+ const ctx = localContext({ cwd: process.cwd(), log, signal: controller.signal });
149
+ const interactive = process.stdin.isTTY === true && !values.json;
150
+ const prompter = interactive ? terminalPrompter() : null;
151
+ try {
152
+ const missing = missingSetup(ctx);
153
+ if (missing.length) {
154
+ if (values.json)
155
+ return void process.stdout.write(`${JSON.stringify({ status: "needs_setup", missing }, null, 2)}\n`);
156
+ throw new Error(`Run \`openmerit setup\` first:\n - ${missing.join("\n - ")}`);
129
157
  }
130
- const result = await service.run(host, params);
131
- process.stdout.write(values.json ? `${JSON.stringify(result.report, null, 2)}\n` : `${result.markdown}\nReport: ${result.files.markdown}\n`);
132
- return;
133
- }
134
- if (command === "scout") {
135
- const task = positionals.join(" ").trim();
136
- if (!task)
137
- throw new Error("Describe the task: openmerit scout \"<what the model does>\"");
138
- const { createAgentSession, DefaultResourceLoader, SessionManager, getAgentDir } = await import("@earendil-works/pi-coding-agent");
139
- const { createScoutExtension } = await import("./pi/extension.js");
140
- const { runtime, scout, host } = await hostFor(values);
141
- if (!scout)
142
- throw new Error("No model provider is connected for Scout. Set an API key or run `openmerit login`.");
143
- const cwd = host.cwd;
144
- const resourceLoader = new DefaultResourceLoader({
145
- cwd, agentDir: getAgentDir(), noSkills: true, noPromptTemplates: true,
146
- extensionFactories: [{ name: "openmerit-scout", factory: createScoutExtension({ allowSpend: values.execute, log, scoutModel: host.scoutModel }) }],
147
- });
148
- await resourceLoader.reload();
149
- const { session } = await createAgentSession({
150
- cwd, model: scout, thinkingLevel: "medium", modelRuntime: runtime, resourceLoader, sessionManager: SessionManager.inMemory(cwd),
151
- tools: ["read", "grep", "find", "ls", "scout_connections", "scout_candidates", "scout_dataset", "scout_save_dataset", "scout_run"],
152
- });
153
- log(`Scout on ${host.scoutModel} in ${cwd}${values.execute ? "" : " (plan only: add --execute to spend)"}`);
154
- session.subscribe((event) => {
155
- if (event.type === "tool_execution_start")
156
- log(`→ ${event.toolName} ${JSON.stringify(event.args ?? {}).slice(0, 300)}`);
157
- });
158
- const hints = [
159
- values["langfuse-dataset"] && `Use the Langfuse dataset "${values["langfuse-dataset"]}".`,
160
- values.dataset && `Use the local dataset ${values.dataset}.`,
161
- values.evaluator && `Use the evaluator module ${values.evaluator}.`,
162
- (values.providers || values.tools || values["max-price"] || values.exclude) && `Candidate filter: ${JSON.stringify(filterFrom(values))}.`,
163
- values["sample-size"] && `sampleSize ${values["sample-size"]}.`, values.finalists && `finalists ${values.finalists}.`, values.repeats && `repeats ${values.repeats}.`,
164
- ].filter(Boolean).join(" ");
165
- try {
166
- await session.prompt(`Recommend the model for this task.\n\nTask: ${task}\n\n${hints}`.trim());
167
- const answer = session.getLastAssistantText() ?? "";
168
- process.stdout.write(values.json ? `${JSON.stringify({ task, scoutModel: host.scoutModel, answer }, null, 2)}\n` : `${answer}\n`);
158
+ const runId = values.resume ?? startRun(ctx);
159
+ let answers = Object.fromEntries((values.answer ?? []).map(item => {
160
+ const at = item.indexOf("=");
161
+ if (at < 1)
162
+ throw new Error(`--answer ${item}: write <question id>=<value>`);
163
+ return [item.slice(0, at), item.slice(at + 1)];
164
+ }));
165
+ let compared = false;
166
+ for (;;) {
167
+ const ticker = values.json ? null : workingTicker();
168
+ const result = await advanceInDaemon({ runId, cwd: process.cwd(), env: process.env, answers, log: ticker?.log ?? log, signal: controller.signal }).finally(() => ticker?.stop());
169
+ answers = {};
170
+ for (const notice of result.notices)
171
+ log(notice);
172
+ if (result.questions.length && prompter && !controller.signal.aborted) {
173
+ // The comparison comes before the question whether to configure its result.
174
+ if (result.result && !compared) {
175
+ process.stdout.write(`${result.result.tasks.map(task => renderComparison(task, runId)).join("\n\n")}\n`);
176
+ compared = true;
177
+ }
178
+ for (const question of result.questions) {
179
+ if (question.id === "budget_amount" && answers.budget_action === "decline")
180
+ continue;
181
+ answers[question.id] = await askQuestion(prompter, question, log);
182
+ }
183
+ continue;
184
+ }
185
+ if (values.json)
186
+ process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
187
+ else {
188
+ if (result.questions.length)
189
+ log(`Waiting for ${result.questions.map(question => question.id).join(", ")}. Answer: openmerit scout --resume ${runId} --answer <id>=<value>`);
190
+ process.stdout.write(`${renderStep(result, { comparisons: !compared })}\n`);
191
+ }
192
+ if (result.status === "failed" || result.status === "interrupted")
193
+ process.exitCode = 1;
194
+ return;
169
195
  }
170
- finally {
171
- session.dispose();
196
+ }
197
+ finally {
198
+ prompter?.close();
199
+ ctx.state.close();
200
+ }
201
+ }
202
+ async function history(args) {
203
+ const { values, positionals } = parseArgs({ args, allowPositionals: true, options: { json: { type: "boolean", default: false } } });
204
+ const { localContext } = await import("./local.js");
205
+ const { step } = await import("./scout/workflow.js");
206
+ const { renderHistory, renderRun } = await import("./cli/render.js");
207
+ const ctx = localContext({ cwd: process.cwd() });
208
+ try {
209
+ ctx.state.reapInterrupted(ctx.repo.id);
210
+ const id = positionals[0];
211
+ if (id) {
212
+ const run = ctx.state.getRun(id);
213
+ if (!run || run.repoId !== ctx.repo.id)
214
+ throw new Error(`No run ${id} in ${ctx.repo.root}.`);
215
+ const current = step(ctx, id, [], []);
216
+ process.stdout.write(values.json ? `${JSON.stringify({ run, step: current, units: ctx.state.units(id), budget: ctx.state.budgetEvents(id), decisions: ctx.state.decisions(id), events: ctx.state.events(id) }, null, 2)}\n` : `${renderRun(ctx.state, run, current)}\n`);
217
+ return;
172
218
  }
219
+ const runs = ctx.state.listRuns(ctx.repo.id);
220
+ process.stdout.write(values.json ? `${JSON.stringify(runs.map(run => ({ ...run, spend: ctx.state.spend(run.id) })), null, 2)}\n` : `${renderHistory(ctx.state, runs)}\n`);
221
+ }
222
+ finally {
223
+ ctx.state.close();
224
+ }
225
+ }
226
+ async function daemon(args) {
227
+ const { healthy, readDaemonInfo, serveDaemon, stopDaemon } = await import("./daemon/daemon.js");
228
+ const [action] = args;
229
+ if (!action)
230
+ return serveDaemon();
231
+ if (action === "stop")
232
+ return void log((await stopDaemon()) ? "Stopped the OpenMerit daemon. Runs it was working on continue with `openmerit scout --resume <run>`." : "The OpenMerit daemon is not running.");
233
+ if (action !== "status")
234
+ throw new Error(`Unknown daemon command "${action}". Use \`openmerit daemon status\` or \`openmerit daemon stop\`.`);
235
+ const info = readDaemonInfo();
236
+ if (info && await healthy(info))
237
+ return void process.stdout.write(`running: openmerit ${info.version}, pid ${info.pid}, 127.0.0.1:${info.port}, since ${info.startedAt}\n`);
238
+ process.stdout.write("not running\n");
239
+ }
240
+ async function main() {
241
+ if (command === "help" || command === "--help" || command === "-h")
242
+ return void process.stdout.write(`${HELP}\n`);
243
+ if (command === "--version" || command === "-v" || command === "version")
244
+ return void process.stdout.write(`${(await import("./update.js")).packageVersion()}\n`);
245
+ if (command === "mcp")
246
+ return (await import("./mcp.js")).serveMcp();
247
+ if (command === "daemon")
248
+ return daemon(rest);
249
+ if (!["setup", "connect", "scout", "history"].includes(command))
250
+ throw new Error(`Unknown command "${command}". Run \`openmerit help\`.`);
251
+ const took = await update(rest.includes("--json"));
252
+ if (took !== null) {
253
+ process.exitCode = took;
173
254
  return;
174
255
  }
175
- throw new Error(`Unknown command "${command}". Run \`openmerit help\`.`);
256
+ if (command === "setup")
257
+ return setup(rest);
258
+ if (command === "connect")
259
+ return connectCommand(rest);
260
+ if (command === "scout")
261
+ return scout(rest);
262
+ return history(rest);
176
263
  }
177
264
  main().catch((error) => {
178
265
  process.stderr.write(`openmerit: ${error instanceof Error ? error.message : String(error)}\n`);
@@ -0,0 +1,34 @@
1
+ import type { Prompter } from "./cli/prompt.js";
2
+ import { type CredentialRef, type HarnessKind, type HarnessSettings } from "./state/config.js";
3
+ export interface ConnectFlags {
4
+ /** Model for claude-code (default DEFAULT_CLAUDE_MODEL) or codex (default: the best the ChatGPT plan allows). */
5
+ model?: string;
6
+ apiProvider?: string;
7
+ apiModel?: string;
8
+ apiKeyEnv?: string;
9
+ /** Sign in to ChatGPT with a device code instead of the browser (the default without a terminal). */
10
+ deviceCode?: boolean;
11
+ /** Accept defaults and never prompt. */
12
+ nonInteractive?: boolean;
13
+ }
14
+ export interface PromptContext {
15
+ prompter: Prompter | null;
16
+ log: (line: string) => void;
17
+ env: Record<string, string | undefined>;
18
+ flags: {
19
+ nonInteractive?: boolean;
20
+ };
21
+ }
22
+ export interface ConnectContext extends PromptContext {
23
+ root: string;
24
+ flags: ConnectFlags;
25
+ }
26
+ export declare const HARNESS_LABEL: Record<HarnessKind, string>;
27
+ export declare function confirm(ctx: PromptContext, question: string, fallback: boolean): Promise<boolean>;
28
+ /** A key from an environment variable (kept as a reference) or typed in (stored in ~/.openmerit/secrets.json). */
29
+ export declare function credential(ctx: PromptContext, label: string, envNames: string[], secretName: string, flagEnv?: string): Promise<CredentialRef | null>;
30
+ /**
31
+ * Connects OpenMerit Harness. With `kind`, connects that; without it, keeps a working saved
32
+ * connection when the user agrees, and otherwise asks which to use.
33
+ */
34
+ export declare function connect(ctx: ConnectContext, kind?: HarnessKind): Promise<HarnessSettings | null>;