openmerit 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +239 -58
- package/data/catalogue.json +6778 -0
- package/dist/catalogue/catalogue.d.ts +41 -0
- package/dist/catalogue/catalogue.js +33 -0
- package/dist/cli/prompt.d.ts +26 -0
- package/dist/cli/prompt.js +114 -0
- package/dist/cli/render.d.ts +12 -0
- package/dist/cli/render.js +104 -0
- package/dist/cli.js +245 -158
- package/dist/connect.d.ts +34 -0
- package/dist/connect.js +158 -0
- package/dist/connectors/langfuse/index.d.ts +37 -110
- package/dist/connectors/langfuse/index.js +95 -246
- package/dist/daemon/daemon.d.ts +81 -0
- package/dist/daemon/daemon.js +408 -0
- package/dist/daemon/registry.d.ts +111 -0
- package/dist/daemon/registry.js +109 -0
- package/dist/eval/proxy.d.ts +54 -0
- package/dist/eval/proxy.js +174 -0
- package/dist/eval/runner.d.ts +29 -0
- package/dist/eval/runner.js +80 -0
- package/dist/harness/claude-bridge.d.ts +21 -0
- package/dist/harness/claude-bridge.js +65 -0
- package/dist/harness/harness.d.ts +60 -0
- package/dist/harness/harness.js +221 -0
- package/dist/index.d.ts +9 -10
- package/dist/index.js +9 -10
- package/dist/local.d.ts +12 -0
- package/dist/local.js +40 -0
- package/dist/mcp.d.ts +1 -0
- package/dist/mcp.js +155 -0
- package/dist/scout/pool.d.ts +17 -0
- package/dist/scout/pool.js +56 -0
- package/dist/scout/scan.d.ts +42 -0
- package/dist/scout/scan.js +169 -0
- package/dist/scout/search.d.ts +25 -98
- package/dist/scout/search.js +56 -156
- package/dist/scout/workflow.d.ts +221 -0
- package/dist/scout/workflow.js +941 -0
- package/dist/setup.d.ts +44 -0
- package/dist/setup.js +211 -0
- package/dist/state/config.d.ts +116 -0
- package/dist/state/config.js +60 -0
- package/dist/state/db.d.ts +159 -0
- package/dist/state/db.js +260 -0
- package/dist/state/paths.d.ts +12 -0
- package/dist/state/paths.js +23 -0
- package/dist/update.d.ts +22 -0
- package/dist/update.js +58 -0
- package/package.json +12 -17
- package/dist/connectors/langfuse/otlp.d.ts +0 -60
- package/dist/connectors/langfuse/otlp.js +0 -40
- package/dist/pi/extension.d.ts +0 -13
- package/dist/pi/extension.js +0 -148
- package/dist/pi/harness.d.ts +0 -20
- package/dist/pi/harness.js +0 -49
- package/dist/pi/instructions.d.ts +0 -2
- package/dist/pi/instructions.js +0 -12
- package/dist/scout/evaluators/module.d.ts +0 -24
- package/dist/scout/evaluators/module.js +0 -19
- package/dist/scout/evaluators/pi.d.ts +0 -53
- package/dist/scout/evaluators/pi.js +0 -117
- package/dist/scout/pipeline.d.ts +0 -111
- package/dist/scout/pipeline.js +0 -157
- package/dist/scout/report.d.ts +0 -3
- package/dist/scout/report.js +0 -42
- package/dist/scout/service.d.ts +0 -148
- package/dist/scout/service.js +0 -143
- package/examples/health-declaration-codes/dataset.jsonl +0 -14
- package/examples/sahra-enrollment/evaluator.mts +0 -152
package/dist/cli.js
CHANGED
|
@@ -1,178 +1,265 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { spawnSync } from "node:child_process";
|
|
3
|
+
import { realpathSync } from "node:fs";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
2
5
|
import { parseArgs } from "node:util";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const HELP = `OpenMerit: find the right model for your task.
|
|
6
|
+
const HELP = `OpenMerit: find cheaper models that perform at least as well as the ones your code uses today,
|
|
7
|
+
measured with your own evals. Runs locally; no OpenMerit account.
|
|
6
8
|
|
|
7
9
|
Usage:
|
|
8
|
-
openmerit
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
openmerit
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
openmerit
|
|
10
|
+
openmerit setup Connect OpenMerit Harness (as openmerit connect does), register the MCP
|
|
11
|
+
server, and configure this repository: read-only Langfuse, candidate
|
|
12
|
+
endpoints and allowed models, default eval budget. Rerunnable.
|
|
13
|
+
openmerit connect [claude-code|codex|api]
|
|
14
|
+
Power OpenMerit Harness with Claude Code (its own login), Codex models
|
|
15
|
+
(a ChatGPT Plus/Pro sign-in) or an API key. Signs in when needed.
|
|
16
|
+
openmerit scout Scan model usages, pick a model and tasks, binary-search cheaper models on your eval
|
|
17
|
+
candidates, compare with the current model, optionally configure.
|
|
18
|
+
openmerit scout --resume <run> Continue a paused, interrupted or failed run.
|
|
19
|
+
openmerit history [<run>] This repository's runs, or one run in detail.
|
|
20
|
+
openmerit mcp The local MCP server (stdio), for Claude Code, Codex and other clients.
|
|
21
|
+
openmerit daemon [status|stop] The background process that runs Scout (started by scout and mcp when needed).
|
|
22
|
+
Without an argument it runs in the foreground.
|
|
15
23
|
|
|
16
|
-
|
|
17
|
-
--
|
|
18
|
-
--
|
|
19
|
-
--dataset <file.jsonl> Local labelled items {id?, input, expectedOutput}; pushed to Langfuse
|
|
20
|
-
--evaluator <module.mts> Evaluator module that runs your real harness (relative to the current directory)
|
|
21
|
-
--system-prompt <text> run: production system prompt for the built-in Pi evaluator
|
|
22
|
-
--grader <exact|contains|json|judge> run: how the Pi evaluator checks a reply (default json)
|
|
23
|
-
--rubric <text> run: rubric for the judge grader
|
|
24
|
-
--task <text> run: task description recorded with the experiments
|
|
25
|
-
--providers <a,b> Candidate providers (default: every connected provider)
|
|
26
|
-
--tools Candidates must support tool calls
|
|
27
|
-
--max-price <usd> Max list price per million input+output tokens
|
|
28
|
-
--exclude <a,b> Drop candidates whose provider/id contains any of these
|
|
29
|
-
--model <provider/id> Model Scout itself runs on
|
|
30
|
-
--sample-size <n> Items per binary-search probe (default 5)
|
|
31
|
-
--finalists <n> Finalists run on every item (default 5)
|
|
32
|
-
--repeats <n> Runs of every item per finalist (default 3)
|
|
33
|
-
--execute Allow spending provider credit (otherwise plan only)
|
|
34
|
-
--json Machine-readable output
|
|
24
|
+
Scout options:
|
|
25
|
+
--answer <id=value> Answer a question without a prompt (repeatable), as --json returns them
|
|
26
|
+
--json Print the run state as JSON and stop at the first question
|
|
35
27
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
28
|
+
Connect options (setup takes them too, with --harness <claude-code|codex|api> and --harness-model <id>):
|
|
29
|
+
--model <id> Model for claude-code or codex
|
|
30
|
+
--device-code Sign in to ChatGPT with a code entered on any device (for servers)
|
|
31
|
+
--api-provider <pi provider> --api-model <id> --api-key-env <VAR>
|
|
32
|
+
--yes Accept defaults; never prompt
|
|
33
|
+
|
|
34
|
+
Setup options (for scripts; interactive setup asks instead):
|
|
35
|
+
--harness <claude-code|codex|api> --harness-model <id> --device-code
|
|
36
|
+
--api-provider <pi provider> --api-model <id> --api-key-env <VAR>
|
|
37
|
+
--endpoint <openrouter:VAR | openai:VAR | id=https://host/v1:VAR> (repeatable)
|
|
38
|
+
--allowed-models <a,b> --budget <usd> --register-mcp | --no-register-mcp
|
|
39
|
+
--langfuse-url <url> --langfuse-public-key-env <VAR> --langfuse-secret-key-env <VAR>
|
|
40
|
+
--yes Accept defaults; never prompt`;
|
|
41
|
+
const [command = "help", ...rest] = process.argv.slice(2);
|
|
42
|
+
const log = (line) => process.stderr.write(`${line}\n`);
|
|
43
|
+
async function update(json) {
|
|
44
|
+
const { checkForUpdate, offerUpdate } = await import("./update.js");
|
|
45
|
+
const info = await checkForUpdate();
|
|
46
|
+
if (!info.available)
|
|
47
|
+
return null;
|
|
48
|
+
const interactive = process.stdin.isTTY === true && !json;
|
|
49
|
+
if (!interactive) {
|
|
50
|
+
log(`openmerit ${info.latest} is available (installed ${info.current}). Update with: npm install -g openmerit@${info.latest}`);
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
const { terminalPrompter } = await import("./cli/prompt.js");
|
|
54
|
+
const prompter = terminalPrompter();
|
|
55
|
+
try {
|
|
56
|
+
return await offerUpdate(info, { interactive, confirm: question => prompter.confirm(question, true), log, argv: process.argv.slice(2) });
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
prompter.close();
|
|
60
|
+
}
|
|
45
61
|
}
|
|
46
|
-
function
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
},
|
|
56
|
-
});
|
|
62
|
+
function mcpCommand() {
|
|
63
|
+
const self = realpathSync(fileURLToPath(import.meta.url));
|
|
64
|
+
const found = spawnSync("sh", ["-c", "command -v openmerit"], { encoding: "utf8" }).stdout.trim();
|
|
65
|
+
try {
|
|
66
|
+
if (found && realpathSync(found) === self)
|
|
67
|
+
return ["openmerit", "mcp"];
|
|
68
|
+
}
|
|
69
|
+
catch { /* fall through */ }
|
|
70
|
+
return [process.execPath, self, "mcp"];
|
|
57
71
|
}
|
|
58
|
-
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
const
|
|
67
|
-
|
|
72
|
+
async function setup(args) {
|
|
73
|
+
const { values } = parseArgs({ args, options: {
|
|
74
|
+
harness: { type: "string" }, "harness-model": { type: "string" }, "api-provider": { type: "string" }, "api-model": { type: "string" }, "api-key-env": { type: "string" },
|
|
75
|
+
endpoint: { type: "string", multiple: true }, "allowed-models": { type: "string" }, budget: { type: "string" }, "register-mcp": { type: "boolean" }, "no-register-mcp": { type: "boolean" },
|
|
76
|
+
"langfuse-url": { type: "string" }, "langfuse-public-key-env": { type: "string" }, "langfuse-secret-key-env": { type: "string" }, yes: { type: "boolean" }, "device-code": { type: "boolean" },
|
|
77
|
+
} });
|
|
78
|
+
if (values.harness && !["claude-code", "codex", "api"].includes(values.harness))
|
|
79
|
+
throw new Error("--harness is claude-code, codex or api");
|
|
80
|
+
const { runSetup } = await import("./setup.js");
|
|
81
|
+
const { State } = await import("./state/db.js");
|
|
82
|
+
const { findRepo } = await import("./state/paths.js");
|
|
83
|
+
const { terminalPrompter } = await import("./cli/prompt.js");
|
|
84
|
+
const interactive = process.stdin.isTTY === true && !values.yes;
|
|
85
|
+
const prompter = interactive ? terminalPrompter() : null;
|
|
86
|
+
const state = new State();
|
|
87
|
+
try {
|
|
88
|
+
const result = await runSetup({
|
|
89
|
+
repo: findRepo(process.cwd()), state, prompter, log, env: process.env, mcpCommand: mcpCommand(),
|
|
90
|
+
flags: {
|
|
91
|
+
...(values.harness ? { harness: values.harness } : {}), ...(values["harness-model"] ? { harnessModel: values["harness-model"] } : {}),
|
|
92
|
+
...(values["device-code"] ? { deviceCode: true } : {}), ...(values["api-provider"] ? { apiProvider: values["api-provider"] } : {}), ...(values["api-model"] ? { apiModel: values["api-model"] } : {}), ...(values["api-key-env"] ? { apiKeyEnv: values["api-key-env"] } : {}),
|
|
93
|
+
...(values.endpoint ? { endpoints: values.endpoint } : {}), ...(values["allowed-models"] ? { allowedModels: values["allowed-models"].split(",").map(item => item.trim()).filter(Boolean) } : {}),
|
|
94
|
+
...(values.budget ? { budget: Number(values.budget) } : {}), ...(values["register-mcp"] ? { registerMcp: true } : values["no-register-mcp"] ? { registerMcp: false } : {}),
|
|
95
|
+
...(values["langfuse-url"] ? { langfuseUrl: values["langfuse-url"] } : {}), ...(values["langfuse-public-key-env"] ? { langfusePublicKeyEnv: values["langfuse-public-key-env"] } : {}),
|
|
96
|
+
...(values["langfuse-secret-key-env"] ? { langfuseSecretKeyEnv: values["langfuse-secret-key-env"] } : {}), ...(!interactive ? { nonInteractive: true } : {}),
|
|
97
|
+
},
|
|
98
|
+
});
|
|
99
|
+
if (!result.harness)
|
|
100
|
+
process.exitCode = 1;
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
prompter?.close();
|
|
104
|
+
state.close();
|
|
105
|
+
}
|
|
68
106
|
}
|
|
69
|
-
async function
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
if (
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
107
|
+
async function connectCommand(args) {
|
|
108
|
+
const { values, positionals } = parseArgs({ args, allowPositionals: true, options: {
|
|
109
|
+
model: { type: "string" }, "device-code": { type: "boolean" }, "api-provider": { type: "string" }, "api-model": { type: "string" }, "api-key-env": { type: "string" }, yes: { type: "boolean" },
|
|
110
|
+
} });
|
|
111
|
+
const [kind] = positionals;
|
|
112
|
+
if (kind && !["claude-code", "codex", "api"].includes(kind))
|
|
113
|
+
throw new Error("Connect claude-code, codex or api.");
|
|
114
|
+
const { connect } = await import("./connect.js");
|
|
115
|
+
const { findRepo } = await import("./state/paths.js");
|
|
116
|
+
const { terminalPrompter } = await import("./cli/prompt.js");
|
|
117
|
+
const interactive = process.stdin.isTTY === true && !values.yes;
|
|
118
|
+
const prompter = interactive ? terminalPrompter() : null;
|
|
119
|
+
let root = process.cwd();
|
|
120
|
+
try {
|
|
121
|
+
root = findRepo(root).root;
|
|
82
122
|
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
process.stdout.write(`Scout model: ${status.scoutModel ?? "none"}\n`);
|
|
94
|
-
process.stdout.write(`Langfuse: ${status.langfuse.connected ? `${status.langfuse.project} at ${status.langfuse.baseUrl}; datasets ${status.langfuse.datasets?.join(", ") || "none"}` : `not connected${status.langfuse.error ? ` (${status.langfuse.error})` : ""}`}\n`);
|
|
95
|
-
return;
|
|
123
|
+
catch { /* outside a repository: the check reads nothing */ }
|
|
124
|
+
try {
|
|
125
|
+
const saved = await connect({ root, prompter, log, env: process.env, flags: {
|
|
126
|
+
...(values.model ? { model: values.model } : {}), ...(values["device-code"] ? { deviceCode: true } : {}), ...(values["api-provider"] ? { apiProvider: values["api-provider"] } : {}),
|
|
127
|
+
...(values["api-model"] ? { apiModel: values["api-model"] } : {}), ...(values["api-key-env"] ? { apiKeyEnv: values["api-key-env"] } : {}), ...(!interactive ? { nonInteractive: true } : {}),
|
|
128
|
+
} }, kind);
|
|
129
|
+
if (saved)
|
|
130
|
+
log("Saved to ~/.openmerit/config.json. Next: openmerit setup in a repository, or openmerit scout.");
|
|
131
|
+
else
|
|
132
|
+
process.exitCode = 1;
|
|
96
133
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
const ranked = await service.candidates(host, filterFrom(values));
|
|
100
|
-
if (values.json)
|
|
101
|
-
return void process.stdout.write(`${JSON.stringify(ranked, null, 2)}\n`);
|
|
102
|
-
for (const item of ranked.candidates)
|
|
103
|
-
process.stdout.write(`${item.eci.toFixed(1).padStart(6)} ${item.eciSource === "published" ? " " : `${item.eciSource.slice(0, 4)} ±${(item.eciError ?? 0).toFixed(1)}`.padEnd(9)} ${item.model} ${item.price === null ? "?" : `$${item.price.toFixed(2)}/M`}\n`);
|
|
104
|
-
process.stdout.write(`${ranked.candidates.length} candidates (${ranked.counts.published} published ECI, ${ranked.counts.estimated} estimated, ${ranked.counts.calibrated} calibrated from data/eci-estimates.csv), ${ranked.unranked.length} unranked, ${ranked.dropped.length} filtered. ${ranked.eci.attribution}\n`);
|
|
105
|
-
return;
|
|
134
|
+
finally {
|
|
135
|
+
prompter?.close();
|
|
106
136
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
const out = { status: "plan_only", candidates: planned.ranked.candidates.length, datasetSize: planned.datasetSize, estimate: planned.estimate, evaluator: planned.plugin?.name ?? "pi", note: "Add --execute to run it." };
|
|
128
|
-
return void process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
|
|
137
|
+
}
|
|
138
|
+
async function scout(args) {
|
|
139
|
+
const { values } = parseArgs({ args, options: { resume: { type: "string" }, answer: { type: "string", multiple: true }, json: { type: "boolean", default: false } } });
|
|
140
|
+
const { localContext, missingSetup, startRun } = await import("./local.js");
|
|
141
|
+
const { advanceInDaemon } = await import("./daemon/daemon.js");
|
|
142
|
+
const { renderComparison, renderStep } = await import("./cli/render.js");
|
|
143
|
+
const { askQuestion, terminalPrompter, workingTicker } = await import("./cli/prompt.js");
|
|
144
|
+
const controller = new AbortController();
|
|
145
|
+
let interrupts = 0;
|
|
146
|
+
process.on("SIGINT", () => { if (++interrupts > 1)
|
|
147
|
+
process.exit(130); log("\nStopping; the run is saved. Ctrl-C again to quit now."); controller.abort(); });
|
|
148
|
+
const ctx = localContext({ cwd: process.cwd(), log, signal: controller.signal });
|
|
149
|
+
const interactive = process.stdin.isTTY === true && !values.json;
|
|
150
|
+
const prompter = interactive ? terminalPrompter() : null;
|
|
151
|
+
try {
|
|
152
|
+
const missing = missingSetup(ctx);
|
|
153
|
+
if (missing.length) {
|
|
154
|
+
if (values.json)
|
|
155
|
+
return void process.stdout.write(`${JSON.stringify({ status: "needs_setup", missing }, null, 2)}\n`);
|
|
156
|
+
throw new Error(`Run \`openmerit setup\` first:\n - ${missing.join("\n - ")}`);
|
|
129
157
|
}
|
|
130
|
-
const
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
const answer = session.getLastAssistantText() ?? "";
|
|
168
|
-
process.stdout.write(values.json ? `${JSON.stringify({ task, scoutModel: host.scoutModel, answer }, null, 2)}\n` : `${answer}\n`);
|
|
158
|
+
const runId = values.resume ?? startRun(ctx);
|
|
159
|
+
let answers = Object.fromEntries((values.answer ?? []).map(item => {
|
|
160
|
+
const at = item.indexOf("=");
|
|
161
|
+
if (at < 1)
|
|
162
|
+
throw new Error(`--answer ${item}: write <question id>=<value>`);
|
|
163
|
+
return [item.slice(0, at), item.slice(at + 1)];
|
|
164
|
+
}));
|
|
165
|
+
let compared = false;
|
|
166
|
+
for (;;) {
|
|
167
|
+
const ticker = values.json ? null : workingTicker();
|
|
168
|
+
const result = await advanceInDaemon({ runId, cwd: process.cwd(), env: process.env, answers, log: ticker?.log ?? log, signal: controller.signal }).finally(() => ticker?.stop());
|
|
169
|
+
answers = {};
|
|
170
|
+
for (const notice of result.notices)
|
|
171
|
+
log(notice);
|
|
172
|
+
if (result.questions.length && prompter && !controller.signal.aborted) {
|
|
173
|
+
// The comparison comes before the question whether to configure its result.
|
|
174
|
+
if (result.result && !compared) {
|
|
175
|
+
process.stdout.write(`${result.result.tasks.map(task => renderComparison(task, runId)).join("\n\n")}\n`);
|
|
176
|
+
compared = true;
|
|
177
|
+
}
|
|
178
|
+
for (const question of result.questions) {
|
|
179
|
+
if (question.id === "budget_amount" && answers.budget_action === "decline")
|
|
180
|
+
continue;
|
|
181
|
+
answers[question.id] = await askQuestion(prompter, question, log);
|
|
182
|
+
}
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
if (values.json)
|
|
186
|
+
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
187
|
+
else {
|
|
188
|
+
if (result.questions.length)
|
|
189
|
+
log(`Waiting for ${result.questions.map(question => question.id).join(", ")}. Answer: openmerit scout --resume ${runId} --answer <id>=<value>`);
|
|
190
|
+
process.stdout.write(`${renderStep(result, { comparisons: !compared })}\n`);
|
|
191
|
+
}
|
|
192
|
+
if (result.status === "failed" || result.status === "interrupted")
|
|
193
|
+
process.exitCode = 1;
|
|
194
|
+
return;
|
|
169
195
|
}
|
|
170
|
-
|
|
171
|
-
|
|
196
|
+
}
|
|
197
|
+
finally {
|
|
198
|
+
prompter?.close();
|
|
199
|
+
ctx.state.close();
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
async function history(args) {
|
|
203
|
+
const { values, positionals } = parseArgs({ args, allowPositionals: true, options: { json: { type: "boolean", default: false } } });
|
|
204
|
+
const { localContext } = await import("./local.js");
|
|
205
|
+
const { step } = await import("./scout/workflow.js");
|
|
206
|
+
const { renderHistory, renderRun } = await import("./cli/render.js");
|
|
207
|
+
const ctx = localContext({ cwd: process.cwd() });
|
|
208
|
+
try {
|
|
209
|
+
ctx.state.reapInterrupted(ctx.repo.id);
|
|
210
|
+
const id = positionals[0];
|
|
211
|
+
if (id) {
|
|
212
|
+
const run = ctx.state.getRun(id);
|
|
213
|
+
if (!run || run.repoId !== ctx.repo.id)
|
|
214
|
+
throw new Error(`No run ${id} in ${ctx.repo.root}.`);
|
|
215
|
+
const current = step(ctx, id, [], []);
|
|
216
|
+
process.stdout.write(values.json ? `${JSON.stringify({ run, step: current, units: ctx.state.units(id), budget: ctx.state.budgetEvents(id), decisions: ctx.state.decisions(id), events: ctx.state.events(id) }, null, 2)}\n` : `${renderRun(ctx.state, run, current)}\n`);
|
|
217
|
+
return;
|
|
172
218
|
}
|
|
219
|
+
const runs = ctx.state.listRuns(ctx.repo.id);
|
|
220
|
+
process.stdout.write(values.json ? `${JSON.stringify(runs.map(run => ({ ...run, spend: ctx.state.spend(run.id) })), null, 2)}\n` : `${renderHistory(ctx.state, runs)}\n`);
|
|
221
|
+
}
|
|
222
|
+
finally {
|
|
223
|
+
ctx.state.close();
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
async function daemon(args) {
|
|
227
|
+
const { healthy, readDaemonInfo, serveDaemon, stopDaemon } = await import("./daemon/daemon.js");
|
|
228
|
+
const [action] = args;
|
|
229
|
+
if (!action)
|
|
230
|
+
return serveDaemon();
|
|
231
|
+
if (action === "stop")
|
|
232
|
+
return void log((await stopDaemon()) ? "Stopped the OpenMerit daemon. Runs it was working on continue with `openmerit scout --resume <run>`." : "The OpenMerit daemon is not running.");
|
|
233
|
+
if (action !== "status")
|
|
234
|
+
throw new Error(`Unknown daemon command "${action}". Use \`openmerit daemon status\` or \`openmerit daemon stop\`.`);
|
|
235
|
+
const info = readDaemonInfo();
|
|
236
|
+
if (info && await healthy(info))
|
|
237
|
+
return void process.stdout.write(`running: openmerit ${info.version}, pid ${info.pid}, 127.0.0.1:${info.port}, since ${info.startedAt}\n`);
|
|
238
|
+
process.stdout.write("not running\n");
|
|
239
|
+
}
|
|
240
|
+
async function main() {
|
|
241
|
+
if (command === "help" || command === "--help" || command === "-h")
|
|
242
|
+
return void process.stdout.write(`${HELP}\n`);
|
|
243
|
+
if (command === "--version" || command === "-v" || command === "version")
|
|
244
|
+
return void process.stdout.write(`${(await import("./update.js")).packageVersion()}\n`);
|
|
245
|
+
if (command === "mcp")
|
|
246
|
+
return (await import("./mcp.js")).serveMcp();
|
|
247
|
+
if (command === "daemon")
|
|
248
|
+
return daemon(rest);
|
|
249
|
+
if (!["setup", "connect", "scout", "history"].includes(command))
|
|
250
|
+
throw new Error(`Unknown command "${command}". Run \`openmerit help\`.`);
|
|
251
|
+
const took = await update(rest.includes("--json"));
|
|
252
|
+
if (took !== null) {
|
|
253
|
+
process.exitCode = took;
|
|
173
254
|
return;
|
|
174
255
|
}
|
|
175
|
-
|
|
256
|
+
if (command === "setup")
|
|
257
|
+
return setup(rest);
|
|
258
|
+
if (command === "connect")
|
|
259
|
+
return connectCommand(rest);
|
|
260
|
+
if (command === "scout")
|
|
261
|
+
return scout(rest);
|
|
262
|
+
return history(rest);
|
|
176
263
|
}
|
|
177
264
|
main().catch((error) => {
|
|
178
265
|
process.stderr.write(`openmerit: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { Prompter } from "./cli/prompt.js";
|
|
2
|
+
import { type CredentialRef, type HarnessKind, type HarnessSettings } from "./state/config.js";
|
|
3
|
+
export interface ConnectFlags {
|
|
4
|
+
/** Model for claude-code (default DEFAULT_CLAUDE_MODEL) or codex (default: the best the ChatGPT plan allows). */
|
|
5
|
+
model?: string;
|
|
6
|
+
apiProvider?: string;
|
|
7
|
+
apiModel?: string;
|
|
8
|
+
apiKeyEnv?: string;
|
|
9
|
+
/** Sign in to ChatGPT with a device code instead of the browser (the default without a terminal). */
|
|
10
|
+
deviceCode?: boolean;
|
|
11
|
+
/** Accept defaults and never prompt. */
|
|
12
|
+
nonInteractive?: boolean;
|
|
13
|
+
}
|
|
14
|
+
export interface PromptContext {
|
|
15
|
+
prompter: Prompter | null;
|
|
16
|
+
log: (line: string) => void;
|
|
17
|
+
env: Record<string, string | undefined>;
|
|
18
|
+
flags: {
|
|
19
|
+
nonInteractive?: boolean;
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
export interface ConnectContext extends PromptContext {
|
|
23
|
+
root: string;
|
|
24
|
+
flags: ConnectFlags;
|
|
25
|
+
}
|
|
26
|
+
export declare const HARNESS_LABEL: Record<HarnessKind, string>;
|
|
27
|
+
export declare function confirm(ctx: PromptContext, question: string, fallback: boolean): Promise<boolean>;
|
|
28
|
+
/** A key from an environment variable (kept as a reference) or typed in (stored in ~/.openmerit/secrets.json). */
|
|
29
|
+
export declare function credential(ctx: PromptContext, label: string, envNames: string[], secretName: string, flagEnv?: string): Promise<CredentialRef | null>;
|
|
30
|
+
/**
|
|
31
|
+
* Connects OpenMerit Harness. With `kind`, connects that; without it, keeps a working saved
|
|
32
|
+
* connection when the user agrees, and otherwise asks which to use.
|
|
33
|
+
*/
|
|
34
|
+
export declare function connect(ctx: ConnectContext, kind?: HarnessKind): Promise<HarnessSettings | null>;
|