openmerit 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -18
- package/dist/cli.js +26 -153
- package/dist/daemon.js +56 -14
- package/dist/diagnostics.js +227 -0
- package/dist/llm.js +6 -2
- package/dist/pi-config.js +46 -0
- package/dist/pi-trials.js +30 -10
- package/dist/policy.js +71 -1
- package/dist/routes.js +15 -0
- package/dist/standalone.js +224 -0
- package/dist/store.js +42 -10
- package/dist/strategist.js +1 -1
- package/dist/trials.js +13 -4
- package/examples/task.example.json +1 -0
- package/extension/openmerit.ts +182 -29
- package/instructions/OPENMERIT.md +9 -0
- package/instructions/openmerit.policy.json +6 -2
- package/package.json +1 -1
- package/rules.md +4 -0
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/** Read-only diagnostics plus a provider-free self-test for first-run support. */
|
|
2
|
+
import { spawnSync } from "node:child_process";
|
|
3
|
+
import { appendFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { basename, join } from "node:path";
|
|
6
|
+
import { LocalJsonlEventSink, meritEvent } from "./integrations.js";
|
|
7
|
+
import { piExecutable, availablePiRoutes } from "./pi-trials.js";
|
|
8
|
+
import { DEFAULT_POLICY, loadPolicy, providerAllowed } from "./policy.js";
|
|
9
|
+
import { buildRecommendation } from "./recommend.js";
|
|
10
|
+
import { applyRouteOverrides, modelRoute, routeCatalog, routeKey, routeLabel } from "./routes.js";
|
|
11
|
+
import { validatePiProviderExtensions } from "./pi-config.js";
|
|
12
|
+
import { appendJsonl, paths, readJsonReport, readJsonlReport, writeJson, } from "./store.js";
|
|
13
|
+
function check(id, status, message, details) {
|
|
14
|
+
return { id, status, message, ...(details?.length ? { details } : {}) };
|
|
15
|
+
}
|
|
16
|
+
function packageSources(output) {
|
|
17
|
+
return [...new Set(output.split("\n").filter((line) => /^ {2}\S/.test(line)).map((line) => line.trim())
|
|
18
|
+
.filter((line) => /openmerit/i.test(line) && /^(npm:|git:|https?:|\.?\.?\/|\/)/.test(line)))];
|
|
19
|
+
}
|
|
20
|
+
function safePackageSource(source) {
|
|
21
|
+
if (source.startsWith("npm:/"))
|
|
22
|
+
return `npm:local/${basename(source)}`;
|
|
23
|
+
if (source.startsWith("/"))
|
|
24
|
+
return `local:${basename(source)}`;
|
|
25
|
+
return source;
|
|
26
|
+
}
|
|
27
|
+
/** No credentials, prompts, or trace contents are included in this report. */
|
|
28
|
+
export function collectDoctorReport() {
|
|
29
|
+
const checks = [];
|
|
30
|
+
const executable = piExecutable();
|
|
31
|
+
const versionResult = spawnSync(executable, ["--version"], { encoding: "utf8" });
|
|
32
|
+
const piVersion = versionResult.status === 0 ? versionResult.stdout.trim() : null;
|
|
33
|
+
checks.push(piVersion
|
|
34
|
+
? check("pi", "pass", `Pi detected (${piVersion})`)
|
|
35
|
+
: check("pi", "fail", "Pi is not executable"));
|
|
36
|
+
let policy = DEFAULT_POLICY;
|
|
37
|
+
try {
|
|
38
|
+
policy = loadPolicy(paths.policy());
|
|
39
|
+
checks.push(check("policy", "pass", `Policy is valid (v${policy.version}, ${policy.mode})`));
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
checks.push(check("policy", "fail", `Policy is invalid: ${error.message}`));
|
|
43
|
+
}
|
|
44
|
+
const extensions = validatePiProviderExtensions(policy.pi.provider_extensions);
|
|
45
|
+
const safeExtensionErrors = extensions.errors.map((message) => {
|
|
46
|
+
const configured = policy.pi.provider_extensions.find((file) => message.startsWith(`${file}:`));
|
|
47
|
+
return configured ? message.replace(configured, `local:${basename(configured)}`) : message;
|
|
48
|
+
});
|
|
49
|
+
let isolatedExtensionRoutes = null;
|
|
50
|
+
let extensionLoadError = null;
|
|
51
|
+
if (!safeExtensionErrors.length && policy.pi.provider_extensions.length && piVersion) {
|
|
52
|
+
try {
|
|
53
|
+
isolatedExtensionRoutes = availablePiRoutes(undefined, extensions.valid);
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
extensionLoadError = "Pi could not load the allowlisted provider extensions in isolated mode";
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
checks.push(safeExtensionErrors.length || extensionLoadError
|
|
60
|
+
? check("provider-extensions", "fail", "One or more allowlisted Pi provider extensions are unavailable", [...safeExtensionErrors, ...(extensionLoadError ? [extensionLoadError] : [])])
|
|
61
|
+
: check("provider-extensions", "pass", policy.pi.provider_extensions.length
|
|
62
|
+
? `${policy.pi.provider_extensions.length} explicit Pi provider extension${policy.pi.provider_extensions.length === 1 ? " loads" : "s load"} in isolated mode`
|
|
63
|
+
: "No Pi provider extensions are allowlisted"));
|
|
64
|
+
const stateRead = readJsonReport(paths.harnessState(), {});
|
|
65
|
+
checks.push(!stateRead.valid
|
|
66
|
+
? check("harness-state", "fail", "harness-state.json is malformed; start Pi to replace it atomically")
|
|
67
|
+
: stateRead.exists
|
|
68
|
+
? check("harness-state", "pass", "Pi extension state snapshot is readable")
|
|
69
|
+
: check("harness-state", "warn", "No Pi extension state yet; start Pi once with OpenMerit installed"));
|
|
70
|
+
let routes = stateRead.value.routes ?? [];
|
|
71
|
+
const liveSnapshot = routes.length > 0;
|
|
72
|
+
if (!routes.length && piVersion) {
|
|
73
|
+
try {
|
|
74
|
+
routes = isolatedExtensionRoutes ?? availablePiRoutes(undefined, extensions.valid);
|
|
75
|
+
}
|
|
76
|
+
catch { /* the check below explains it */ }
|
|
77
|
+
}
|
|
78
|
+
routes = applyRouteOverrides(routes, policy.route_overrides);
|
|
79
|
+
const visibleRouteKeys = new Set(routes.map(routeKey));
|
|
80
|
+
routes = routes.filter((route) => providerAllowed(policy, route.meritId, route.provider));
|
|
81
|
+
const unusedOverrides = Object.keys(policy.route_overrides).filter((key) => !visibleRouteKeys.has(key));
|
|
82
|
+
checks.push(unusedOverrides.length
|
|
83
|
+
? check("route-overrides", "warn", `${unusedOverrides.length} route metadata override${unusedOverrides.length === 1 ? " does" : "s do"} not match a visible route`, unusedOverrides)
|
|
84
|
+
: check("route-overrides", "pass", `${Object.keys(policy.route_overrides).length} route metadata override${Object.keys(policy.route_overrides).length === 1 ? "" : "s"} applied or ready`));
|
|
85
|
+
const priced = routes.filter((route) => !!route.cost);
|
|
86
|
+
checks.push(routes.length >= 2
|
|
87
|
+
? check("routes", "pass", `${routes.length} Pi model routes are visible`, routes.slice(0, 8).map((route) => `${routeLabel(route)}${route.cost ? "" : " (price unknown)"}`))
|
|
88
|
+
: check("routes", "fail", `Only ${routes.length} policy-eligible Pi route${routes.length === 1 ? " is" : "s are"} visible; at least two are required`));
|
|
89
|
+
checks.push(liveSnapshot
|
|
90
|
+
? priced.length >= 2
|
|
91
|
+
? check("pricing", "pass", `${priced.length} eligible routes expose pricing`)
|
|
92
|
+
: check("pricing", "warn", "Fewer than two eligible routes expose pricing; unknown-price routes cannot auto-apply")
|
|
93
|
+
: check("pricing", "warn", "Route prices are unavailable until the extension reports a live Pi snapshot"));
|
|
94
|
+
const session = stateRead.value.sessionFile;
|
|
95
|
+
checks.push(!session
|
|
96
|
+
? check("session", "warn", "No active saved Pi session is reported yet")
|
|
97
|
+
: existsSync(session)
|
|
98
|
+
? check("session", "pass", "The reported Pi session file is readable")
|
|
99
|
+
: check("session", "warn", "The reported Pi session file no longer exists"));
|
|
100
|
+
if (piVersion) {
|
|
101
|
+
const listed = spawnSync(executable, ["list"], { encoding: "utf8" });
|
|
102
|
+
const sources = listed.status === 0 ? packageSources(listed.stdout) : [];
|
|
103
|
+
const safeSources = sources.map(safePackageSource);
|
|
104
|
+
checks.push(sources.length > 1
|
|
105
|
+
? check("installation", "fail", "OpenMerit is installed from multiple Pi package sources", safeSources)
|
|
106
|
+
: sources.length === 1
|
|
107
|
+
? check("installation", "pass", `One Pi package source is installed`, safeSources)
|
|
108
|
+
: check("installation", "warn", "OpenMerit is not listed as an installed Pi package; use `pi install npm:openmerit`"));
|
|
109
|
+
}
|
|
110
|
+
const jsonlFiles = [
|
|
111
|
+
["recommendations.jsonl", paths.recommendations()],
|
|
112
|
+
["trials.jsonl", paths.trials()],
|
|
113
|
+
["traces/observations.jsonl", paths.observations()],
|
|
114
|
+
["events.jsonl", paths.events()],
|
|
115
|
+
];
|
|
116
|
+
const malformed = jsonlFiles.flatMap(([label, file]) => {
|
|
117
|
+
const invalid = readJsonlReport(file).invalidLines;
|
|
118
|
+
return invalid.length ? [`${label}: lines ${invalid.join(", ")}`] : [];
|
|
119
|
+
});
|
|
120
|
+
checks.push(malformed.length
|
|
121
|
+
? check("jsonl", "warn", "Malformed JSONL lines were skipped; valid later records remain readable", malformed)
|
|
122
|
+
: check("jsonl", "pass", "Append-only JSONL state is readable"));
|
|
123
|
+
const ledger = readJsonReport(paths.ledger(), { date: "", trials: 0, usd: 0 });
|
|
124
|
+
checks.push(ledger.valid
|
|
125
|
+
? check("ledger", "pass", "Trial-spend ledger is readable")
|
|
126
|
+
: check("ledger", "fail", "Trial-spend ledger is malformed; spending remains disabled until repaired"));
|
|
127
|
+
let openRouter = false;
|
|
128
|
+
try {
|
|
129
|
+
const env = process.env.OPENROUTER_API_KEY;
|
|
130
|
+
if (env?.trim())
|
|
131
|
+
openRouter = true;
|
|
132
|
+
else if (existsSync(paths.envFile()))
|
|
133
|
+
openRouter = /^\s*OPENROUTER_API_KEY\s*=\s*\S+/m
|
|
134
|
+
.test(readFileSync(paths.envFile(), "utf8"));
|
|
135
|
+
}
|
|
136
|
+
catch { /* optional */ }
|
|
137
|
+
checks.push(check("openrouter-enrichment", "pass", openRouter ? "Optional OpenRouter enrichment is configured" : "Optional OpenRouter enrichment is not configured"));
|
|
138
|
+
const activeRoute = stateRead.value.currentRoute ? routeLabel(stateRead.value.currentRoute) : null;
|
|
139
|
+
return {
|
|
140
|
+
schemaVersion: 1,
|
|
141
|
+
ok: !checks.some((item) => item.status === "fail"),
|
|
142
|
+
checks,
|
|
143
|
+
summary: { piVersion, routeCount: routes.length, pricedRouteCount: priced.length, activeRoute },
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
export function renderDiagnosticReport(report) {
|
|
147
|
+
const icon = { pass: "PASS", warn: "WARN", fail: "FAIL" };
|
|
148
|
+
return report.checks.flatMap((item) => [
|
|
149
|
+
`${icon[item.status].padEnd(4)} ${item.id.padEnd(22)} ${item.message}`,
|
|
150
|
+
...(item.details ?? []).map((detail) => ` - ${detail}`),
|
|
151
|
+
]).join("\n");
|
|
152
|
+
}
|
|
153
|
+
/** Exercise core state, routing, recommendation, and event paths without provider calls. */
|
|
154
|
+
export function runOfflineVerify() {
|
|
155
|
+
const previousHome = process.env.OPENMERIT_HOME;
|
|
156
|
+
const root = mkdtempSync(join(tmpdir(), "openmerit-verify-"));
|
|
157
|
+
const checks = [];
|
|
158
|
+
try {
|
|
159
|
+
process.env.OPENMERIT_HOME = root;
|
|
160
|
+
const active = modelRoute("native", "model-a", {
|
|
161
|
+
input: ["text"], cost: { input: 1, output: 2 }, contextWindow: 8192, maxTokens: 4096,
|
|
162
|
+
});
|
|
163
|
+
const candidate = modelRoute("local", "model-b", {
|
|
164
|
+
input: ["text"], cost: { input: 0.2, output: 0.4 }, contextWindow: 8192, maxTokens: 4096,
|
|
165
|
+
});
|
|
166
|
+
writeJson(paths.harnessState(), { schemaVersion: 1, currentRoute: active, routes: [active, candidate] });
|
|
167
|
+
const state = readJsonReport(paths.harnessState(), {});
|
|
168
|
+
checks.push(state.valid && state.value.routes?.length === 2
|
|
169
|
+
? check("atomic-json", "pass", "Atomic JSON state round-trip passed")
|
|
170
|
+
: check("atomic-json", "fail", "Atomic JSON state round-trip failed"));
|
|
171
|
+
const entries = routeCatalog([active, candidate]);
|
|
172
|
+
const point = (route, score) => {
|
|
173
|
+
const entry = entries.get(routeKey(route));
|
|
174
|
+
return { schemaVersion: 1, model: route.meritId, route, score, price: entry.price,
|
|
175
|
+
priceKnown: entry.priceKnown, ts: new Date().toISOString(), source: "pi_trial" };
|
|
176
|
+
};
|
|
177
|
+
const recommendation = buildRecommendation("verify-task", "offline verification", active.meritId, [point(active, 0.6), point(candidate, 0.9)], DEFAULT_POLICY, "/verify/session.jsonl", active);
|
|
178
|
+
checks.push(recommendation?.recommended.route?.provider === "local" &&
|
|
179
|
+
recommendation.policy.reasons.length > 0
|
|
180
|
+
? check("provider-neutral", "pass", "Exact provider routes and policy reasons survive recommendation")
|
|
181
|
+
: check("provider-neutral", "fail", "Provider route or policy evidence was lost"));
|
|
182
|
+
const configured = applyRouteOverrides([modelRoute("custom", "model-c")], {
|
|
183
|
+
"custom:model-c": { cost: { input: 0, output: 0 }, context_window: 4096,
|
|
184
|
+
max_tokens: 1024, input: ["text"] },
|
|
185
|
+
})[0];
|
|
186
|
+
const providerExtension = join(root, "custom-provider.ts");
|
|
187
|
+
writeFileSync(providerExtension, "export default function provider() {}\n");
|
|
188
|
+
const extension = validatePiProviderExtensions([providerExtension]);
|
|
189
|
+
checks.push(configured.cost?.input === 0 && configured.contextWindow === 4096 && extension.valid.length === 1
|
|
190
|
+
? check("custom-routes", "pass", "Exact route metadata and allowlisted provider extensions validate")
|
|
191
|
+
: check("custom-routes", "fail", "Custom route or provider-extension configuration failed"));
|
|
192
|
+
if (recommendation)
|
|
193
|
+
appendJsonl(paths.recommendations(), recommendation);
|
|
194
|
+
appendJsonl(paths.recommendations(), { schemaVersion: 0, id: "legacy", status: "dismissed" });
|
|
195
|
+
appendFileSync(paths.recommendations(), "{interrupted");
|
|
196
|
+
const history = readJsonlReport(paths.recommendations());
|
|
197
|
+
checks.push(history.records.length === 2 && history.invalidLines.length === 1
|
|
198
|
+
? check("jsonl-recovery", "pass", "Valid records survive legacy and interrupted JSONL lines")
|
|
199
|
+
: check("jsonl-recovery", "fail", "JSONL recovery did not preserve valid history"));
|
|
200
|
+
const sink = new LocalJsonlEventSink();
|
|
201
|
+
if (recommendation)
|
|
202
|
+
sink.emit(meritEvent("recommendation.created", recommendation));
|
|
203
|
+
const events = readJsonlReport(paths.events());
|
|
204
|
+
checks.push(events.records.length === 1
|
|
205
|
+
? check("events", "pass", "Versioned neutral event sink round-trip passed")
|
|
206
|
+
: check("events", "fail", "Neutral event sink round-trip failed"));
|
|
207
|
+
checks.push(loadPolicy(paths.policy()).mode === "recommend"
|
|
208
|
+
? check("compatibility", "pass", "Missing 0.1.x policy fields normalize to safe defaults")
|
|
209
|
+
: check("compatibility", "fail", "Policy defaults are unsafe"));
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
checks.push(check("verify", "fail", error.message));
|
|
213
|
+
}
|
|
214
|
+
finally {
|
|
215
|
+
if (previousHome === undefined)
|
|
216
|
+
delete process.env.OPENMERIT_HOME;
|
|
217
|
+
else
|
|
218
|
+
process.env.OPENMERIT_HOME = previousHome;
|
|
219
|
+
rmSync(root, { recursive: true, force: true });
|
|
220
|
+
}
|
|
221
|
+
return {
|
|
222
|
+
schemaVersion: 1,
|
|
223
|
+
ok: !checks.some((item) => item.status === "fail"),
|
|
224
|
+
checks,
|
|
225
|
+
summary: { piVersion: null, routeCount: 2, pricedRouteCount: 2, activeRoute: "native/model-a via native" },
|
|
226
|
+
};
|
|
227
|
+
}
|
package/dist/llm.js
CHANGED
|
@@ -5,6 +5,7 @@ import { tmpdir } from "node:os";
|
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { paths } from "./store.js";
|
|
7
7
|
import { legacyOpenRouterRoute } from "./routes.js";
|
|
8
|
+
import { piProviderExtensionArgs } from "./pi-config.js";
|
|
8
9
|
const OR = "https://openrouter.ai/api/v1";
|
|
9
10
|
/** Load OPENROUTER_API_KEY from env or ~/.openmerit/.env. */
|
|
10
11
|
export function loadKey() {
|
|
@@ -123,10 +124,12 @@ function textContent(content) {
|
|
|
123
124
|
export class PiCliChatClient {
|
|
124
125
|
executable;
|
|
125
126
|
onSpend;
|
|
127
|
+
providerExtensions;
|
|
126
128
|
id = "pi-cli";
|
|
127
|
-
constructor(executable = process.env.OPENMERIT_PI_BIN?.trim() || "pi", onSpend) {
|
|
129
|
+
constructor(executable = process.env.OPENMERIT_PI_BIN?.trim() || "pi", onSpend, providerExtensions = []) {
|
|
128
130
|
this.executable = executable;
|
|
129
131
|
this.onSpend = onSpend;
|
|
132
|
+
this.providerExtensions = providerExtensions;
|
|
130
133
|
}
|
|
131
134
|
chat(model, prompt, _maxTokens, _temperature) {
|
|
132
135
|
return this.run(model, prompt, []);
|
|
@@ -148,7 +151,8 @@ export class PiCliChatClient {
|
|
|
148
151
|
});
|
|
149
152
|
const child = spawn(this.executable, [
|
|
150
153
|
"--provider", route.provider, "--model", route.modelId, "--mode", "json", "--offline",
|
|
151
|
-
"--no-extensions",
|
|
154
|
+
"--no-extensions", ...piProviderExtensionArgs(this.providerExtensions),
|
|
155
|
+
"--no-tools", "--print", ...imageArgs, prompt,
|
|
152
156
|
], { cwd: dir, stdio: ["ignore", "pipe", "pipe"] });
|
|
153
157
|
let stdout = "";
|
|
154
158
|
let stderr = "";
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/** Safe, explicit Pi subprocess configuration. */
|
|
2
|
+
import { existsSync, realpathSync, statSync } from "node:fs";
|
|
3
|
+
import { basename, isAbsolute } from "node:path";
|
|
4
|
+
/**
|
|
5
|
+
* Provider extensions execute as code. Require exact, existing, absolute files
|
|
6
|
+
* and refuse OpenMerit itself so an isolated trial cannot recursively launch
|
|
7
|
+
* another merit loop.
|
|
8
|
+
*/
|
|
9
|
+
export function validatePiProviderExtensions(paths) {
|
|
10
|
+
const valid = [];
|
|
11
|
+
const errors = [];
|
|
12
|
+
for (const configured of [...new Set(paths)]) {
|
|
13
|
+
if (!isAbsolute(configured)) {
|
|
14
|
+
errors.push(`${configured}: path is not absolute`);
|
|
15
|
+
continue;
|
|
16
|
+
}
|
|
17
|
+
if (!existsSync(configured)) {
|
|
18
|
+
errors.push(`${configured}: file does not exist`);
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
let actual = configured;
|
|
22
|
+
try {
|
|
23
|
+
actual = realpathSync(configured);
|
|
24
|
+
if (!statSync(actual).isFile()) {
|
|
25
|
+
errors.push(`${configured}: path is not a file`);
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
errors.push(`${configured}: ${error.message}`);
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
if (/^openmerit\.[cm]?[jt]s$/i.test(basename(actual))) {
|
|
34
|
+
errors.push(`${configured}: refusing to load OpenMerit recursively`);
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
valid.push(actual);
|
|
38
|
+
}
|
|
39
|
+
return { valid, errors };
|
|
40
|
+
}
|
|
41
|
+
export function piProviderExtensionArgs(paths) {
|
|
42
|
+
const checked = validatePiProviderExtensions(paths);
|
|
43
|
+
if (checked.errors.length)
|
|
44
|
+
throw new Error(`invalid pi.provider_extensions: ${checked.errors.join("; ")}`);
|
|
45
|
+
return checked.valid.flatMap((file) => ["--extension", file]);
|
|
46
|
+
}
|
package/dist/pi-trials.js
CHANGED
|
@@ -10,6 +10,7 @@ import { paths, readJson } from "./store.js";
|
|
|
10
10
|
import { sessionFiles, sessionImages, taskInputKey } from "./task-input.js";
|
|
11
11
|
import { legacyOpenRouterRoute, meritModelId, modelRoute, routeKey } from "./routes.js";
|
|
12
12
|
import { directChatClient } from "./llm.js";
|
|
13
|
+
import { piProviderExtensionArgs } from "./pi-config.js";
|
|
13
14
|
function answerText(content) {
|
|
14
15
|
if (typeof content === "string")
|
|
15
16
|
return content;
|
|
@@ -20,8 +21,12 @@ function answerText(content) {
|
|
|
20
21
|
}
|
|
21
22
|
/** Pi-backed harness adapter. Future harnesses can implement the same seam. */
|
|
22
23
|
export class PiHarnessAdapter {
|
|
24
|
+
providerExtensions;
|
|
25
|
+
constructor(providerExtensions = []) {
|
|
26
|
+
this.providerExtensions = providerExtensions;
|
|
27
|
+
}
|
|
23
28
|
runTask(model, input) {
|
|
24
|
-
return executePiTask(model, input.task, input.cwd, input.images, input.files);
|
|
29
|
+
return executePiTask(model, input.task, input.cwd, input.images, input.files, this.providerExtensions);
|
|
25
30
|
}
|
|
26
31
|
}
|
|
27
32
|
const defaultHarness = new PiHarnessAdapter();
|
|
@@ -113,7 +118,9 @@ export function recordedActiveTask(task, model, images = [], sessionFile, sessio
|
|
|
113
118
|
if (!user || !userImages || taskInputKey(answerText(user.content), userImages, sessionFiles(answerText(user.content))) !==
|
|
114
119
|
taskInputKey(task, images, files))
|
|
115
120
|
return null;
|
|
116
|
-
const matching = assistants.filter((m) =>
|
|
121
|
+
const matching = assistants.filter((m) => typeof model === "string"
|
|
122
|
+
? meritModelId(m.provider ?? "openrouter", m.model ?? "") === model
|
|
123
|
+
: m.provider === model.provider && m.model === model.modelId);
|
|
117
124
|
if (matching.length === 0 || matching.some((m) => m.stopReason === "error"))
|
|
118
125
|
return null;
|
|
119
126
|
const answer = matching.map((m) => answerText(m.content)).filter(Boolean).at(-1) ?? "";
|
|
@@ -170,10 +177,10 @@ export function settledActiveTask(snapshot) {
|
|
|
170
177
|
const files = sessionFiles(task);
|
|
171
178
|
if (!task.trim() || taskInputKey(task, images, files) !== st.settledTaskKey)
|
|
172
179
|
return null;
|
|
173
|
-
const
|
|
180
|
+
const route = st.currentRoute ?? legacyOpenRouterRoute(st.currentModel);
|
|
181
|
+
const run = recordedActiveTask(task, route, images, st.sessionFile, st.sessionBytes);
|
|
174
182
|
if (!run)
|
|
175
183
|
return null;
|
|
176
|
-
const route = st.currentRoute ?? legacyOpenRouterRoute(st.currentModel);
|
|
177
184
|
const routes = st.routes?.length ? st.routes : [route];
|
|
178
185
|
return { task, images, model: st.currentModel, run, sessionFile: st.sessionFile,
|
|
179
186
|
settledAt: st.settledAt, sessionBytes: st.sessionBytes, cwd: st.cwd ?? process.cwd(), usedTools, files,
|
|
@@ -182,19 +189,31 @@ export function settledActiveTask(snapshot) {
|
|
|
182
189
|
export function piExecutable() {
|
|
183
190
|
return process.env.OPENMERIT_PI_BIN?.trim() || "pi";
|
|
184
191
|
}
|
|
192
|
+
function compactNumber(value) {
|
|
193
|
+
const match = value.match(/^(\d+(?:\.\d+)?)([KMG])?$/i);
|
|
194
|
+
if (!match)
|
|
195
|
+
return undefined;
|
|
196
|
+
const scale = { K: 1_000, M: 1_000_000, G: 1_000_000_000 }[(match[2] ?? "").toUpperCase()] ?? 1;
|
|
197
|
+
return Math.round(Number(match[1]) * scale);
|
|
198
|
+
}
|
|
185
199
|
/** Read provider/model routes from pi when no live extension snapshot is available. */
|
|
186
|
-
export function availablePiRoutes(snapshot) {
|
|
200
|
+
export function availablePiRoutes(snapshot, providerExtensions = []) {
|
|
187
201
|
if (snapshot?.length)
|
|
188
202
|
return snapshot;
|
|
189
|
-
const result = spawnSync(piExecutable(), ["--offline", "--
|
|
203
|
+
const result = spawnSync(piExecutable(), ["--offline", "--no-extensions",
|
|
204
|
+
...piProviderExtensionArgs(providerExtensions), "--list-models"], { encoding: "utf8" });
|
|
190
205
|
if (result.error || result.status !== 0)
|
|
191
206
|
throw new Error("could not list models from pi");
|
|
192
207
|
const routes = [];
|
|
193
208
|
for (const line of result.stdout.split("\n").slice(1)) {
|
|
194
|
-
const [provider, id, , , , images] = line.trim().split(/\s+/);
|
|
209
|
+
const [provider, id, context, maxOut, , images] = line.trim().split(/\s+/);
|
|
195
210
|
if (!provider || !id || id.startsWith("~"))
|
|
196
211
|
continue;
|
|
197
|
-
routes.push(modelRoute(provider, id, {
|
|
212
|
+
routes.push(modelRoute(provider, id, {
|
|
213
|
+
input: images === "yes" ? ["text", "image"] : ["text"],
|
|
214
|
+
contextWindow: compactNumber(context),
|
|
215
|
+
maxTokens: compactNumber(maxOut),
|
|
216
|
+
}));
|
|
198
217
|
}
|
|
199
218
|
return routes;
|
|
200
219
|
}
|
|
@@ -203,7 +222,7 @@ export function availablePiModels() {
|
|
|
203
222
|
return new Set(availablePiRoutes().map((route) => route.meritId));
|
|
204
223
|
}
|
|
205
224
|
/** Each invocation is a fresh pi run, so model B does not inherit model A's answer. */
|
|
206
|
-
export async function executePiTask(model, task, cwd = process.cwd(), images = [], files = []) {
|
|
225
|
+
export async function executePiTask(model, task, cwd = process.cwd(), images = [], files = [], providerExtensions = []) {
|
|
207
226
|
const route = typeof model === "string" ? legacyOpenRouterRoute(model) : model;
|
|
208
227
|
const imageDir = images.length ? mkdtempSync(join(tmpdir(), "openmerit-pi-image-")) : null;
|
|
209
228
|
const suffix = {
|
|
@@ -234,7 +253,8 @@ export async function executePiTask(model, task, cwd = process.cwd(), images = [
|
|
|
234
253
|
const traceLines = [];
|
|
235
254
|
const child = spawn(piExecutable(), [
|
|
236
255
|
"--provider", route.provider, "--model", route.modelId, "--mode", "json",
|
|
237
|
-
"--offline", "--no-extensions",
|
|
256
|
+
"--offline", "--no-extensions", ...piProviderExtensionArgs(providerExtensions),
|
|
257
|
+
"--approve", ...piTrialToolArgs(),
|
|
238
258
|
"--print", ...imageArgs, ...fileArgs, candidateTask,
|
|
239
259
|
], { cwd: workspace, stdio: ["ignore", "pipe", "pipe"] });
|
|
240
260
|
let buffer = "";
|
package/dist/policy.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/** Policy loading, validation, and the swap gate. */
|
|
2
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
3
3
|
export const DEFAULT_POLICY = {
|
|
4
|
-
version:
|
|
4
|
+
version: 2,
|
|
5
5
|
mode: "recommend",
|
|
6
6
|
auto_apply: {
|
|
7
7
|
enabled: false,
|
|
@@ -16,10 +16,78 @@ export const DEFAULT_POLICY = {
|
|
|
16
16
|
judge_model: null,
|
|
17
17
|
strategist_model: null,
|
|
18
18
|
max_usd_per_m: 20.0,
|
|
19
|
+
route_overrides: {},
|
|
20
|
+
pi: { provider_extensions: [] },
|
|
19
21
|
};
|
|
20
22
|
function num(v, fallback) {
|
|
21
23
|
return typeof v === "number" && Number.isFinite(v) ? v : fallback;
|
|
22
24
|
}
|
|
25
|
+
function finiteAtLeast(value, min, label) {
|
|
26
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < min)
|
|
27
|
+
throw new Error(`${label} must be a finite number >= ${min}`);
|
|
28
|
+
return value;
|
|
29
|
+
}
|
|
30
|
+
function positiveInteger(value, label) {
|
|
31
|
+
const result = finiteAtLeast(value, 1, label);
|
|
32
|
+
if (!Number.isInteger(result))
|
|
33
|
+
throw new Error(`${label} must be an integer`);
|
|
34
|
+
return result;
|
|
35
|
+
}
|
|
36
|
+
function routeOverrides(value) {
|
|
37
|
+
if (value === undefined)
|
|
38
|
+
return {};
|
|
39
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
40
|
+
throw new Error("route_overrides must be an object keyed by provider:modelId");
|
|
41
|
+
const result = {};
|
|
42
|
+
for (const [key, raw] of Object.entries(value)) {
|
|
43
|
+
const separator = key.indexOf(":");
|
|
44
|
+
if (separator <= 0 || separator === key.length - 1)
|
|
45
|
+
throw new Error(`route override ${key} must use provider:modelId`);
|
|
46
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
47
|
+
throw new Error(`route override ${key} must be an object`);
|
|
48
|
+
const input = raw.input;
|
|
49
|
+
if (input !== undefined && (!Array.isArray(input) || input.length === 0 ||
|
|
50
|
+
input.some((item) => item !== "text" && item !== "image")))
|
|
51
|
+
throw new Error(`route override ${key}.input must contain only text or image`);
|
|
52
|
+
const cost = raw.cost;
|
|
53
|
+
let normalizedCost;
|
|
54
|
+
if (cost !== undefined) {
|
|
55
|
+
if (!cost || typeof cost !== "object" || Array.isArray(cost))
|
|
56
|
+
throw new Error(`route override ${key}.cost must be an object`);
|
|
57
|
+
const c = cost;
|
|
58
|
+
normalizedCost = {
|
|
59
|
+
input: finiteAtLeast(c.input, 0, `route override ${key}.cost.input`),
|
|
60
|
+
output: finiteAtLeast(c.output, 0, `route override ${key}.cost.output`),
|
|
61
|
+
...(c.cacheRead === undefined ? {} : {
|
|
62
|
+
cacheRead: finiteAtLeast(c.cacheRead, 0, `route override ${key}.cost.cacheRead`),
|
|
63
|
+
}),
|
|
64
|
+
...(c.cacheWrite === undefined ? {} : {
|
|
65
|
+
cacheWrite: finiteAtLeast(c.cacheWrite, 0, `route override ${key}.cost.cacheWrite`),
|
|
66
|
+
}),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
const contextWindow = raw.context_window;
|
|
70
|
+
const maxTokens = raw.max_tokens;
|
|
71
|
+
result[key] = {
|
|
72
|
+
...(normalizedCost ? { cost: normalizedCost } : {}),
|
|
73
|
+
...(contextWindow === undefined ? {} : {
|
|
74
|
+
context_window: positiveInteger(contextWindow, `route override ${key}.context_window`),
|
|
75
|
+
}),
|
|
76
|
+
...(maxTokens === undefined ? {} : {
|
|
77
|
+
max_tokens: positiveInteger(maxTokens, `route override ${key}.max_tokens`),
|
|
78
|
+
}),
|
|
79
|
+
...(input === undefined ? {} : { input: [...new Set(input)] }),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
84
|
+
function providerExtensions(value) {
|
|
85
|
+
if (value === undefined)
|
|
86
|
+
return [];
|
|
87
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || !item.trim()))
|
|
88
|
+
throw new Error("pi.provider_extensions must be an array of non-empty file paths");
|
|
89
|
+
return [...new Set(value.map((item) => String(item).trim()))];
|
|
90
|
+
}
|
|
23
91
|
/** Load a policy file, filling defaults for missing keys. Throws on invalid JSON. */
|
|
24
92
|
export function loadPolicy(path) {
|
|
25
93
|
if (!existsSync(path))
|
|
@@ -58,6 +126,8 @@ export function loadPolicy(path) {
|
|
|
58
126
|
judge_model: raw.judge_model ?? null,
|
|
59
127
|
strategist_model: raw.strategist_model ?? null,
|
|
60
128
|
max_usd_per_m: num(raw.max_usd_per_m, d.max_usd_per_m),
|
|
129
|
+
route_overrides: routeOverrides(raw.route_overrides),
|
|
130
|
+
pi: { provider_extensions: providerExtensions(raw.pi?.provider_extensions) },
|
|
61
131
|
};
|
|
62
132
|
}
|
|
63
133
|
/** Match either a logical vendor/model id or its concrete route provider. */
|
package/dist/routes.js
CHANGED
|
@@ -19,6 +19,21 @@ export function legacyOpenRouterRoute(meritId) {
|
|
|
19
19
|
export function routeKey(route) {
|
|
20
20
|
return `${route.provider}:${route.modelId}`;
|
|
21
21
|
}
|
|
22
|
+
/** Apply user metadata only to the exact provider route it names. */
|
|
23
|
+
export function applyRouteOverrides(routes, overrides) {
|
|
24
|
+
return routes.map((route) => {
|
|
25
|
+
const override = overrides[routeKey(route)];
|
|
26
|
+
if (!override)
|
|
27
|
+
return route;
|
|
28
|
+
return {
|
|
29
|
+
...route,
|
|
30
|
+
...(override.cost ? { cost: { ...override.cost } } : {}),
|
|
31
|
+
...(override.context_window === undefined ? {} : { contextWindow: override.context_window }),
|
|
32
|
+
...(override.max_tokens === undefined ? {} : { maxTokens: override.max_tokens }),
|
|
33
|
+
...(override.input ? { input: [...override.input] } : {}),
|
|
34
|
+
};
|
|
35
|
+
});
|
|
36
|
+
}
|
|
22
37
|
export function pointKey(point) {
|
|
23
38
|
return point.route ? routeKey(point.route) : `openrouter:${point.model}`;
|
|
24
39
|
}
|