openmerit 0.1.4 → 0.1.6-preview.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/CHANGELOG.md +40 -0
  2. package/README.md +121 -386
  3. package/dist/core/src/index.d.ts +101 -0
  4. package/dist/core/src/index.js +1649 -0
  5. package/dist/core/src/store.d.ts +35 -0
  6. package/dist/core/src/store.js +102 -0
  7. package/dist/pi/src/index.d.ts +32 -0
  8. package/dist/pi/src/index.js +794 -0
  9. package/dist/pi/src/scheduler.d.ts +11 -0
  10. package/dist/pi/src/scheduler.js +137 -0
  11. package/dist/pi/src/wakeup.d.ts +2 -0
  12. package/dist/pi/src/wakeup.js +108 -0
  13. package/dist/protocol/src/index.d.ts +484 -0
  14. package/dist/protocol/src/index.js +47 -0
  15. package/dist/protocol/src/schemas.d.ts +576 -0
  16. package/dist/protocol/src/schemas.js +280 -0
  17. package/dist/terminal/public/app.js +297 -0
  18. package/dist/terminal/public/brands/anthropic.png +0 -0
  19. package/dist/terminal/public/brands/baai.png +0 -0
  20. package/dist/terminal/public/brands/baseten.png +0 -0
  21. package/dist/terminal/public/brands/cerebras.png +0 -0
  22. package/dist/terminal/public/brands/cohere.png +0 -0
  23. package/dist/terminal/public/brands/deepseek.ico +0 -0
  24. package/dist/terminal/public/brands/google.png +0 -0
  25. package/dist/terminal/public/brands/groq.ico +0 -0
  26. package/dist/terminal/public/brands/lm-studio.png +0 -0
  27. package/dist/terminal/public/brands/meta.ico +0 -0
  28. package/dist/terminal/public/brands/mistral.png +0 -0
  29. package/dist/terminal/public/brands/nomic.png +0 -0
  30. package/dist/terminal/public/brands/ollama.png +0 -0
  31. package/dist/terminal/public/brands/openai.png +0 -0
  32. package/dist/terminal/public/brands/openrouter.png +0 -0
  33. package/dist/terminal/public/brands/qwen.png +0 -0
  34. package/dist/terminal/public/brands/vllm.ico +0 -0
  35. package/dist/terminal/public/brands/vllm.png +0 -0
  36. package/dist/terminal/public/favicon.svg +1 -0
  37. package/dist/terminal/public/flow.css +1 -0
  38. package/dist/terminal/public/flow.js +770 -0
  39. package/dist/terminal/public/index.html +21 -0
  40. package/dist/terminal/public/styles.css +779 -0
  41. package/dist/terminal/src/activity-merge.mjs +64 -0
  42. package/dist/terminal/src/browser.mjs +29 -0
  43. package/dist/terminal/src/cli.mjs +60 -0
  44. package/dist/terminal/src/collect.mjs +311 -0
  45. package/dist/terminal/src/discovery.mjs +93 -0
  46. package/dist/terminal/src/hardware.mjs +57 -0
  47. package/dist/terminal/src/project-activity.mjs +156 -0
  48. package/dist/terminal/src/sample.mjs +171 -0
  49. package/dist/terminal/src/server.mjs +56 -0
  50. package/dist/terminal/src/services.mjs +62 -0
  51. package/dist/terminal/src/topology.mjs +30 -0
  52. package/docs/adapter-guide.md +189 -0
  53. package/docs/architecture.md +59 -0
  54. package/docs/automation.md +74 -0
  55. package/docs/budgets.md +37 -0
  56. package/docs/commands.md +85 -0
  57. package/docs/demo-backfill.md +29 -0
  58. package/docs/demo-fieldkit.md +47 -0
  59. package/docs/demo-placement.md +30 -0
  60. package/docs/demo-spam.md +15 -0
  61. package/docs/demo-support.md +42 -0
  62. package/docs/demo.md +57 -0
  63. package/docs/first-trial.md +60 -0
  64. package/docs/getting-started.md +65 -0
  65. package/docs/index.md +40 -0
  66. package/docs/inference-terminal.md +439 -0
  67. package/docs/lifecycle.md +30 -0
  68. package/docs/memo.md +126 -0
  69. package/docs/metrics-and-evidence.md +48 -0
  70. package/docs/operations.md +40 -0
  71. package/docs/pareto-spec.md +76 -0
  72. package/docs/pi-extension.md +54 -0
  73. package/docs/roadmap.md +28 -0
  74. package/docs/security.md +37 -0
  75. package/docs/site-artwork-linocut.md +23 -0
  76. package/docs/site-artwork-miniature-diverse.md +28 -0
  77. package/docs/site-artwork-miniature.md +26 -0
  78. package/docs/site-demo.md +177 -0
  79. package/docs/site-design.md +94 -0
  80. package/docs/site-documentation.md +83 -0
  81. package/docs/site-dynamic-og.md +35 -0
  82. package/docs/site-faq-maintenance.md +115 -0
  83. package/docs/site-hero-resolution.md +60 -0
  84. package/docs/site-illustration-sequences.md +227 -0
  85. package/docs/site-inference-terminal.md +203 -0
  86. package/docs/site-memo.md +39 -0
  87. package/docs/site-og-image.md +38 -0
  88. package/docs/site-og-workshop.md +21 -0
  89. package/docs/site-section-artwork.md +56 -0
  90. package/docs/site-skill-review.md +57 -0
  91. package/docs/site-terminal-preview.md +85 -0
  92. package/docs/testing.md +118 -0
  93. package/docs/troubleshooting.md +55 -0
  94. package/docs/ux-reference.md +32 -0
  95. package/package.json +74 -42
  96. package/benchmark/invoice_ocr/data/invoice_01_ground_truth.json +0 -38
  97. package/benchmark/invoice_ocr/data/invoice_01_row_2.jpg +0 -0
  98. package/benchmark/invoice_ocr/data/invoice_02_ground_truth.json +0 -32
  99. package/benchmark/invoice_ocr/data/invoice_02_row_5.jpg +0 -0
  100. package/benchmark/invoice_ocr/data/invoice_03_ground_truth.json +0 -26
  101. package/benchmark/invoice_ocr/data/invoice_03_row_6.jpg +0 -0
  102. package/benchmark/invoice_ocr/data/invoice_04_ground_truth.json +0 -26
  103. package/benchmark/invoice_ocr/data/invoice_04_row_7.jpg +0 -0
  104. package/benchmark/invoice_ocr/data/invoice_05_ground_truth.json +0 -38
  105. package/benchmark/invoice_ocr/data/invoice_05_row_947.jpg +0 -0
  106. package/benchmark/invoice_ocr/data/invoice_06_ground_truth.json +0 -38
  107. package/benchmark/invoice_ocr/data/invoice_06_row_948.jpg +0 -0
  108. package/benchmark/invoice_ocr/data/invoice_07_ground_truth.json +0 -20
  109. package/benchmark/invoice_ocr/data/invoice_07_row_949.jpg +0 -0
  110. package/benchmark/invoice_ocr/data/invoice_08_ground_truth.json +0 -38
  111. package/benchmark/invoice_ocr/data/invoice_08_row_1888.jpg +0 -0
  112. package/benchmark/invoice_ocr/data/invoice_09_ground_truth.json +0 -26
  113. package/benchmark/invoice_ocr/data/invoice_09_row_1890.jpg +0 -0
  114. package/benchmark/invoice_ocr/data/invoice_10_ground_truth.json +0 -20
  115. package/benchmark/invoice_ocr/data/invoice_10_row_1892.jpg +0 -0
  116. package/benchmark/invoice_ocr/data/manifest.json +0 -97
  117. package/dist/benchmarks.js +0 -98
  118. package/dist/catalog.js +0 -61
  119. package/dist/cli.js +0 -188
  120. package/dist/daemon.js +0 -407
  121. package/dist/diagnostics.js +0 -227
  122. package/dist/frontier.js +0 -56
  123. package/dist/harness.js +0 -1
  124. package/dist/integrations.js +0 -19
  125. package/dist/invoice-eval.js +0 -33
  126. package/dist/invoice-score.js +0 -124
  127. package/dist/judge.js +0 -43
  128. package/dist/llm.js +0 -207
  129. package/dist/pi-config.js +0 -46
  130. package/dist/pi-trials.js +0 -373
  131. package/dist/policy.js +0 -185
  132. package/dist/providers.js +0 -1
  133. package/dist/recommend.js +0 -76
  134. package/dist/routes.js +0 -74
  135. package/dist/standalone.js +0 -224
  136. package/dist/store.js +0 -89
  137. package/dist/strategist.js +0 -68
  138. package/dist/task-input.js +0 -54
  139. package/dist/traces.js +0 -127
  140. package/dist/trials.js +0 -140
  141. package/dist/types.js +0 -2
  142. package/examples/invoice-prompt.txt +0 -19
  143. package/examples/task.example.json +0 -7
  144. package/extension/openmerit.ts +0 -947
  145. package/instructions/OPENMERIT.md +0 -63
  146. package/instructions/openmerit.policy.json +0 -37
  147. package/rules.md +0 -43
package/dist/frontier.js DELETED
@@ -1,56 +0,0 @@
1
- /** Pareto frontier over (score up, price down, latency down) + best-fit/fallback picks. */
2
- import { pointKey } from "./routes.js";
3
- function effectivePrice(point) {
4
- return point.priceKnown === false ? Number.POSITIVE_INFINITY : point.price;
5
- }
6
- /** a dominates b if a is no worse on every objective and strictly better on one. */
7
- export function dominates(a, b) {
8
- const aLat = a.latencyMs ?? Number.POSITIVE_INFINITY;
9
- const bLat = b.latencyMs ?? Number.POSITIVE_INFINITY;
10
- const aPrice = effectivePrice(a);
11
- const bPrice = effectivePrice(b);
12
- const noWorse = a.score >= b.score && aPrice <= bPrice && aLat <= bLat;
13
- const better = a.score > b.score || aPrice < bPrice || aLat < bLat;
14
- return noWorse && better;
15
- }
16
- /**
17
- * Non-dominated set, cheapest first. When no point carries latency this
18
- * reduces to the classic "strictly increasing score as price rises" chain
19
- * from the original model-search prototype.
20
- */
21
- export function paretoFrontier(points) {
22
- const fr = points.filter((p) => !points.some((q) => q !== p && dominates(q, p)));
23
- return fr.sort((a, b) => effectivePrice(a) - effectivePrice(b) || b.score - a.score);
24
- }
25
- /** Simple 2-objective chain (score vs price only), cheapest first. */
26
- export function scorePriceChain(points) {
27
- const pts = [...points].sort((a, b) => effectivePrice(a) - effectivePrice(b) || b.score - a.score);
28
- const chain = [];
29
- let best = -1;
30
- for (const p of pts) {
31
- if (p.score > best) {
32
- chain.push(p);
33
- best = p.score;
34
- }
35
- }
36
- return chain;
37
- }
38
- /** Highest score; ties broken by lower price then lower latency. */
39
- export function pickBest(points) {
40
- return [...points].sort((a, b) => b.score - a.score || effectivePrice(a) - effectivePrice(b) ||
41
- (a.latencyMs ?? 1e18) - (b.latencyMs ?? 1e18))[0];
42
- }
43
- /**
44
- * Fallback for `best`: prefer the cheapest frontier point that still clears
45
- * `minScore`; otherwise the next-best scoring point overall. Never returns
46
- * `best` itself.
47
- */
48
- export function pickFallback(points, best, minScore) {
49
- const others = points.filter((p) => pointKey(p) !== pointKey(best));
50
- if (others.length === 0)
51
- return undefined;
52
- const fr = paretoFrontier(others).filter((p) => p.score >= minScore);
53
- if (fr.length > 0)
54
- return fr[0]; // cheapest adequate frontier point
55
- return pickBest(others);
56
- }
package/dist/harness.js DELETED
@@ -1 +0,0 @@
1
- export {};
@@ -1,19 +0,0 @@
1
- /** Stable boundaries for trace inputs and telemetry outputs. */
2
- import { randomUUID } from "node:crypto";
3
- import { appendJsonl, paths } from "./store.js";
4
- import { ingestNewTraces } from "./traces.js";
5
- export class PiTraceObservationSource {
6
- sessionsDir;
7
- id = "pi-jsonl";
8
- constructor(sessionsDir) {
9
- this.sessionsDir = sessionsDir;
10
- }
11
- read() { return ingestNewTraces(this.sessionsDir); }
12
- }
13
- export class LocalJsonlEventSink {
14
- id = "local-jsonl";
15
- emit(event) { appendJsonl(paths.events(), event); }
16
- }
17
- export function meritEvent(type, payload, source = "openmerit") {
18
- return { schemaVersion: 1, id: randomUUID(), ts: new Date().toISOString(), source, type, payload };
19
- }
@@ -1,33 +0,0 @@
1
- /** Reuse the audited invoice benchmark scorer for its pinned image/prompt cases. */
2
- import { createHash } from "node:crypto";
3
- import { readFileSync } from "node:fs";
4
- import { dirname, join } from "node:path";
5
- import { fileURLToPath } from "node:url";
6
- import { INVOICE_BENCHMARK_PROMPT, parseInvoiceJson, projectVisibleInvoice, scoreInvoice, } from "./invoice-score.js";
7
- function root() { return join(dirname(fileURLToPath(import.meta.url)), ".."); }
8
- export async function knownInvoiceScore(task, images, answer) {
9
- if (images.length !== 1 || images[0].mimeType !== "image/jpeg")
10
- return null;
11
- const benchDir = join(root(), "benchmark", "invoice_ocr");
12
- const piPrompt = readFileSync(join(root(), "examples", "invoice-prompt.txt"), "utf8").trim();
13
- if (task.trim() !== INVOICE_BENCHMARK_PROMPT.trim() && task.trim() !== piPrompt)
14
- return null;
15
- const manifest = JSON.parse(readFileSync(join(benchDir, "data", "manifest.json"), "utf8"));
16
- const inputHash = createHash("sha256").update(Buffer.from(images[0].data, "base64")).digest("hex");
17
- const row = manifest.rows.find((r) => createHash("sha256")
18
- .update(readFileSync(join(benchDir, "data", r.image))).digest("hex") === inputHash);
19
- if (!row)
20
- return null;
21
- const expected = JSON.parse(readFileSync(join(benchDir, "data", row.ground_truth), "utf8"));
22
- let actual = null;
23
- try {
24
- actual = parseInvoiceJson(answer);
25
- }
26
- catch { /* invalid JSON scores zero */ }
27
- const result = scoreInvoice(projectVisibleInvoice(expected), projectVisibleInvoice(actual));
28
- return {
29
- score: result.exact_leaf_accuracy,
30
- why: `pinned ${row.invoice_id}: audited exact-leaf ${result.exact_leaf_accuracy.toFixed(3)}, ` +
31
- `exact document=${result.exact_document}`,
32
- };
33
- }
@@ -1,124 +0,0 @@
1
- /** Runtime-safe copy of the audited invoice benchmark's exact scoring rules. */
2
- export const INVOICE_BENCHMARK_PROMPT = `Extract the invoice into the required JSON schema using only the image.
3
-
4
- Rules:
5
- - Copy names, identifiers, and descriptions exactly as printed.
6
- - Return dates as YYYY-MM-DD, regardless of the printed date format.
7
- - Return currency as its three-letter ISO code.
8
- - Return monetary values and quantities as JSON numbers without currency symbols or grouping separators.
9
- - Preserve line-item order and include every printed line item exactly once.
10
- - Use null only when a scalar field is not present. Never infer a missing value.
11
- - document_total_net is the subtotal/net amount before tax; document_total_amount is the final amount including tax.
12
- - Return only the schema-conforming JSON object.`;
13
- export const EXCLUDED_INVOICE_QUALITY_FIELDS = new Set(["po_number"]);
14
- function isPlainObject(value) {
15
- return typeof value === "object" && value !== null && !Array.isArray(value);
16
- }
17
- /** Match the Python equality semantics used by the original audited scorer. */
18
- function pyEqual(a, b) {
19
- if (typeof a === "boolean" || typeof b === "boolean") {
20
- if (typeof a === "boolean" && typeof b === "boolean")
21
- return a === b;
22
- if (typeof a === "number" && typeof b === "boolean")
23
- return a === Number(b);
24
- if (typeof a === "boolean" && typeof b === "number")
25
- return Number(a) === b;
26
- return false;
27
- }
28
- if (typeof a === "number" && typeof b === "number")
29
- return a === b;
30
- if (typeof a === "string" && typeof b === "string")
31
- return a === b;
32
- if (a === null || b === null)
33
- return a === b;
34
- if (Array.isArray(a) && Array.isArray(b)) {
35
- return a.length === b.length && a.every((value, index) => pyEqual(value, b[index]));
36
- }
37
- if (isPlainObject(a) && isPlainObject(b)) {
38
- const aKeys = Object.keys(a);
39
- const bKeys = Object.keys(b);
40
- return aKeys.length === bKeys.length && aKeys.every((key) => key in b && pyEqual(a[key], b[key]));
41
- }
42
- return false;
43
- }
44
- function flatten(value, path = "") {
45
- if (isPlainObject(value)) {
46
- let flattened = {};
47
- for (const [key, child] of Object.entries(value)) {
48
- const childPath = path ? `${path}.${key}` : key;
49
- flattened = Object.assign(flattened, flatten(child, childPath));
50
- }
51
- return flattened;
52
- }
53
- if (Array.isArray(value)) {
54
- if (value.length === 0)
55
- return { [`${path}.__length__`]: 0 };
56
- let flattened = { [`${path}.__length__`]: value.length };
57
- for (let index = 0; index < value.length; index++) {
58
- flattened = Object.assign(flattened, flatten(value[index], `${path}[${index}]`));
59
- }
60
- return flattened;
61
- }
62
- return { [path]: value };
63
- }
64
- function exactValue(expected, actual) {
65
- if (typeof expected === "boolean" || typeof actual === "boolean") {
66
- return typeof expected === typeof actual && expected === actual;
67
- }
68
- if (typeof expected === "number" && typeof actual === "number")
69
- return expected === actual;
70
- return typeof expected === typeof actual && expected === actual;
71
- }
72
- export function scoreInvoice(expected, actual) {
73
- if (actual === null) {
74
- const expectedFlat = flatten(expected);
75
- return {
76
- exact_document: false,
77
- matched_leaves: 0,
78
- total_union_leaves: Object.keys(expectedFlat).length,
79
- exact_leaf_accuracy: 0,
80
- scalar_accuracy: 0,
81
- line_item_accuracy: 0,
82
- };
83
- }
84
- const expectedFlat = flatten(expected);
85
- const actualFlat = flatten(actual);
86
- const paths = new Set([...Object.keys(expectedFlat), ...Object.keys(actualFlat)]);
87
- let matches = 0;
88
- for (const path of paths) {
89
- if (path in expectedFlat && path in actualFlat && exactValue(expectedFlat[path], actualFlat[path])) {
90
- matches += 1;
91
- }
92
- }
93
- const scalarPaths = Object.keys(expectedFlat).filter((path) => !path.startsWith("line_items"));
94
- const itemPaths = [...paths].filter((path) => path.startsWith("line_items"));
95
- const scalarMatches = scalarPaths.filter((path) => path in actualFlat && exactValue(expectedFlat[path], actualFlat[path])).length;
96
- const itemMatches = itemPaths.filter((path) => path in expectedFlat && path in actualFlat && exactValue(expectedFlat[path], actualFlat[path])).length;
97
- return {
98
- exact_document: pyEqual(expected, actual),
99
- matched_leaves: matches,
100
- total_union_leaves: paths.size,
101
- exact_leaf_accuracy: paths.size ? matches / paths.size : 1,
102
- scalar_accuracy: scalarPaths.length ? scalarMatches / scalarPaths.length : 1,
103
- line_item_accuracy: itemPaths.length ? itemMatches / itemPaths.length : 1,
104
- };
105
- }
106
- export function parseInvoiceJson(content) {
107
- let stripped = content.trim();
108
- if (stripped.startsWith("```")) {
109
- const lines = stripped.split(/\r\n|\r|\n/);
110
- stripped = lines.slice(1, -1).join("\n");
111
- }
112
- const parsed = JSON.parse(stripped);
113
- if (!isPlainObject(parsed))
114
- throw new TypeError("Model output is not a JSON object");
115
- return parsed;
116
- }
117
- export function projectVisibleInvoice(document) {
118
- if (document === null || document === undefined)
119
- return document;
120
- const projected = structuredClone(document);
121
- for (const field of EXCLUDED_INVOICE_QUALITY_FIELDS)
122
- delete projected[field];
123
- return projected;
124
- }
package/dist/judge.js DELETED
@@ -1,43 +0,0 @@
1
- /** Judge: score a model's answer against a rubric. */
2
- import { directChatClient } from "./llm.js";
3
- export const JUDGE_PREFS = ["openai/gpt-4o-mini", "openai/gpt-4o"];
4
- const JUDGE_PROMPT = `You are grading a model's answer.
5
- TASK: {task}
6
- RUBRIC: {rubric}
7
- ANSWER: {answer}
8
- Return ONLY json: {{"score": <0..1>, "why": "<one line>"}}`;
9
- /** Extract the first {...} JSON object from model output. */
10
- export function parseObj(txt) {
11
- const m = txt.match(/\{.*\}/s) ?? txt.match(/\{[^}]*\}/s);
12
- if (!m)
13
- throw new Error("no json object found");
14
- return JSON.parse(m[0]);
15
- }
16
- export async function judge(key, judgeModel, task, rubric, answer, client = directChatClient(key)) {
17
- const { content } = await client.chat(judgeModel, JUDGE_PROMPT.replace("{task}", task)
18
- .replace("{rubric}", rubric)
19
- .replace("{answer}", answer.slice(0, 16000)), 1024, 0);
20
- try {
21
- const j = parseObj(content);
22
- return { score: Number(j.score ?? 0), why: String(j.why ?? "") };
23
- }
24
- catch {
25
- return { score: 0, why: "unparseable judge output" };
26
- }
27
- }
28
- /** Grade OCR answers against the actual uploaded pixels, not answer text alone. */
29
- export async function judgeWithImages(key, judgeModel, task, rubric, answer, images, client = directChatClient(key)) {
30
- const prompt = `You are grading an answer to a visual task. Inspect the attached image carefully.\n` +
31
- `TASK: ${task}\nRUBRIC: ${rubric}\nANSWER: ${answer.slice(0, 16000)}\n` +
32
- `For invoice extraction, check every visible field and line item against the image. ` +
33
- `Penalize missing, invented, or mistyped values and invalid JSON. ` +
34
- `Return ONLY json: {"score": <0..1>, "why": "<one line>"}`;
35
- const { content } = await client.chatWithImages(judgeModel, prompt, images, 1024);
36
- try {
37
- const j = parseObj(content);
38
- return { score: Number(j.score ?? 0), why: String(j.why ?? "") };
39
- }
40
- catch {
41
- return { score: 0, why: "unparseable visual judge output" };
42
- }
43
- }
package/dist/llm.js DELETED
@@ -1,207 +0,0 @@
1
- /** Minimal provider clients for judge and strategist requests. */
2
- import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3
- import { spawn } from "node:child_process";
4
- import { tmpdir } from "node:os";
5
- import { join } from "node:path";
6
- import { paths } from "./store.js";
7
- import { legacyOpenRouterRoute } from "./routes.js";
8
- import { piProviderExtensionArgs } from "./pi-config.js";
9
- const OR = "https://openrouter.ai/api/v1";
10
- /** Load OPENROUTER_API_KEY from env or ~/.openmerit/.env. */
11
- export function loadKey() {
12
- const envFile = paths.envFile();
13
- if (existsSync(envFile)) {
14
- for (const line of readFileSync(envFile, "utf8").split("\n")) {
15
- const t = line.trim();
16
- if (!t || t.startsWith("#") || !t.includes("="))
17
- continue;
18
- const [k, ...v] = t.split("=");
19
- if (process.env[k.trim()] === undefined)
20
- process.env[k.trim()] = v.join("=").trim();
21
- }
22
- }
23
- const key = process.env.OPENROUTER_API_KEY;
24
- if (!key)
25
- throw new Error("missing OPENROUTER_API_KEY (env or ~/.openmerit/.env)");
26
- return key;
27
- }
28
- async function request(key, path, body, baseUrl = OR, tries = 3) {
29
- for (let attempt = 0; attempt < tries; attempt++) {
30
- const res = await fetch(baseUrl + path, {
31
- method: body === undefined ? "GET" : "POST",
32
- headers: {
33
- Authorization: `Bearer ${key}`,
34
- ...(body === undefined ? {} : { "Content-Type": "application/json", "X-Title": "openmerit" }),
35
- },
36
- body: body === undefined ? undefined : JSON.stringify(body),
37
- });
38
- if (res.ok)
39
- return (await res.json());
40
- const detail = (await res.text()).slice(0, 400);
41
- if ([429, 500, 502, 503].includes(res.status) && attempt < tries - 1) {
42
- await new Promise((r) => setTimeout(r, 2000 * (attempt + 1)));
43
- continue;
44
- }
45
- throw new Error(`OpenRouter HTTP ${res.status}: ${detail}`);
46
- }
47
- throw new Error("unreachable");
48
- }
49
- class OpenAICompatibleProvider {
50
- id;
51
- baseUrl;
52
- key;
53
- constructor(id, baseUrl, key) {
54
- this.id = id;
55
- this.baseUrl = baseUrl;
56
- this.key = key;
57
- }
58
- model(model) { return this.id === "openrouter" ? model : model.split("/", 2).at(-1) ?? model; }
59
- async chat(model, prompt, maxTokens, temperature) {
60
- const r = await request(this.key, "/chat/completions", { model: this.model(model), messages: [{ role: "user", content: prompt }], max_tokens: maxTokens, temperature }, this.baseUrl);
61
- return { content: r.choices[0]?.message?.content ?? "", usage: r.usage ?? {} };
62
- }
63
- async chatWithImages(model, prompt, images, maxTokens) {
64
- const content = [{ type: "text", text: prompt }, ...images.map((image) => ({ type: "image_url", image_url: { url: `data:${image.mimeType};base64,${image.data}` } }))];
65
- const r = await request(this.key, "/chat/completions", { model: this.model(model), messages: [{ role: "user", content }], max_tokens: maxTokens, temperature: 0 }, this.baseUrl);
66
- return { content: r.choices[0]?.message?.content ?? "", usage: r.usage ?? {} };
67
- }
68
- }
69
- class GeminiProvider {
70
- key;
71
- id = "google";
72
- constructor(key) {
73
- this.key = key;
74
- }
75
- async call(model, prompt, images = [], maxTokens) {
76
- const name = model.split("/", 2).at(-1) ?? model;
77
- const parts = [{ text: prompt }, ...images.map((image) => ({ inline_data: { mime_type: image.mimeType, data: image.data } }))];
78
- const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(name)}:generateContent?key=${encodeURIComponent(this.key)}`, {
79
- method: "POST", headers: { "Content-Type": "application/json" },
80
- body: JSON.stringify({ contents: [{ role: "user", parts }], generationConfig: { maxOutputTokens: maxTokens, temperature: images.length ? 0 : undefined } }),
81
- });
82
- if (!res.ok)
83
- throw new Error(`Google Gemini HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
84
- const raw = await res.json();
85
- return { content: raw.candidates?.[0]?.content?.parts?.map((p) => p.text ?? "").join("") ?? "", usage: { prompt_tokens: raw.usageMetadata?.promptTokenCount, completion_tokens: raw.usageMetadata?.candidatesTokenCount, total_tokens: raw.usageMetadata?.totalTokenCount } };
86
- }
87
- chat(model, prompt, maxTokens) { return this.call(model, prompt, [], maxTokens); }
88
- chatWithImages(model, prompt, images, maxTokens) { return this.call(model, prompt, images, maxTokens); }
89
- }
90
- export function providerFor(model, fallbackKey) {
91
- const id = model.split("/", 1)[0] || "openrouter";
92
- // OpenRouter model IDs are vendor/model (for example openai/gpt-4o-mini).
93
- // Direct providers require an explicit opt-in to avoid routing those IDs
94
- // away from OpenRouter when users have multiple credentials configured.
95
- if (id === "openai" && process.env.OPENMERIT_DIRECT_PROVIDER === "openai" && process.env.OPENAI_API_KEY)
96
- return new OpenAICompatibleProvider(id, "https://api.openai.com/v1", process.env.OPENAI_API_KEY);
97
- if (id === "google" && process.env.OPENMERIT_DIRECT_PROVIDER === "google" && process.env.GEMINI_API_KEY)
98
- return new GeminiProvider(process.env.GEMINI_API_KEY);
99
- return new OpenAICompatibleProvider("openrouter", OR, fallbackKey);
100
- }
101
- export async function chat(key, model, prompt, maxTokens, temperature) {
102
- return providerFor(model, key).chat(model, prompt, maxTokens, temperature);
103
- }
104
- /** Send a visual grading request through the same OpenRouter account. */
105
- export async function chatWithImages(key, model, prompt, images, maxTokens) {
106
- return providerFor(model, key).chatWithImages(model, prompt, images, maxTokens);
107
- }
108
- export function directChatClient(key) {
109
- const id = (model) => typeof model === "string" ? model : model.meritId;
110
- return {
111
- chat: (model, prompt, maxTokens, temperature) => chat(key, id(model), prompt, maxTokens, temperature),
112
- chatWithImages: (model, prompt, images, maxTokens) => chatWithImages(key, id(model), prompt, images, maxTokens),
113
- };
114
- }
115
- function textContent(content) {
116
- if (typeof content === "string")
117
- return content;
118
- if (!Array.isArray(content))
119
- return "";
120
- return content.filter((part) => !!part && typeof part === "object" && part.type === "text")
121
- .map((part) => part.text ?? "").join("\n");
122
- }
123
- /** Use pi's model registry and credential store without exposing provider secrets to OpenMerit. */
124
- export class PiCliChatClient {
125
- executable;
126
- onSpend;
127
- providerExtensions;
128
- id = "pi-cli";
129
- constructor(executable = process.env.OPENMERIT_PI_BIN?.trim() || "pi", onSpend, providerExtensions = []) {
130
- this.executable = executable;
131
- this.onSpend = onSpend;
132
- this.providerExtensions = providerExtensions;
133
- }
134
- chat(model, prompt, _maxTokens, _temperature) {
135
- return this.run(model, prompt, []);
136
- }
137
- chatWithImages(model, prompt, images, _maxTokens) {
138
- return this.run(model, prompt, images);
139
- }
140
- async run(target, prompt, images) {
141
- const route = typeof target === "string" ? legacyOpenRouterRoute(target) : target;
142
- const dir = mkdtempSync(join(tmpdir(), "openmerit-pi-chat-"));
143
- try {
144
- const imageArgs = images.map((image, index) => {
145
- const suffix = { "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp", "image/gif": "gif" }[image.mimeType];
146
- if (!suffix)
147
- throw new Error(`unsupported pi image: ${image.mimeType}`);
148
- const file = join(dir, `image-${index}.${suffix}`);
149
- writeFileSync(file, Buffer.from(image.data, "base64"));
150
- return `@${file}`;
151
- });
152
- const child = spawn(this.executable, [
153
- "--provider", route.provider, "--model", route.modelId, "--mode", "json", "--offline",
154
- "--no-extensions", ...piProviderExtensionArgs(this.providerExtensions),
155
- "--no-tools", "--print", ...imageArgs, prompt,
156
- ], { cwd: dir, stdio: ["ignore", "pipe", "pipe"] });
157
- let stdout = "";
158
- let stderr = "";
159
- child.stdout.on("data", (chunk) => { stdout += chunk.toString("utf8"); });
160
- child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf8"); });
161
- const code = await new Promise((resolve, reject) => {
162
- child.on("error", reject);
163
- child.on("close", (value) => resolve(value ?? 1));
164
- });
165
- let answer = "";
166
- let promptTokens = 0;
167
- let completionTokens = 0;
168
- let totalTokens = 0;
169
- let costUsd = 0;
170
- let failed = false;
171
- for (const line of stdout.split("\n")) {
172
- if (!line.trim())
173
- continue;
174
- let event;
175
- try {
176
- event = JSON.parse(line);
177
- }
178
- catch {
179
- continue;
180
- }
181
- if (event.type !== "message_end" || event.message?.role !== "assistant")
182
- continue;
183
- answer = textContent(event.message.content) || answer;
184
- promptTokens += event.message.usage?.input ?? 0;
185
- completionTokens += event.message.usage?.output ?? 0;
186
- totalTokens += event.message.usage?.totalTokens ?? 0;
187
- costUsd += event.message.usage?.cost?.total ?? 0;
188
- if (event.message.stopReason === "error")
189
- failed = true;
190
- }
191
- if (code !== 0 || failed || !answer.trim()) {
192
- throw new Error(`pi model call ${route.provider}/${route.modelId} failed: ` +
193
- (stderr.trim().slice(0, 240) || `exit ${code}, empty answer`));
194
- }
195
- this.onSpend?.(costUsd);
196
- return { content: answer, usage: {
197
- prompt_tokens: promptTokens || undefined,
198
- completion_tokens: completionTokens || undefined,
199
- total_tokens: totalTokens || promptTokens + completionTokens || undefined,
200
- costUsd,
201
- } };
202
- }
203
- finally {
204
- rmSync(dir, { recursive: true, force: true });
205
- }
206
- }
207
- }
package/dist/pi-config.js DELETED
@@ -1,46 +0,0 @@
1
- /** Safe, explicit Pi subprocess configuration. */
2
- import { existsSync, realpathSync, statSync } from "node:fs";
3
- import { basename, isAbsolute } from "node:path";
4
- /**
5
- * Provider extensions execute as code. Require exact, existing, absolute files
6
- * and refuse OpenMerit itself so an isolated trial cannot recursively launch
7
- * another merit loop.
8
- */
9
- export function validatePiProviderExtensions(paths) {
10
- const valid = [];
11
- const errors = [];
12
- for (const configured of [...new Set(paths)]) {
13
- if (!isAbsolute(configured)) {
14
- errors.push(`${configured}: path is not absolute`);
15
- continue;
16
- }
17
- if (!existsSync(configured)) {
18
- errors.push(`${configured}: file does not exist`);
19
- continue;
20
- }
21
- let actual = configured;
22
- try {
23
- actual = realpathSync(configured);
24
- if (!statSync(actual).isFile()) {
25
- errors.push(`${configured}: path is not a file`);
26
- continue;
27
- }
28
- }
29
- catch (error) {
30
- errors.push(`${configured}: ${error.message}`);
31
- continue;
32
- }
33
- if (/^openmerit\.[cm]?[jt]s$/i.test(basename(actual))) {
34
- errors.push(`${configured}: refusing to load OpenMerit recursively`);
35
- continue;
36
- }
37
- valid.push(actual);
38
- }
39
- return { valid, errors };
40
- }
41
- export function piProviderExtensionArgs(paths) {
42
- const checked = validatePiProviderExtensions(paths);
43
- if (checked.errors.length)
44
- throw new Error(`invalid pi.provider_extensions: ${checked.errors.join("; ")}`);
45
- return checked.valid.flatMap((file) => ["--extension", file]);
46
- }