openmerit 0.3.10 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -22
- package/data/eci-estimates.csv +179 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +180 -0
- package/dist/connectors/langfuse/client.d.ts +42 -0
- package/dist/connectors/langfuse/client.js +141 -0
- package/dist/connectors/langfuse/index.d.ts +163 -0
- package/dist/connectors/langfuse/index.js +378 -0
- package/dist/connectors/langfuse/otlp.d.ts +60 -0
- package/dist/connectors/langfuse/otlp.js +40 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +11 -0
- package/dist/pi/extension.d.ts +13 -0
- package/dist/pi/extension.js +148 -0
- package/dist/pi/harness.d.ts +20 -0
- package/dist/pi/harness.js +49 -0
- package/dist/pi/instructions.d.ts +2 -0
- package/dist/pi/instructions.js +12 -0
- package/dist/scout/candidates.d.ts +62 -0
- package/dist/scout/candidates.js +90 -0
- package/dist/scout/eci.d.ts +42 -0
- package/dist/scout/eci.js +84 -0
- package/dist/scout/evaluators/module.d.ts +24 -0
- package/dist/scout/evaluators/module.js +19 -0
- package/dist/scout/evaluators/pi.d.ts +53 -0
- package/dist/scout/evaluators/pi.js +117 -0
- package/dist/scout/pipeline.d.ts +111 -0
- package/dist/scout/pipeline.js +157 -0
- package/dist/scout/report.d.ts +3 -0
- package/dist/scout/report.js +42 -0
- package/dist/scout/search.d.ts +110 -0
- package/dist/scout/search.js +205 -0
- package/dist/scout/service.d.ts +148 -0
- package/dist/scout/service.js +143 -0
- package/examples/health-declaration-codes/dataset.jsonl +14 -0
- package/examples/sahra-enrollment/evaluator.ts +151 -0
- package/package.json +34 -78
- package/CHANGELOG.md +0 -250
- package/dist/pi/scout-extension.mjs +0 -287
- package/dist/pi/src/index.d.ts +0 -6
- package/dist/pi/src/index.js +0 -56
- package/dist/terminal/scout-update.json +0 -1
- package/dist/terminal/src/browser.d.mts +0 -9
- package/dist/terminal/src/browser.mjs +0 -28
- package/dist/terminal/src/cli.mjs +0 -28
- package/dist/terminal/src/connections.js +0 -151
- package/dist/terminal/src/install.d.mts +0 -2
- package/dist/terminal/src/install.mjs +0 -524
- package/dist/terminal/src/scout-claude.js +0 -25
- package/dist/terminal/src/scout-credentials.js +0 -41
- package/dist/terminal/src/scout-instructions.js +0 -213
- package/dist/terminal/src/scout-pi.js +0 -84
- package/dist/terminal/src/scout.js +0 -229
- package/dist/terminal/src/telemetry.d.mts +0 -30
- package/dist/terminal/src/telemetry.mjs +0 -138
- package/dist/terminal/src/updates.js +0 -397
- package/docs/agent-evaluations.md +0 -15
- package/docs/commands.md +0 -9
- package/docs/getting-started.md +0 -37
- package/docs/index.md +0 -25
- package/docs/inference-terminal.md +0 -25
- package/docs/pi-extension.md +0 -58
- package/docs/roadmap.md +0 -16
- package/docs/scout-close-loop.md +0 -359
- package/docs/scout-data-policy.md +0 -87
- package/docs/scout.md +0 -190
- package/docs/security.md +0 -107
- package/docs/testing.md +0 -114
- package/docs/troubleshooting.md +0 -50
package/package.json
CHANGED
|
@@ -1,103 +1,59 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "openmerit",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.4.0",
|
|
4
|
+
"description": "Find the right model for your task: Scout reads your task or codebase, binary-searches benchmark-ranked models on your own labelled examples, and picks the winner from Langfuse evaluations.",
|
|
5
5
|
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
6
7
|
"bin": {
|
|
7
|
-
"openmerit": "./dist/
|
|
8
|
+
"openmerit": "./dist/cli.js"
|
|
8
9
|
},
|
|
9
|
-
"license": "MIT",
|
|
10
|
-
"keywords": [
|
|
11
|
-
"pi-package",
|
|
12
|
-
"pi-extension",
|
|
13
|
-
"model-routing",
|
|
14
|
-
"evaluation"
|
|
15
|
-
],
|
|
16
|
-
"files": [
|
|
17
|
-
"dist/pi/",
|
|
18
|
-
"dist/terminal/",
|
|
19
|
-
"!dist/terminal/src/*.ts",
|
|
20
|
-
"docs/",
|
|
21
|
-
"CHANGELOG.md",
|
|
22
|
-
"LICENSE",
|
|
23
|
-
"README.md"
|
|
24
|
-
],
|
|
25
10
|
"exports": {
|
|
11
|
+
".": {
|
|
12
|
+
"types": "./dist/index.d.ts",
|
|
13
|
+
"default": "./dist/index.js"
|
|
14
|
+
},
|
|
26
15
|
"./pi": {
|
|
27
|
-
"types": "./dist/pi/
|
|
28
|
-
"default": "./dist/pi/
|
|
16
|
+
"types": "./dist/pi/extension.d.ts",
|
|
17
|
+
"default": "./dist/pi/extension.js"
|
|
29
18
|
},
|
|
30
19
|
"./package.json": "./package.json"
|
|
31
20
|
},
|
|
32
21
|
"pi": {
|
|
33
22
|
"extensions": [
|
|
34
|
-
"./dist/pi/
|
|
23
|
+
"./dist/pi/extension.js"
|
|
35
24
|
]
|
|
36
25
|
},
|
|
26
|
+
"files": [
|
|
27
|
+
"dist/",
|
|
28
|
+
"!dist/**/*.test.*",
|
|
29
|
+
"data/",
|
|
30
|
+
"examples/",
|
|
31
|
+
"README.md",
|
|
32
|
+
"LICENSE"
|
|
33
|
+
],
|
|
37
34
|
"repository": {
|
|
38
35
|
"type": "git",
|
|
39
|
-
"url": "git+https://github.com/
|
|
36
|
+
"url": "git+https://github.com/openmerit-ai/openmerit-core.git",
|
|
37
|
+
"directory": "openmerit-core"
|
|
40
38
|
},
|
|
41
|
-
"homepage": "https://github.com/laz-aslam/openmerit#readme",
|
|
42
|
-
"bugs": {
|
|
43
|
-
"url": "https://github.com/laz-aslam/openmerit/issues"
|
|
44
|
-
},
|
|
45
|
-
"publishConfig": {
|
|
46
|
-
"access": "public"
|
|
47
|
-
},
|
|
48
|
-
"workspaces": [
|
|
49
|
-
"packages/*",
|
|
50
|
-
"scout/agent"
|
|
51
|
-
],
|
|
52
39
|
"engines": {
|
|
53
40
|
"node": ">=22.19.0"
|
|
54
41
|
},
|
|
55
42
|
"scripts": {
|
|
56
|
-
"
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"
|
|
60
|
-
"
|
|
61
|
-
"
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
"
|
|
65
|
-
"
|
|
66
|
-
"
|
|
67
|
-
"check:docs": "npm run build:site && node --test scripts/docs.test.ts",
|
|
68
|
-
"memo": "node demo/memo/preview.ts",
|
|
69
|
-
"check:demo": "npm ci --ignore-scripts --prefix demo && npm run build:site && tsc -p tsconfig.memo.json && node --test demo/test/*.test.mjs demo/test/*.test.ts",
|
|
70
|
-
"prepare": "npm run build",
|
|
71
|
-
"verify": "npm test && npm run check && npm run typecheck && npm run check:typescript && npm run build && npm run check:demo && npm run check:scout && npm run check:docs && npm run build:scout && git diff --check",
|
|
72
|
-
"prepublishOnly": "npm run verify",
|
|
73
|
-
"scout:refresh": "node scout/refresh.mjs",
|
|
74
|
-
"check:scout": "node --test scout/test/*.test.mjs scout/test/*.test.ts && node --check scout/worker.mjs",
|
|
75
|
-
"build:scout": "node scripts/build-scout.mjs"
|
|
76
|
-
},
|
|
77
|
-
"peerDependencies": {
|
|
78
|
-
"@earendil-works/pi-coding-agent": "1.5.0"
|
|
79
|
-
},
|
|
80
|
-
"peerDependenciesMeta": {
|
|
81
|
-
"@earendil-works/pi-coding-agent": {
|
|
82
|
-
"optional": true
|
|
83
|
-
}
|
|
43
|
+
"build": "rm -rf dist && tsc -p tsconfig.build.json && chmod +x dist/cli.js",
|
|
44
|
+
"typecheck": "tsc -p tsconfig.json",
|
|
45
|
+
"test": "node --test \"src/**/*.test.ts\" \"test/**/*.test.ts\"",
|
|
46
|
+
"check": "npm run typecheck && npm test",
|
|
47
|
+
"prepack": "npm run build",
|
|
48
|
+
"openmerit": "node src/cli.ts"
|
|
49
|
+
},
|
|
50
|
+
"dependencies": {
|
|
51
|
+
"@earendil-works/pi-ai": "1.5.0",
|
|
52
|
+
"@earendil-works/pi-coding-agent": "1.5.0",
|
|
53
|
+
"typebox": "1.3.27"
|
|
84
54
|
},
|
|
85
55
|
"devDependencies": {
|
|
86
|
-
"@
|
|
87
|
-
"
|
|
88
|
-
"@earendil-works/pi-coding-agent": "1.5.0",
|
|
89
|
-
"@lezer/javascript": "1.5.6",
|
|
90
|
-
"@resvg/resvg-js": "2.6.2",
|
|
91
|
-
"@types/node": "^22.18.6",
|
|
92
|
-
"esbuild": "0.28.2",
|
|
93
|
-
"fflate": "0.8.2",
|
|
94
|
-
"hyparquet": "1.31.5",
|
|
95
|
-
"markdown-it": "15.0.2",
|
|
96
|
-
"miniflare": "5.20260921.0-alpha",
|
|
97
|
-
"typebox": "1.3.27",
|
|
98
|
-
"typescript": "^5.9.3",
|
|
99
|
-
"web-vitals": "6.2.3",
|
|
100
|
-
"wrangler": "4.136.3",
|
|
101
|
-
"yaml": "2.9.1"
|
|
56
|
+
"@types/node": "22.19.1",
|
|
57
|
+
"typescript": "5.9.3"
|
|
102
58
|
}
|
|
103
59
|
}
|
package/CHANGELOG.md
DELETED
|
@@ -1,250 +0,0 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
3
|
-
## 0.3.10 — 2026-11-06 (release preparation)
|
|
4
|
-
|
|
5
|
-
- Refocus the Scout guide on supported setup, model selection, customer-owned evals and data handling. Preserve historical operator receipts in an internal archive excluded from npm, retain existing section links, remove repository-development instructions from the package README, and distinguish released Langfuse guidance from old preparation notes. This does not change model selection, credentials or permissions.
|
|
6
|
-
- Align the data policy's opening aggregate-results description with its existing per-user, cross-conversation connection contract. Hosted policy publication requires the separate Scout-serving deployment; npm publication alone does not update it.
|
|
7
|
-
|
|
8
|
-
## 0.3.9 — 2026-11-06 (release preparation)
|
|
9
|
-
|
|
10
|
-
- Fix Claude Code project setup, status and conflict detection in linked git worktrees. Claude Code 2.1.63 and newer share private project MCP configuration under the main checkout; older versions use the linked worktree root. The installer follows the Claude CLI on PATH, leaves the project's skill and rule in place, and does not migrate or delete existing MCP entries.
|
|
11
|
-
|
|
12
|
-
## 0.3.8 — 2026-11-06
|
|
13
|
-
|
|
14
|
-
- The Scout skill tells coding agents to refresh an already approved Langfuse connection on the first consultation and before a final model decision, without asking for new candidate-spend or customer-platform-write approval for that read. Old project notes are not a substitute for connector refresh; failed sync must be disclosed as stale evidence. The server still requires an explicit sync request, and candidate runs, platform writes and deletion still require approval.
|
|
15
|
-
|
|
16
|
-
## 0.3.7 — 2026-11-06
|
|
17
|
-
|
|
18
|
-
- The installed Scout skill distinguishes no eval setup, insufficient evals and suitable existing evals. Langfuse is the default customer-owned platform suggestion, not a requirement; local/custom evals can report aggregates. The customer's coding agent implements evals, while OpenMerit remains Viewer-only. Instructions require a local Markdown decision record before proposing explicitly approved cleanup of newly created comparison runs; existing records and reusable fixtures/definitions stay intact. This adds guidance, not an eval runner, automatic deletion or another platform connector. Fresh-agent acceptance remains a release gate.
|
|
19
|
-
|
|
20
|
-
## 0.3.6 — 2026-11-06
|
|
21
|
-
|
|
22
|
-
- Distinguish a connected Langfuse key from incomplete score-import setup. Connection status exposes only score-mapping and model-alias counts, and the web UI explains when either is missing. The CLI preserves those counts when the server provides them. Requires the corresponding Scout deployment and npm patch; does not change credentials, mappings or experiment attribution.
|
|
23
|
-
- Update an existing Langfuse connection's non-secret settings through CLI, MCP or API without retrieving or re-entering its Viewer key. Omit both Viewer credentials; Scout reuses and reverifies the encrypted key internally. New connections still require a protected Viewer pair, and partial pairs are rejected. Settings changes invalidate imported aggregates; sync again afterwards. Requires the matching Scout deployment.
|
|
24
|
-
|
|
25
|
-
## 0.3.5 — 2026-11-05
|
|
26
|
-
|
|
27
|
-
- Add optional headless Langfuse setup through the CLI, HTTP API and `scout_connection` MCP tool. A separately approved `scout:connections` key manages the authenticated user's connection and evaluation data; recommendation keys retain their existing permission. Viewer credentials enter through protected CLI input or trusted client configuration, never model-visible tool arguments. The web connection UI remains available, with revised spacing, mobile controls and the official Langfuse logo. Hosted features require a matching Scout deployment; saving Langfuse credentials also requires its dedicated encryption secret. Publishing this package does not deploy or enable the hosted connector.
|
|
28
|
-
|
|
29
|
-
## 0.3.4 — 2026-11-05
|
|
30
|
-
|
|
31
|
-
- `openmerit scout --results` refuses a file with more than 50 results before sending it, instead of failing with HTTP 413; split larger reports into files of up to 50.
|
|
32
|
-
- Claude Code's install success message names the MCP server, startup rule and skill instead of Pi's extension.
|
|
33
|
-
|
|
34
|
-
## 0.3.3 — 2026-11-05
|
|
35
|
-
|
|
36
|
-
- Fix `openmerit install --client claude-code` exiting with `ENOENT … dist/terminal/src/updates.ts` (0.3.1 and 0.3.2). Setting up automatic Scout instruction updates copied the updater from its TypeScript source, which the package does not ship; it now copies the module that is running. The MCP server, skill and rule were already written, but automatic updates stayed off and the installer reported failure.
|
|
37
|
-
- `openmerit scout --help` and the installed Scout instructions describe the new optional `kind` field for reported results (`quality`, `usd` or `seconds`), so measured cost replaces Scout's estimate, and tell the agent to report each agent's own checks at the agent level because end-to-end results also measure the agents it did not swap. Terminal output shows measured cost per 1,000 agent runs or tasks. The matching Scout serving release is deployed.
|
|
38
|
-
|
|
39
|
-
## 0.3.2 — 2026-11-05
|
|
40
|
-
|
|
41
|
-
- Fix Claude Code project-scope installation for a project inside a larger git repository, such as a monorepo package. Claude Code keeps its private MCP entry under the repository root, so the installer reported "Claude Code did not save the openmerit MCP server", `status` reported it missing and a rerun clashed with it. The installer now reads the entry where Claude Code keeps it and says that the MCP server applies to the whole repository; the skill and rule stay in the project.
|
|
42
|
-
- Make device approvals easier to relay. The installer prints the code's expiry in UTC with the minutes left instead of an unlabelled local time. When it returns with exit code 10, the message includes the approval link and expiry, `--json` adds `approvalUrl`, `userCode` and `expiresAt`, and the installed Scout instructions tell the agent to show the link and expiry to the person approving. The matching Scout serving release extends device codes from 10 to 30 minutes.
|
|
43
|
-
|
|
44
|
-
## 0.3.1 — 2026-11-05
|
|
45
|
-
|
|
46
|
-
- `openmerit scout --conversation <id> --results <file.json>` reports aggregate results you measured with your own evals. Scout then ranks them above public benchmarks for your recommendations only. Terminal output shows each task's evaluation plan, its agent and calls per run, and marks quality that rests on measured results.
|
|
47
|
-
- The installed Scout instructions for Claude Code and Pi tell the agent to measure shortlisted candidates as extra runs of the project's existing evals, never a second eval, and to report aggregates only. This needs the matching hosted Scout release, whose `scout_models` tool returns `tasks[].evaluation` and accepts `results`.
|
|
48
|
-
|
|
49
|
-
## 0.3.0 — 2026-11-04
|
|
50
|
-
|
|
51
|
-
- **Breaking:** retire the standalone `openmerit eval` CLI, including `init`, `doctor`, `plan`, `run`, `watch` and `report`. Existing local evaluation files are not deleted. The hosted `/scout/api/recommend` retirement ships separately with the Scout serving Worker.
|
|
52
|
-
- **Breaking:** retire the local Inference Terminal. `openmerit dash` and all of its options, including `--demo`, `--json` and `--diagnostics`, are removed, along with repository indexing, request-log and activity-export import, runtime and hardware discovery, the browser interface and its opt-in terminal analytics events. Existing `.openmerit/terminal.json`, `.openmerit/inference.jsonl` and application logs are not deleted. Scout setup, `openmerit scout` and `openmerit update` are unchanged.
|
|
53
|
-
- Drop the terminal-only runtime dependencies (`@iarna/toml`, `@lezer/python`) and browser bundle dependencies.
|
|
54
|
-
|
|
55
|
-
## 0.2.0 — 2026-11-04
|
|
56
|
-
|
|
57
|
-
- Add `openmerit scout`, a conversation with Scout from the terminal: clarifying questions answered with `--brief` or `--skip`, then up to five models per task, cheapest first among models that reach Scout's quality bar. When you ask for the fastest or newest model, the output names that order and shows each model's time per task or release date. It uses this device's Claude Code or Pi key and limits new conversations to the endpoint IDs in `.openmerit/scout.json`.
|
|
58
|
-
- The installed Scout instructions for Claude Code and Pi tell the agent to relay Scout's questions to the user and to read `tasks[].models`. This needs the matching hosted Scout release, whose `scout_models` MCP tool now runs the Scout agent.
|
|
59
|
-
- **Breaking:** remove the `openmerit`, `openmerit/core` and `openmerit/protocol` package exports, along with the evaluation coordinator, harness protocol, Pareto conformance rules and the repository's `npm run demo` sandbox. The package keeps `openmerit/pi` and the `openmerit` command: `openmerit dash`, `openmerit scout`, Scout setup commands and `openmerit eval`.
|
|
60
|
-
- Remove the hosted Inference Terminal sample at `dash.openmerit.ai`, its `/demo` before/after comparison and the opt-in `?map=flow` map experiment. The local terminal, including its fictional sample (`--demo`), is unchanged.
|
|
61
|
-
- Remove the retired demo guides and the guides that documented only the coordinator: architecture, adapter development, Pareto specification, first trial, metrics and evidence, comparisons and swaps, automatic checks, budgets and permissions, and project files and logs.
|
|
62
|
-
|
|
63
|
-
## 0.1.12 — 2026-11-03
|
|
64
|
-
|
|
65
|
-
- Replace the obsolete Pi 0.87 evaluation adapter with native Scout support for Pi 1.5.0. Share browser device approval, private client/scope/project credentials, installer lifecycle and Scout instructions with Claude Code; keep Pi's native MCP, tools, model, permissions and session ownership intact.
|
|
66
|
-
- Add targeted Pi setup, status, diagnostics, login, logout and removal, preserving edited files and independent credentials. Install a self-contained automatically discovered extension; updates require an explicit installer rerun and a new session. No legacy evaluation commands, scheduler, LaunchAgent or auto-update hooks remain in the Pi adapter.
|
|
67
|
-
- Add the logo-first Add to Pi action below Claude Code, with copy-only setup and a clipboard-denied fallback. Scout serving and npm publication remain separate releases.
|
|
68
|
-
- Keep the standalone evaluation CLI and core coordinator; move the prepared demo transport out of the removed Pi adapter and declare its Pi runtime dependency explicitly.
|
|
69
|
-
|
|
70
|
-
## 0.1.11 — 2026-11-01
|
|
71
|
-
|
|
72
|
-
- Refresh packaged Scout guides and include the Saga articles maintained in Markdown.
|
|
73
|
-
- Recognize the Claude Code instruction updater when invoked through a symbolic link, so linked hook entry points run their update checks.
|
|
74
|
-
- Preserve the existing CLI, Claude Code instruction-update contract, Pi integration and public exports. Scout, Saga and homepage source changes require separate website deployments; this npm release does not publish them.
|
|
75
|
-
|
|
76
|
-
## 0.1.10 — 2026-10-26
|
|
77
|
-
|
|
78
|
-
- Add default-on Claude Code background checks for compatible Scout skill/startup-rule updates, with daily checks, 15-minute failure backoff, non-blocking failure notices with the “update openmerit” fallback, manual `openmerit update`, persistent opt-out and a natural-language update instruction.
|
|
79
|
-
- Verify npm package integrity and read a bounded instruction bundle without executing downloaded code. Preserve edited files, credentials, MCP configuration, privacy settings and project policy; serialize maintenance and recover interrupted instruction writes.
|
|
80
|
-
- Disclose setup behavior, retain other Claude hooks/settings and require a new session to reliably load updated instructions. Existing installations need a one-time installer rerun after publication; incompatible releases require a reviewed manual upgrade. Hosted Scout and npm remain separate releases.
|
|
81
|
-
- Enable Scout setup usage/installation diagnostics by default, honoring telemetry, DNT and account-unlink opt-outs. Terminal/evaluation analytics remain opt-in; MCP prompt-content capture remains off.
|
|
82
|
-
|
|
83
|
-
## 0.1.9 — 2026-10-24
|
|
84
|
-
|
|
85
|
-
- Add opt-in coding-agent installation milestones, component/connection diagnostics and consent-gated setup/installation correlation retained across resumed approval and upgrades. Carry browser usage and unlink choices into setup; keep standalone CLI analytics default-off and MCP prompt-content capture off.
|
|
86
|
-
- Add bounded installation-event retries with stable delivery IDs. Report verification and remote revocation failures even when existing local command exit semantics remain successful.
|
|
87
|
-
- Coordinate the independently released Scout browser/device/MCP lifecycle telemetry, correct HTTP-200 logical-error accounting, and distinguish key authorization from verified installation. These are observed events, not skipped-call detection, guaranteed delivery or downstream model-execution tracking.
|
|
88
|
-
- Preserve existing package exports, model-decision guidance and legal metadata. Existing installations need an installer rerun; npm publication does not deploy Scout or the main website.
|
|
89
|
-
|
|
90
|
-
## 0.1.8 — 2026-10-24
|
|
91
|
-
|
|
92
|
-
- Install an owned startup rule requiring Scout before explicit or implicit application-model choices, including pretrained/local/browser models, weight downloads and SDK defaults. Keep detailed access-policy and search instructions in the skill. These instructions are guidance, not hard enforcement.
|
|
93
|
-
- Upgrade unchanged skill-only installations without another device approval; include rule ownership, conflict detection, interrupted-update recovery, status, diagnostics and safe removal.
|
|
94
|
-
- Require a new Claude Code session after setup or upgrade; reconnecting `/mcp` alone does not reload startup instructions. Pause model selection when Scout is unavailable or lacks suitable candidates rather than falling back to a remembered model.
|
|
95
|
-
|
|
96
|
-
## 0.1.7 — 2026-10-24
|
|
97
|
-
|
|
98
|
-
- Release coding-agent device sign-in, Claude Code MCP/Scout skill installation, status, diagnostics, removal and project endpoint policy on the stable `latest` channel. Setup requires browser approval; it never requests a pasted key.
|
|
99
|
-
- Use `openmerit@latest` in setup and installed skill commands. Scout's independently deployed button opens a prefilled Claude Desktop Code session; the user sends the prompt.
|
|
100
|
-
- Promote the agent-evaluation CLI, explicit scope/budget selection and corrected accounting from the preview channel. Provider billing, native permissions and unknown costs remain separate from local reservations.
|
|
101
|
-
- Include automatic light/dark terminal appearance and optional, default-off CLI/terminal analytics with bounded diagnostics. Do Not Track overrides opt-in; prompts, credentials, local names and raw errors are excluded.
|
|
102
|
-
- Preserve existing package exports and legal metadata. The hosted Scout service and website deploy independently; npm installation does not start hourly research or execute recommended models.
|
|
103
|
-
|
|
104
|
-
## 0.1.7-preview.5 — 2026-10-17
|
|
105
|
-
|
|
106
|
-
- Update packaged Scout documentation for the independently deployed matching repair: shared evidence targeting, retrieval coverage and bounded semantic consistency checks.
|
|
107
|
-
- Keep earlier failed acceptance and task-execution results visible; matching checks do not establish task accuracy or globally optimal recommendations.
|
|
108
|
-
- Preserve the package runtime, CLI commands, exports and legal metadata. Target the `preview` channel; stable `latest` remains unchanged.
|
|
109
|
-
|
|
110
|
-
## 0.1.7-preview.4 — 2026-10-17
|
|
111
|
-
|
|
112
|
-
- Update the packaged Scout guide for the independently deployed task-relevance beta, including metadata filters, existing ambiguity responses, measured-evidence preference and bounded matching.
|
|
113
|
-
- Document retained execution-test failures, incomplete coverage and provisional Pareto comparisons; no improvement in exact task accuracy or globally optimal membership is claimed.
|
|
114
|
-
- Preserve the package runtime, CLI commands, exports and legal metadata. Publish to `preview`; stable `latest` remains unchanged.
|
|
115
|
-
|
|
116
|
-
## 0.1.7-preview.3 — 2026-10-16
|
|
117
|
-
|
|
118
|
-
- Update packaged Scout API/MCP and agent-evaluation guides for the sole `/scout/` service and current request, inventory, data-handling and reliability contracts.
|
|
119
|
-
- Keep the hosted hourly research and matching service separate from npm installation. Hosted publication recovery, per-source freshness, bounded Decisions retries and reduced overlapping pool reads were deployed independently.
|
|
120
|
-
- Exclude internal website and deployment maintenance notes from the npm artifact. Keep existing commands, exports, legal metadata and stable `latest` behavior unchanged.
|
|
121
|
-
|
|
122
|
-
## Scout semantic benchmark interpretation — hosted beta, 2026-10-15
|
|
123
|
-
|
|
124
|
-
- Replace benchmark capability/name rules and family-name regexes with hourly Decisions interpretation of source definitions and methodology.
|
|
125
|
-
- Cache unchanged interpretations; leave unsupported or failed scopes unclassified and preserve evidence provenance.
|
|
126
|
-
- Keep prompt matching local to prepared benchmark metadata, with unchanged UI, prompt/specialist calls and hourly schedule. All 24 semantic benchmark cases, 107 local/live HTTP/MCP matching cases and 546 automated checks passed. npm publication remains deferred.
|
|
127
|
-
- Store compressed analytics payloads as gzip bytes to avoid base64 overhead on larger evidence records; retain legacy decoding and complete-record size limits.
|
|
128
|
-
|
|
129
|
-
## Scout semantic specialist matching — hosted beta, 2026-10-15
|
|
130
|
-
|
|
131
|
-
- Replace Hugging Face domain/language keyword dictionaries and partial label overlap with Decisions compatibility checks.
|
|
132
|
-
- Prepare and cache specialist profiles during the hourly refresh; batch only relevant compatibility checks when a prompt arrives.
|
|
133
|
-
- Require all applicable domain, language and fixed-label checks to pass. Preserve unknown metrics and ordinary results if specialist checking fails.
|
|
134
|
-
- Keep the UI unchanged. All 107 cases passed locally and through live HTTP/MCP; 526 automated tests passed. npm remains deferred.
|
|
135
|
-
|
|
136
|
-
## Scout semantic formats and SQL interpretation — hosted beta, 2026-10-15
|
|
137
|
-
|
|
138
|
-
- Remove prompt keyword scans from strict-format and structured-data interpretation; use the existing Decisions answers, including described record layouts and non-English requests.
|
|
139
|
-
- Recognize read-only SQL query writing and repair without requiring schema details in the task description. Keep query source distinct from executing it, its result format, prose explanation and data modification.
|
|
140
|
-
- Keep uncertain format answers visible and exclude them from narrow calibration matching. Preserve response validation, general consistency checks, confidence thresholds, one interpretation request, hourly collection and the UI.
|
|
141
|
-
- All 91 semantic cases passed locally and through live HTTP/MCP, including the previously failing SQL request and four fresh SQL holdouts. All 511 automated tests passed. npm publication remains deferred.
|
|
142
|
-
|
|
143
|
-
## Scout category and content interpretation — hosted beta, 2026-10-15
|
|
144
|
-
|
|
145
|
-
- Recognize conversationally supplied categories without requiring a formal label list; keep subtype and individual-identity discovery open-ended.
|
|
146
|
-
- Require document understanding for requested written/table content, including mixed outputs, without inferring it merely from a photographed object.
|
|
147
|
-
- Update two existing semantic questions with unchanged confidence thresholds, UI, hourly refresh and request count. Local and live HTTP/MCP acceptance passed 43 cases; an opt-in semantic audit preserves those cases for future interpretation changes. npm publication remains deferred.
|
|
148
|
-
|
|
149
|
-
## Scout task interpretation and ambiguous matching — hosted beta, 2026-10-15
|
|
150
|
-
|
|
151
|
-
- Frame every task description as quoted data for requirement analysis, preserving supplied-label and negation semantics without prompt-specific exceptions or lower confidence thresholds.
|
|
152
|
-
- Keep decision-plus-prose or extraction requests on generative endpoints, reject narrow calibration for uncertain operations/outputs, and ignore weak document-layout guesses for ordinary photos.
|
|
153
|
-
- All 25 routing cases passed locally and through live HTTP/MCP, including the prior binary-label failure. UI, hourly refresh and the single interpretation request are unchanged. npm publication remains deferred.
|
|
154
|
-
|
|
155
|
-
## Scout identification matching — hosted beta, 2026-10-15
|
|
156
|
-
|
|
157
|
-
- Distinguish open identification from supplied-label classification within the existing interpretation call. Open or uncertain identification uses evidence-ranked generative endpoints; fixed-choice APIs and label-dependent classifiers require supplied choices.
|
|
158
|
-
- Require relevant visual benchmark evidence for visual requests, preserve classification label scope in calibration matching, and use ordinary output-length estimates for generated names. UI and hourly refresh are unchanged; no npm release is included.
|
|
159
|
-
|
|
160
|
-
## Scout hosted beta — 2026-10-15
|
|
161
|
-
|
|
162
|
-
- Apply shared modality, output-format, tool and operation requirements across Scout recommendation paths. Catalogue refresh now retains explicit text-output contracts and excludes mixed media output from general text candidates.
|
|
163
|
-
- Preserve unknown and partial token prices as `null`; keep genuine advertised zero rates distinct.
|
|
164
|
-
- Suppress separate specialist discovery for endpoint-scoped requests, including empty inventories.
|
|
165
|
-
- Honor latency priority when choosing a model's endpoint, using only timing tied to that route. UI, hourly scheduling and interpretation calls are unchanged. These four service corrections were deployed separately from npm.
|
|
166
|
-
|
|
167
|
-
## 0.1.7-preview.2 — 2026-10-15
|
|
168
|
-
|
|
169
|
-
- Ask for a $10, custom or no-fixed-cap evaluation budget after model scope; save the explicit choice, handle reservations internally, and preserve cumulative usage across reruns and budget changes. Add `eval plan --budget <USD|unlimited>` for coding agents and noninteractive setup.
|
|
170
|
-
- Separate native budget and environment blocks from task scores in agent evaluations; probe the Codex Linux sandbox before making model requests.
|
|
171
|
-
- Recover partial Claude usage, retain partial cost estimates, keep missing native pricing unknown, and preserve budget reservations independently of estimated charges. Report version 2 separates these accounting rules from earlier comparisons.
|
|
172
|
-
|
|
173
|
-
## 0.1.7-preview.1 — 2026-10-15
|
|
174
|
-
|
|
175
|
-
- Add `openmerit eval init/doctor/plan/run/watch/report`: Scout shortlists, explicit model scope, Claude Code/Codex/OpenCode adapters, fresh task attempts, local graders, balanced adaptive 3–10 repetitions, revision comparisons and cumulative budget accounting.
|
|
176
|
-
- Bundle project agent instructions without install-time hooks. Missing usage/cost remains unknown and partial comparisons are disclosed. Terminal/Pi behavior is unchanged.
|
|
177
|
-
- Use the hosted Scout HTTP/MCP `modelIds` filter to select eligible candidates before ranking. This API change was deployed separately on 15 October 2026.
|
|
178
|
-
|
|
179
|
-
## 0.1.6-preview.5 — 2026-10-10
|
|
180
|
-
|
|
181
|
-
- Open the terminal before initial collection, then index repository declarations in a background worker with visible discovery, indexing, and linking progress. Replace repository-wide scan caps with continuing work batches.
|
|
182
|
-
- Reuse unchanged source/configuration metadata across scans, re-resolve changed dotenv bindings, and remove deleted declarations. Keep activity collection independent and disclose skipped files or interrupted indexing.
|
|
183
|
-
- Reuse unchanged reference catalogs in the browser and report complete issue counts alongside the bounded detail list. Request-history and runtime inventory limits are unchanged.
|
|
184
|
-
- Preserve ordinary function names containing `task_` instead of mistaking their suffix for a credential prefix.
|
|
185
|
-
|
|
186
|
-
## 0.1.6-preview.4 — 2026-10-10
|
|
187
|
-
|
|
188
|
-
- Index supported application code, nested configuration, custom endpoints, and conservative model/call-site references automatically during `openmerit dash`, with file-and-line evidence.
|
|
189
|
-
- Separate static declarations, aliases, examples, benchmarks, candidates, fallbacks, and unresolved references from recorded inference usage. Retain custom service identities without guessing vendors or hardware.
|
|
190
|
-
- Add a searchable code/configuration view, coverage notes, and a counts-only `openmerit dash --diagnostics` report for troubleshooting without repository access.
|
|
191
|
-
|
|
192
|
-
## 0.1.6-preview.3 — 2026-10-10
|
|
193
|
-
|
|
194
|
-
- Expand passive project endpoint discovery to 39 service identities across model APIs, inference clouds, managed cloud endpoints, and gateways; share classification with the service inventory and preserve unknown/custom endpoint boundaries.
|
|
195
|
-
|
|
196
|
-
- Recognize project-declared Hugging Face dedicated endpoints and the inference router as separate services, without cloud calls or token-based inference assumptions.
|
|
197
|
-
|
|
198
|
-
- Detect changed JSONL checkpoints before resuming incremental reads, including same-inode truncation followed by growth between polls.
|
|
199
|
-
- Disclose conflicting completed/failed outcomes when reconciling copies of a request.
|
|
200
|
-
- Prefer fresh model residency over stale reports from other exported hosts.
|
|
201
|
-
- Preserve unavailable usage in model details when no request source is connected.
|
|
202
|
-
|
|
203
|
-
## 0.1.6-preview.2 — 2026-10-09
|
|
204
|
-
|
|
205
|
-
- Require selected-project declarations for service and runtime discovery; remove unconditional Ollama/standard-port probes and inherited shell service detection.
|
|
206
|
-
- Keep endpoint identity separate from physical placement. A loopback URL does not prove that inference runs on this machine; local compute requires explicit host links.
|
|
207
|
-
- Retain OpenMerit task summaries separately from request counts, models, tokens, and spend.
|
|
208
|
-
- Preserve unknown request status, currency, and cost basis. Missing activity evidence no longer appears as measured zero usage or failures.
|
|
209
|
-
- Disclose bounded inventory omissions and partial source coverage.
|
|
210
|
-
|
|
211
|
-
## 0.1.6-preview.1 — 2026-10-09
|
|
212
|
-
|
|
213
|
-
- Retain historical application requests beyond 24 hours; add 7d, 30d, and default All time with calendar labels, available-history dates, and explicit partial-coverage disclosure. Existing source/read limits still apply.
|
|
214
|
-
|
|
215
|
-
## 0.1.6-preview.0 — 2026-10-09
|
|
216
|
-
|
|
217
|
-
A test prerelease of the existing `openmerit` npm package, installed with `npm install -g openmerit@preview`. Includes the compiled Inference Terminal, automatic browser opening, and a free loopback-port fallback. `latest` remains `0.1.5`.
|
|
218
|
-
|
|
219
|
-
- Add the Inference Terminal: `openmerit dash` serves a local, read-only view of models, request metadata, connections, capacity, and compute, sampled every 10–30 seconds. Agent and task identities come from supplied metadata; the terminal does not intercept inference or discover every call automatically.
|
|
220
|
-
- Bind task profiles, metrics, assessments, frontier artifacts, swaps, and
|
|
221
|
-
automation signals to a confirmed application LLM target; Pi's own model
|
|
222
|
-
telemetry cannot satisfy application evidence or become a swap target.
|
|
223
|
-
- Make Pi's automatic setup conditional on bounded source detection of an
|
|
224
|
-
application LLM call. Projects without a detected target remain idle until
|
|
225
|
-
`/openmerit setup` is requested.
|
|
226
|
-
|
|
227
|
-
## 0.1.5 — 2026-09-23
|
|
228
|
-
|
|
229
|
-
OpenMerit now ships as one npm package with a Pi extension, a reusable
|
|
230
|
-
coordinator, and a harness-neutral protocol.
|
|
231
|
-
|
|
232
|
-
### Changed
|
|
233
|
-
|
|
234
|
-
- Pi issues typed, bounded work to the coding harness and checks structured
|
|
235
|
-
results and durable evidence before advancing the model-improvement lifecycle.
|
|
236
|
-
- Project state, audit events, and evidence manifests live under `.openmerit/`
|
|
237
|
-
in the product repository.
|
|
238
|
-
- Baseline assessment, candidate trials, frontier verification, supervised
|
|
239
|
-
swaps, post-swap verification, and rollback follow a confirmed project policy.
|
|
240
|
-
- The package exposes `openmerit/core`, `openmerit/protocol`, and `openmerit/pi`.
|
|
241
|
-
- Node.js 22.19 or newer and Pi 0.87 are required for the Pi integration.
|
|
242
|
-
|
|
243
|
-
### Removed from the 0.1.4 package
|
|
244
|
-
|
|
245
|
-
- The background watcher, direct provider clients, standalone CLI, and
|
|
246
|
-
session-bound route comparison flow.
|
|
247
|
-
- Automatic use of the earlier home-directory state. The new project-local
|
|
248
|
-
state starts with a confirmed task profile and leaves older files untouched.
|
|
249
|
-
|
|
250
|
-
See [the README](README.md) for installation and the current command surface.
|