openmerit 0.1.12 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +11 -27
- package/dist/pi/scout-extension.mjs +23 -10
- package/dist/terminal/scout-update.json +1 -1
- package/dist/terminal/src/browser.mjs +2 -4
- package/dist/terminal/src/cli.mjs +6 -56
- package/dist/terminal/src/install.mjs +3 -2
- package/dist/terminal/src/scout-instructions.js +23 -10
- package/dist/terminal/src/scout-pi.js +1 -1
- package/dist/terminal/src/scout.js +181 -0
- package/dist/terminal/src/telemetry.mjs +8 -60
- package/docs/agent-evaluations.md +8 -148
- package/docs/commands.md +4 -86
- package/docs/getting-started.md +2 -16
- package/docs/index.md +4 -27
- package/docs/inference-terminal.md +13 -655
- package/docs/pi-extension.md +1 -1
- package/docs/roadmap.md +4 -17
- package/docs/scout.md +34 -251
- package/docs/security.md +8 -33
- package/docs/testing.md +7 -47
- package/docs/troubleshooting.md +3 -3
- package/package.json +13 -46
- package/dist/core/src/index.d.ts +0 -101
- package/dist/core/src/index.js +0 -1649
- package/dist/core/src/store.d.ts +0 -35
- package/dist/core/src/store.js +0 -102
- package/dist/eval/src/adapters.mjs +0 -112
- package/dist/eval/src/cli.mjs +0 -151
- package/dist/eval/src/config.mjs +0 -108
- package/dist/eval/src/process.mjs +0 -44
- package/dist/eval/src/report.mjs +0 -59
- package/dist/eval/src/runner.mjs +0 -123
- package/dist/eval/src/scout.mjs +0 -48
- package/dist/protocol/src/index.d.ts +0 -484
- package/dist/protocol/src/index.js +0 -47
- package/dist/protocol/src/schemas.d.ts +0 -576
- package/dist/protocol/src/schemas.js +0 -280
- package/dist/terminal/public/app.js +0 -298
- package/dist/terminal/public/brands/anthropic.png +0 -0
- package/dist/terminal/public/brands/baai.png +0 -0
- package/dist/terminal/public/brands/baseten.png +0 -0
- package/dist/terminal/public/brands/cerebras.png +0 -0
- package/dist/terminal/public/brands/cohere.png +0 -0
- package/dist/terminal/public/brands/deepseek.ico +0 -0
- package/dist/terminal/public/brands/google.png +0 -0
- package/dist/terminal/public/brands/groq.ico +0 -0
- package/dist/terminal/public/brands/huggingface.svg +0 -8
- package/dist/terminal/public/brands/lm-studio.png +0 -0
- package/dist/terminal/public/brands/meta.ico +0 -0
- package/dist/terminal/public/brands/mistral.png +0 -0
- package/dist/terminal/public/brands/nomic.png +0 -0
- package/dist/terminal/public/brands/ollama.png +0 -0
- package/dist/terminal/public/brands/openai.png +0 -0
- package/dist/terminal/public/brands/openrouter.png +0 -0
- package/dist/terminal/public/brands/qwen.png +0 -0
- package/dist/terminal/public/brands/vllm.ico +0 -0
- package/dist/terminal/public/brands/vllm.png +0 -0
- package/dist/terminal/public/favicon.svg +0 -1
- package/dist/terminal/public/flow.css +0 -1
- package/dist/terminal/public/flow.js +0 -770
- package/dist/terminal/public/index.html +0 -22
- package/dist/terminal/public/styles.css +0 -1046
- package/dist/terminal/src/activity-merge.mjs +0 -66
- package/dist/terminal/src/collect.mjs +0 -370
- package/dist/terminal/src/coverage.mjs +0 -33
- package/dist/terminal/src/demo-policy.mjs +0 -205
- package/dist/terminal/src/discovery.mjs +0 -99
- package/dist/terminal/src/enterprise-sample.mjs +0 -151
- package/dist/terminal/src/hardware.mjs +0 -57
- package/dist/terminal/src/history.mjs +0 -32
- package/dist/terminal/src/log-continuity.mjs +0 -26
- package/dist/terminal/src/project-activity.mjs +0 -157
- package/dist/terminal/src/repository-background.mjs +0 -37
- package/dist/terminal/src/repository-index.mjs +0 -267
- package/dist/terminal/src/repository-policy.mjs +0 -3
- package/dist/terminal/src/repository-syntax.mjs +0 -236
- package/dist/terminal/src/repository-worker.mjs +0 -15
- package/dist/terminal/src/sample.mjs +0 -171
- package/dist/terminal/src/scout-credentials.d.ts +0 -12
- package/dist/terminal/src/scout-credentials.ts +0 -44
- package/dist/terminal/src/scout-instructions.d.ts +0 -5
- package/dist/terminal/src/scout-instructions.ts +0 -106
- package/dist/terminal/src/scout-pi.d.ts +0 -44
- package/dist/terminal/src/scout-pi.ts +0 -76
- package/dist/terminal/src/server.mjs +0 -95
- package/dist/terminal/src/service-registry.mjs +0 -78
- package/dist/terminal/src/services.mjs +0 -44
- package/dist/terminal/src/topology.mjs +0 -30
- package/dist/terminal/src/updates.d.ts +0 -62
- package/dist/terminal/src/updates.ts +0 -313
- package/docs/adapter-guide.md +0 -191
- package/docs/architecture.md +0 -61
- package/docs/automation.md +0 -76
- package/docs/budgets.md +0 -39
- package/docs/demo-backfill.md +0 -31
- package/docs/demo-fieldkit.md +0 -49
- package/docs/demo-placement.md +0 -32
- package/docs/demo-spam.md +0 -17
- package/docs/demo-support.md +0 -44
- package/docs/demo.md +0 -74
- package/docs/first-trial.md +0 -62
- package/docs/interactive-demo.md +0 -118
- package/docs/lifecycle.md +0 -32
- package/docs/metrics-and-evidence.md +0 -48
- package/docs/operations.md +0 -40
- package/docs/pareto-spec.md +0 -76
- package/docs/ux-reference.md +0 -32
package/docs/security.md
CHANGED
|
@@ -1,54 +1,29 @@
|
|
|
1
1
|
# Security
|
|
2
2
|
|
|
3
|
-
**Pi
|
|
3
|
+
**Pi adapter — npm 0.1.12 and later:** Pi 1.5.0 uses native MCP plus shared Scout rule/skill sections, not the former evaluation adapter. It never initiates login, changes the host model/tools/session, or provisions background jobs. Browser-approved credentials are independent per client/scope/project, stored privately outside the repository, and passed as MCP HTTP headers, never instructions. Pi's installed connection JSON contains only the private credential path. See [Pi paths and targeted removal](pi-extension.md). The standalone evaluation CLI remains separate.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
**Retirement:** npm 0.3.0 removes the standalone [agent-evaluation runner](agent-evaluations.md), including planning; unreleased Scout source removes `/scout/api/recommend`. Existing local reports, budgets, configuration and host-agent instructions are not deleted; review them separately. No provider billing control or remote data deletion is implied.
|
|
6
6
|
|
|
7
|
-
The
|
|
7
|
+
The Scout agent, MCP, publisher and account analytics/deletion capabilities remain. Historical analytics and stored catalogue/research data are not erased by endpoint retirement. Necessary provider processing for the agent and publisher continues under the [Scout data policy](scout-data-policy.md).
|
|
8
8
|
|
|
9
9
|
## Credential ownership
|
|
10
10
|
|
|
11
11
|
OpenMerit accepts no model-provider or Jev credential. Credentials belong to the coding harness, its environment, or an external secret store.
|
|
12
12
|
|
|
13
|
-
Never place secrets in
|
|
14
|
-
|
|
15
|
-
## Mutation authorization
|
|
16
|
-
|
|
17
|
-
This section describes core/custom evaluation harness policy. Pi-specific lifecycle and background-runner behavior below is **historical Pi 0.87 functionality**, removed from the new Scout adapter. Installing the replacement does not establish that an old background job was removed; manage that job separately.
|
|
18
|
-
|
|
19
|
-
Most intents carry `modelMutationAllowed: false`. Only application-target apply-swap and rollback intents can authorize a mutation, limited to the relevant proposal, prior configuration, and confirmed target ID. Pi's model is explicitly excluded. Automatic swaps require explicit policy opt-in and meeting the configured verified-swap threshold, which can be zero. Otherwise a proposal requires approval. Post-swap evaluation depends on the harness performing the checks and reporting their results. OpenMerit is not a sandbox around the harness's tools.
|
|
20
|
-
|
|
21
|
-
Persistent wakeup provisioning is a separate authorization boundary. An adapter that needs to install a system scheduler, create a paid cloud agent, or change infrastructure must declare `wakeupProvisioning: "infrastructure_change"`; OpenMerit will not request that provisioning unless the confirmed check policy sets `infrastructureChangesAllowed: true`.
|
|
22
|
-
|
|
23
|
-
The macOS Pi adapter installs a user LaunchAgent only under that authorization and persistent execution mode. The plist stores the project path, selected Pi model identifier, and executable paths, but no API key. It verifies that Pi authentication works without shell-only credentials before installing. The agent launches Pi headlessly only for due elapsed-time checks, so the configured model provider may receive project context and incur charges under the confirmed policy. Pause unloads the agent; already-running work is not cancelled.
|
|
24
|
-
|
|
25
|
-
The background runner imposes a 15-minute wall-clock limit and records a nonzero status if Pi fails to complete a due intent. This is a safety bound, not a provider-spend cap. Unattended candidate discovery and model changes still need representative product validation.
|
|
13
|
+
Never place secrets in evaluation configuration, task fixtures, Scout project policy, traces, or committed environment files.
|
|
26
14
|
|
|
27
15
|
## Local state
|
|
28
16
|
|
|
29
|
-
`.openmerit/` may
|
|
17
|
+
`.openmerit/` may hold evaluation configuration, task fixtures, reports and budget usage, and Scout project policy, plus files left by the retired Inference Terminal (`terminal.json`, `inference.jsonl`), which upgrades do not delete. It may reveal task names, model choices, budgets or endpoint identifiers. Treat it as sensitive project metadata even though it must not contain credentials.
|
|
30
18
|
|
|
31
|
-
|
|
19
|
+
With native Scout, Pi manages its own session/tool records and provider context; the adapter adds no separate evaluation-history store. Task descriptions sent to hosted Scout follow its data policy. Local OpenMerit storage does not imply that all processing stays on your machine.
|
|
32
20
|
|
|
33
21
|
## Evidence integrity
|
|
34
22
|
|
|
35
|
-
OpenMerit
|
|
23
|
+
OpenMerit does not establish the truth of imported or external evidence. Use authenticated observability, immutable artifacts, and digests where stronger provenance is required.
|
|
36
24
|
|
|
37
25
|
Do not include credentials, private traces, or proprietary evaluation data in public issues.
|
|
38
26
|
|
|
39
|
-
## Inference Terminal
|
|
40
|
-
|
|
41
|
-
The terminal reads supported project metadata, request logs, explicit exports, and authorized loopback runtime inventory/metrics endpoints. It calls no inference endpoint and exposes no mutation API. A local syntax index reads supported Python, JavaScript/TypeScript, JSON, YAML, TOML, and project-local dotenv declarations. It does not execute or import project code, source shell files, or resolve arbitrary expressions. Known credential-variable presence can identify configuration; credential values are discarded and never used for provider calls. Endpoint addresses are sanitized before retention. Static URLs found by the syntax index never authorize network probes.
|
|
42
|
-
|
|
43
|
-
Loopback runtime probes require supported project endpoint declarations or explicit terminal configuration. No standard ports, remote provider accounts, credential stores, running processes, or general home directories are scanned. Whole-machine readings are context, not per-request measurements; the compute view requires explicit links or supplied host evidence.
|
|
44
|
-
|
|
45
|
-
Coding-harness histories are outside the terminal's application scope and are not scanned or imported. Application request logs and explicit exports retain only allowlisted metadata, without prompts or responses. Sources must belong to the selected application; the terminal cannot infer the purpose of an unlabeled API response. Records explicitly marked `scope: "development"` are rejected.
|
|
46
|
-
|
|
47
|
-
The local web service binds to `127.0.0.1` and rejects unexpected Host, Origin, and cross-site browser requests. It only serves bundled assets and specific read-only JSON routes. It is not authenticated against other local processes or machine users. Model names, endpoint identifiers, and relative source paths can still be sensitive. Names and identifiers in imported metadata must be kept free of secrets.
|
|
48
|
-
|
|
49
|
-
Repository metadata and usage caches live in process memory and disappear when the terminal exits. The terminal creates no persistent index or files in the application. Per-file/parser protections, request-log read limits, runtime inventory limits, and explicit coverage gaps still apply. See [terminal data handling and limits](inference-terminal.md) for supported sources, precedence, retention bounds, diagnostics, and sample-data separation.
|
|
50
|
-
|
|
51
|
-
|
|
52
27
|
## Hosted Scout authentication
|
|
53
28
|
|
|
54
29
|
Hosted Scout uses Clerk Google/GitHub sign-in. The page and status are public; JSON recommendations require a valid Clerk session bearer token, not just a cookie. MCP methods require a device key from the setup below. Missing/invalid credentials return HTTP 401, and missing authentication configuration fails closed with HTTP 503. Clerk remains the identity system; the analytics integration below can synchronize selected profile fields to PostHog. Raw account IDs are used transiently for the per-user edge limiter and are not joined to historical Cloudflare prompt records. A prompt draft is temporarily stored in the current browser tab during sign-in. No Clerk Billing or automatic agent OAuth is included. See [Scout account sign-in](scout.md#account-sign-in-published-21-october-2026) for token lifetime, rate limits, data handling and the remaining real-login validation limits.
|
|
@@ -91,7 +66,7 @@ Historical records are accessible only through authenticated Cloudflare account
|
|
|
91
66
|
|
|
92
67
|
### Optional product analytics
|
|
93
68
|
|
|
94
|
-
Scout has a visible privacy disclosure and controls. Public homepage/docs navigation events remain source-only; npm 0.1.7 includes opt-in local terminal/map events and aggregate counts. The terminal relay cannot mutate project or inference state, retains loopback Host/Origin guards, and forwards only bounded allowlisted telemetry. Terminal and CLI use a process-only random ID; website events use a separate tab ID. Neither is implicitly linked to Clerk. Application names, raw inputs/outputs, local identifiers and source files remain local. Sanitized error classes/categories (and Scout application file/line frames) exclude raw messages, credentials, source context, arbitrary URLs and filesystem paths. No replay is enabled. All expanded account-linked events pass the same preference/deletion gate as profile synchronization; ordinary event delivery remains best effort. New search details link to identified accounts when profile sync is active, unlike the legacy Cloudflare archive. No external retention setting is changed.
|
|
69
|
+
Scout has a visible privacy disclosure and controls. Public homepage/docs navigation events remain source-only; npm 0.1.7 includes opt-in local terminal/map events and aggregate counts, which npm 0.3.0 removes with the [Inference Terminal](inference-terminal.md). The terminal relay cannot mutate project or inference state, retains loopback Host/Origin guards, and forwards only bounded allowlisted telemetry. Terminal and CLI use a process-only random ID; website events use a separate tab ID. Neither is implicitly linked to Clerk. Application names, raw inputs/outputs, local identifiers and source files remain local. Sanitized error classes/categories (and Scout application file/line frames) exclude raw messages, credentials, source context, arbitrary URLs and filesystem paths. No replay is enabled. All expanded account-linked events pass the same preference/deletion gate as profile synchronization; ordinary event delivery remains best effort. New search details link to identified accounts when profile sync is active, unlike the legacy Cloudflare archive. No external retention setting is changed.
|
|
95
70
|
|
|
96
71
|
The [PostHog integration](scout.md#product-analytics) uses the EU ingestion endpoint; unconfigured deployments remain off. Verified Clerk user IDs become namespaced SHA-256 pseudonyms for cross-client account retention. Profile sync can associate those identities with names and verified email addresses, so they are identifiable personal data. No identity is added to the historical Cloudflare prompt archive. The browser retains an anonymous ID and hashed account/session transition markers in tab storage, rotating the anonymous ID on account changes. There is no autocapture, replay or visitor-IP forwarding. PostHog has its own retention/access policies, not Cloudflare's three-month policy.
|
|
97
72
|
|
package/docs/testing.md
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# Testing and validation
|
|
2
2
|
|
|
3
|
-
**
|
|
3
|
+
**Cleanup:** npm 0.3.0 retires the standalone evaluation CLI and the Inference Terminal; unreleased source also retires the legacy JSON recommendation pipeline and unpublished docs website shell. Dated evaluation/API acceptance records below describe historical releases, not current supported commands. Current verification retains agent, MCP, publisher, status, policy and protected-memo coverage; it does not publish or deploy anything.
|
|
4
|
+
|
|
5
|
+
**Pi adapter — npm 0.1.12 and later:** the Pi 1.5.0 adapter uses native Scout MCP and shared guidance only. Dated acceptance records below describe the packages tested at the time; some mention package exports that have since been removed. Standalone evaluation tests remain separate.
|
|
4
6
|
|
|
5
7
|
## Stable 0.1.7 acceptance — 24 October 2026
|
|
6
8
|
|
|
@@ -34,7 +36,7 @@ The final production deployment passed **203/203 VM assertions and 8/8 filter/am
|
|
|
34
36
|
|
|
35
37
|
The published `0.1.7-preview.4` package passed the full repository verification, including 329 Scout tests and documentation checks. Its packaged runtime is byte-for-byte unchanged from preview.3; version metadata and public guides carry this release. Registry integrity matches the reviewed tarball, the `preview` tag resolves to this version, and stable `latest` remains `0.1.6`. Scout deploys independently and its UI files are unchanged. Broader evidence coverage, task calibration and complete quality/cost/latency validation remain follow-up work.
|
|
36
38
|
|
|
37
|
-
The hosted beta's public post-deployment suite passed 9/11 cases. Explicit SVG generation returned no shortlist; a Qwen-family code-authoring request admitted tool-calling evidence. Restriction checks passed for creator, family, context and dates, including known matches and impossible-context no-match. The actual bird prompt also displayed five recommendations in a browser. Earlier candidate deadline and interpretation failures
|
|
39
|
+
The hosted beta's public post-deployment suite passed 9/11 cases. Explicit SVG generation returned no shortlist; a Qwen-family code-authoring request admitted tool-calling evidence. Restriction checks passed for creator, family, context and dates, including known matches and impossible-context no-match. The actual bird prompt also displayed five recommendations in a browser. Earlier candidate deadline and interpretation failures were recorded separately at the time. These checks do not establish universal semantic reliability or improved task accuracy.
|
|
38
40
|
|
|
39
41
|
Fresh registry installation in the existing Ubuntu ARM64 Lima VM verified all 101 packaged file hashes, registry integrity and public core/protocol/Pi exports. Seven workloads were sent through both HTTP and the official MCP SDK. Six of those fourteen requests failed their functional expectations: books/posters and invoice HTTP requests returned unavailable responses, while SQL generation and safety classification with rewriting returned empty shortlists through both interfaces. The matching MCP books/posters and invoice requests succeeded; bird identification, label-only classification and unsupported audio behaved as expected. The first harness stopped when it assumed SQL had returned an endpoint. A separate continuation tested exact endpoint scope using the successful bird result, request/protocol errors and installed CLI planning, preserving all original failures. Together the runs recorded 187 passing checks and nine failed assertions, including duplicate agreement failures caused by unavailable responses and an installed-CLI candidate check. Both CLI email and SQL plans were empty. Planning created no execution or spend state, and the existing global VM installation was retained. This is not a clean acceptance pass. The seven first HTTP workload requests had a median wall time of 4.924 seconds and a maximum of 6.229 seconds, including unavailable responses; repeated cached requests were faster.
|
|
40
42
|
|
|
@@ -58,53 +60,11 @@ npm run verify
|
|
|
58
60
|
npm pack --dry-run
|
|
59
61
|
```
|
|
60
62
|
|
|
61
|
-
`verify` runs workspace tests, syntax checks, strict TypeScript checks and the TypeScript migration ratchet, the package build,
|
|
62
|
-
|
|
63
|
-
## Coordinator and Pi coverage
|
|
64
|
-
|
|
65
|
-
Automated tests cover protocol contracts, task-specific sampling, application-target binding, setup and post-build target detection, incumbent identity, deterministic baseline sufficiency, durable candidate/assessment handoffs, frontier conformance, approval, swap and rollback requests, persistence, evidence requirements, budget accounting, and audit redaction.
|
|
66
|
-
|
|
67
|
-
Automation coverage includes task and elapsed cadence, signal idempotency, durable pending work, catalogue changes, regression thresholds, delayed post-swap checks, scheduler capability gaps, and persistent pause. Pi's own usage and coding checkpoints are not counted as application measurements.
|
|
68
|
-
|
|
69
|
-
The disposable-project test in `packages/pi/test/live-wakeup.test.ts` verifies that a due, insufficient baseline is recorded without starting Pi. Scheduler tests validate managed LaunchAgent configuration and authorization boundaries. They do not establish reliable unattended candidate discovery, comparison, or swapping; those require a representative headless product trial.
|
|
70
|
-
|
|
71
|
-
## Inference Terminal coverage
|
|
72
|
-
|
|
73
|
-
```sh
|
|
74
|
-
npm test --workspace=@openmerit/terminal
|
|
75
|
-
```
|
|
76
|
-
|
|
77
|
-
Tests cover metadata projection and redaction, source containment, recursive syntax indexing, phased worker progress, incremental reuse, coverage limits, endpoint classification, authorized loopback probes, supported request-log formats, UTF-8 boundaries, rotation/truncation, source reconciliation, historical ranges, unknown versus zero measurements, and the read-only API.
|
|
78
|
-
|
|
79
|
-
Inventory and UI coverage includes model/service identity, explicit deployment and host links, shared capacity, stale/missing hardware, path classification, map completeness, search and filtering, preference-dependent demo decisions, and preservation of unknown costs. Runtime fixtures establish which endpoints collectors call; they do not prove compatibility with every external runtime version.
|
|
80
|
-
|
|
81
|
-
The three fictional scenarios are separate:
|
|
82
|
-
|
|
83
|
-
| Scenario | Agents | Tasks | Models / offers | Requests |
|
|
84
|
-
| --- | ---: | ---: | --- | ---: |
|
|
85
|
-
| Local sample (`--demo` or sample selector) | 40 | 140 | 21 models | 9,600 over 24 hours |
|
|
86
|
-
| Hosted enterprise sample | 200 | 699 | 21 models | 9.6 million over 30 days |
|
|
87
|
-
| Hosted before/after comparison | 280 | 980 | 26 reviewed offers across six services; assignments depend on preference | 9.6 million over 30 days |
|
|
88
|
-
|
|
89
|
-
The comparison tests preserve workload and token volume across all six preferences, enforce service/output compatibility, disclose benchmark/efficiency overrides, and verify modeled costs. Sample tests exercise interface scale and arithmetic, not measured throughput, application quality, or hardware performance.
|
|
90
|
-
|
|
91
|
-
Browser review covers the six views, inspectors, search, activity ranges, map and chart interaction, keyboard navigation, reduced motion, and desktop/narrow layouts. The source light/dark change has local browser verification; it is not yet published to npm or the hosted site. Browser checks are distinct from the committed unit suite.
|
|
63
|
+
`verify` runs workspace tests, syntax checks, strict TypeScript checks and the TypeScript migration ratchet, the package build, memo tests, policy/memo build and Markdown-guide checks, Scout tests/build, and whitespace validation. It does not publish a package or deploy the website. `npm run check:worker-types` separately checks generated Worker types; CI also performs packed-install, CLI help and Pi load smoke checks.
|
|
92
64
|
|
|
93
65
|
## Pi load and package smoke tests
|
|
94
66
|
|
|
95
|
-
|
|
96
|
-
./node_modules/.bin/pi --no-session --no-extensions -e ./packages/pi/src/index.ts --list-models
|
|
97
|
-
```
|
|
98
|
-
|
|
99
|
-
This checks extension loading and catalogue enumeration, not successful provider authentication or a completed product intent. CI also packs the package, installs it in a temporary directory, imports the core/protocol entry points, and loads the installed extension.
|
|
100
|
-
|
|
101
|
-
## Experimental sandbox
|
|
102
|
-
|
|
103
|
-
`npm run demo` builds a disposable sentiment-classification application through Pi and runs live provider calls. It requires Pi 0.87 with OpenRouter authentication and an application OpenRouter credential. It is separate from the fictional dashboard comparison and from the recorded website player.
|
|
104
|
-
|
|
105
|
-
The runner collects a baseline, interleaves incumbent/challenger calls on the prepared cases, records cost/token/latency evidence, submits results to the coordinator, and exercises a change and fresh checks in the temporary app. Missing usage, invalid output, or exhausted allowance stops the run. The default application-spend threshold is $1, checked after responses; Pi's own coding-model spend is excluded. This is not an in-flight provider cutoff.
|
|
106
|
-
|
|
107
|
-
The default models are `openai/gpt-4.1-mini` and `openai/gpt-4o-mini`. The runner supports `OPENMERIT_DEMO_BASELINE_MODEL`, comma-separated `OPENMERIT_DEMO_CANDIDATE_MODELS`, `OPENMERIT_DEMO_PI_MODEL`, `OPENMERIT_DEMO_REPETITIONS` (2–5), `OPENMERIT_DEMO_MAX_SPEND_USD` (up to $5), and `OPENMERIT_DEMO_SEED`. The printed temporary directory retains the app, plan, lifecycle state, per-call artifacts, comparison manifest, and summary. Small prepared examples do not establish representative product reliability.
|
|
67
|
+
CI packs the package, installs it in a temporary directory, imports `openmerit/pi`, checks the installed `openmerit --help`, and loads the installed extension in Pi with `--list-models`. These checks establish packaging and loading, not successful provider authentication or Scout recommendations.
|
|
108
68
|
|
|
109
69
|
## What evidence establishes
|
|
110
70
|
|
|
@@ -113,7 +73,7 @@ The default models are `openai/gpt-4.1-mini` and `openai/gpt-4o-mini`. The runne
|
|
|
113
73
|
- **Product trial verified:** a representative application completes the workflow with inspected evidence.
|
|
114
74
|
- **Production verified:** repeated representative runs support the claimed operating conditions and reliability.
|
|
115
75
|
|
|
116
|
-
Implementation checks
|
|
76
|
+
Implementation checks and public model benchmarks cannot substitute for target-product validation. See [current limitations](roadmap.md).
|
|
117
77
|
|
|
118
78
|
|
|
119
79
|
## Agent-evaluation preview validation — 15 October 2026
|
package/docs/troubleshooting.md
CHANGED
|
@@ -33,15 +33,15 @@ That is expected. Pi has no OpenMerit auto-update hooks. Explicitly rerun `insta
|
|
|
33
33
|
|
|
34
34
|
## `/openmerit` or evaluation tools are missing
|
|
35
35
|
|
|
36
|
-
They were removed, not renamed. The Pi adapter now supplies only native Scout MCP and shared guidance. It does not set up evaluations, recover intents, schedule work, swap models, or install a LaunchAgent. Existing legacy background jobs must be removed separately; installing the new adapter is not evidence that they stopped. Preserve old evaluation state and audit files if needed. For new evaluations use [the standalone CLI](agent-evaluations.md)
|
|
36
|
+
They were removed, not renamed. The Pi adapter now supplies only native Scout MCP and shared guidance. It does not set up evaluations, recover intents, schedule work, swap models, or install a LaunchAgent. Existing legacy background jobs must be removed separately; installing the new adapter is not evidence that they stopped. Preserve old evaluation state and audit files if needed. For new evaluations use [the standalone CLI](agent-evaluations.md).
|
|
37
37
|
|
|
38
38
|
## Scout was not consulted or its evidence is insufficient
|
|
39
39
|
|
|
40
40
|
The rule and skill are model guidance, not a guaranteed tool gate. Confirm they loaded in a new session and that the host permits the MCP tools. When project model policy is missing, ask the user; when Scout is unavailable or lacks suitable evidence, pause the model choice rather than invent a result. Host model/tool/session restrictions remain authoritative. Scout candidates are not measurements of your application's quality.
|
|
41
41
|
|
|
42
|
-
##
|
|
42
|
+
## `openmerit dash` or `openmerit eval` is an unknown command
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
Both are retired in npm 0.3.0; see the [Inference Terminal](inference-terminal.md) and [evaluation CLI](agent-evaluations.md) retirement notes. Existing local files are not deleted. Never mark missing or rejected evidence successful merely to advance a workflow.
|
|
45
45
|
|
|
46
46
|
## Report a reproducible problem
|
|
47
47
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "openmerit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Evidence-driven model decisions for coding harnesses.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -11,33 +11,18 @@
|
|
|
11
11
|
"pi-package",
|
|
12
12
|
"pi-extension",
|
|
13
13
|
"model-routing",
|
|
14
|
-
"evaluation"
|
|
15
|
-
"pareto-frontier"
|
|
14
|
+
"evaluation"
|
|
16
15
|
],
|
|
17
16
|
"files": [
|
|
18
|
-
"dist/core/",
|
|
19
|
-
"dist/protocol/",
|
|
20
17
|
"dist/pi/",
|
|
21
18
|
"dist/terminal/",
|
|
22
|
-
"dist/
|
|
19
|
+
"!dist/terminal/src/*.ts",
|
|
23
20
|
"docs/",
|
|
24
21
|
"CHANGELOG.md",
|
|
25
22
|
"LICENSE",
|
|
26
23
|
"README.md"
|
|
27
24
|
],
|
|
28
25
|
"exports": {
|
|
29
|
-
".": {
|
|
30
|
-
"types": "./dist/core/src/index.d.ts",
|
|
31
|
-
"default": "./dist/core/src/index.js"
|
|
32
|
-
},
|
|
33
|
-
"./core": {
|
|
34
|
-
"types": "./dist/core/src/index.d.ts",
|
|
35
|
-
"default": "./dist/core/src/index.js"
|
|
36
|
-
},
|
|
37
|
-
"./protocol": {
|
|
38
|
-
"types": "./dist/protocol/src/index.d.ts",
|
|
39
|
-
"default": "./dist/protocol/src/index.js"
|
|
40
|
-
},
|
|
41
26
|
"./pi": {
|
|
42
27
|
"types": "./dist/pi/src/index.d.ts",
|
|
43
28
|
"default": "./dist/pi/src/index.js"
|
|
@@ -70,6 +55,7 @@
|
|
|
70
55
|
"scripts": {
|
|
71
56
|
"check": "npm run check --workspaces --if-present",
|
|
72
57
|
"test": "npm run test --workspaces --if-present",
|
|
58
|
+
"pretypecheck": "npm run build:site",
|
|
73
59
|
"typecheck": "tsc -p tsconfig.json && tsc -p tsconfig.scout.json && tsc -p scout/agent/tsconfig.json",
|
|
74
60
|
"check:typescript": "node scripts/check-typescript.ts",
|
|
75
61
|
"types:workers": "node scripts/worker-types.ts",
|
|
@@ -77,31 +63,17 @@
|
|
|
77
63
|
"deploy": "node scripts/deploy.ts",
|
|
78
64
|
"deploy:status": "node scripts/deploy.ts status",
|
|
79
65
|
"build": "node scripts/build.mjs",
|
|
80
|
-
"
|
|
81
|
-
"
|
|
82
|
-
"dash": "node packages/terminal/src/cli.mjs dash",
|
|
83
|
-
"build:docs": "node scripts/build-docs.mjs",
|
|
84
|
-
"check:docs": "npm run build:docs && node --test scripts/docs.test.mjs",
|
|
66
|
+
"build:site": "node scripts/build-site.ts",
|
|
67
|
+
"check:docs": "npm run build:site && node --test scripts/docs.test.ts",
|
|
85
68
|
"memo": "node demo/memo/preview.ts",
|
|
86
|
-
"check:demo": "npm ci --ignore-scripts --prefix demo && npm run build:
|
|
69
|
+
"check:demo": "npm ci --ignore-scripts --prefix demo && npm run build:site && tsc -p tsconfig.memo.json && node --test demo/test/*.test.mjs demo/test/*.test.ts",
|
|
87
70
|
"prepare": "npm run build",
|
|
88
71
|
"verify": "npm test && npm run check && npm run typecheck && npm run check:typescript && npm run build && npm run check:demo && npm run check:scout && npm run check:docs && npm run build:scout && git diff --check",
|
|
89
72
|
"prepublishOnly": "npm run verify",
|
|
90
|
-
"build:dash-preview": "node scripts/build-terminal-preview.mjs",
|
|
91
|
-
"deploy:dash-preview": "node scripts/deploy.ts terminal-preview",
|
|
92
|
-
"scout": "node scout/dev.mjs",
|
|
93
73
|
"scout:refresh": "node scout/refresh.mjs",
|
|
94
74
|
"check:scout": "node --test scout/test/*.test.mjs scout/test/*.test.ts && node --check scout/worker.mjs",
|
|
95
75
|
"build:scout": "node scripts/build-scout.mjs"
|
|
96
76
|
},
|
|
97
|
-
"dependencies": {
|
|
98
|
-
"@iarna/toml": "2.2.5",
|
|
99
|
-
"@lezer/javascript": "1.5.6",
|
|
100
|
-
"@lezer/python": "1.1.19",
|
|
101
|
-
"hyparquet": "1.31.5",
|
|
102
|
-
"typebox": "1.3.27",
|
|
103
|
-
"yaml": "2.9.1"
|
|
104
|
-
},
|
|
105
77
|
"peerDependencies": {
|
|
106
78
|
"@earendil-works/pi-coding-agent": "1.5.0"
|
|
107
79
|
},
|
|
@@ -111,26 +83,21 @@
|
|
|
111
83
|
}
|
|
112
84
|
},
|
|
113
85
|
"devDependencies": {
|
|
114
|
-
"@earendil-works/pi-coding-agent": "1.5.0",
|
|
115
86
|
"@clerk/backend": "3.26.1",
|
|
116
87
|
"@clerk/shared": "4.41.0",
|
|
88
|
+
"@earendil-works/pi-coding-agent": "1.5.0",
|
|
89
|
+
"@lezer/javascript": "1.5.6",
|
|
117
90
|
"@resvg/resvg-js": "2.6.2",
|
|
118
|
-
"@types/markdown-it": "14.1.2",
|
|
119
91
|
"@types/node": "^22.18.6",
|
|
120
|
-
"@xyflow/react": "12.12.0",
|
|
121
|
-
"elkjs": "0.12.0",
|
|
122
92
|
"esbuild": "0.28.2",
|
|
123
93
|
"fflate": "0.8.2",
|
|
124
|
-
"
|
|
94
|
+
"hyparquet": "1.31.5",
|
|
125
95
|
"markdown-it": "15.0.2",
|
|
126
96
|
"miniflare": "5.20260921.0-alpha",
|
|
127
|
-
"
|
|
128
|
-
"morphdom": "2.7.8",
|
|
129
|
-
"react": "19.3.0",
|
|
130
|
-
"react-dom": "19.3.0",
|
|
131
|
-
"torph": "0.1.3",
|
|
97
|
+
"typebox": "1.3.27",
|
|
132
98
|
"typescript": "^5.9.3",
|
|
133
99
|
"web-vitals": "6.2.3",
|
|
134
|
-
"wrangler": "4.136.3"
|
|
100
|
+
"wrangler": "4.136.3",
|
|
101
|
+
"yaml": "2.9.1"
|
|
135
102
|
}
|
|
136
103
|
}
|
package/dist/core/src/index.d.ts
DELETED
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
import type { AnyIntentResult, AutomationPlan, AutomationSignal, CandidateAssessment, FrontierSnapshot, HarnessCapability, HarnessDescriptor, HarnessDispatchReceipt, HarnessWakeupRequest, TaskProfile, ImprovementCycleState, IntentLifecycleUpdate, IntentKind, OpenMeritIntent, OpenMeritProjectConfig, OpenMeritProjectState, ObservabilityCoverage, JsonValue, SupervisedGraduationPolicy } from "../../protocol/src/index.ts";
|
|
2
|
-
import type { OpenMeritStore } from "./store.ts";
|
|
3
|
-
export { OPENMERIT_DIRECTORY, ProjectStore, type AuditEventExporter, type OpenMeritStore, type ProjectStoreOptions, } from "./store.ts";
|
|
4
|
-
export interface HarnessAdapter {
|
|
5
|
-
readonly descriptor: HarnessDescriptor;
|
|
6
|
-
dispatch(intent: OpenMeritIntent): Promise<HarnessDispatchReceipt>;
|
|
7
|
-
scheduleWakeup?(request: HarnessWakeupRequest): Promise<HarnessDispatchReceipt>;
|
|
8
|
-
}
|
|
9
|
-
export interface IntentFactoryOptions {
|
|
10
|
-
readonly taskProfileId?: string | undefined;
|
|
11
|
-
readonly authorizationPolicyId?: string;
|
|
12
|
-
readonly policy?: SupervisedGraduationPolicy | undefined;
|
|
13
|
-
readonly id?: string;
|
|
14
|
-
readonly requestedAt?: string;
|
|
15
|
-
readonly evaluationSpend?: number | undefined;
|
|
16
|
-
readonly targetId?: string | undefined;
|
|
17
|
-
readonly additionalConstraints?: Readonly<Record<string, JsonValue>>;
|
|
18
|
-
}
|
|
19
|
-
export declare function createOpenMeritIntent(kind: IntentKind, options?: IntentFactoryOptions): OpenMeritIntent;
|
|
20
|
-
export declare function missingHarnessCapabilities(descriptor: HarnessDescriptor, kind: IntentKind): readonly HarnessCapability[];
|
|
21
|
-
export declare function validateHarnessDescriptor(descriptor: HarnessDescriptor): VerificationResult;
|
|
22
|
-
export declare function dispatchToHarness(adapter: HarnessAdapter, intent: OpenMeritIntent): Promise<HarnessDispatchReceipt>;
|
|
23
|
-
export declare function startedStage(kind: IntentKind): ImprovementCycleState["stage"];
|
|
24
|
-
export declare function completedStage(kind: IntentKind, succeeded: boolean): ImprovementCycleState["stage"];
|
|
25
|
-
export interface VerificationResult {
|
|
26
|
-
readonly valid: boolean;
|
|
27
|
-
readonly errors: readonly string[];
|
|
28
|
-
}
|
|
29
|
-
export interface ResultInterpretation extends VerificationResult {
|
|
30
|
-
readonly nextStage?: ImprovementCycleState["stage"];
|
|
31
|
-
readonly config?: OpenMeritProjectConfig | undefined;
|
|
32
|
-
readonly cyclePatch?: Partial<ImprovementCycleState> | undefined;
|
|
33
|
-
}
|
|
34
|
-
export declare function validateObservabilityCoverage(profile: TaskProfile, coverage: ObservabilityCoverage, requireReady: boolean): readonly string[];
|
|
35
|
-
export declare function interpretIntentResult(intent: OpenMeritIntent, result: AnyIntentResult, cycle: ImprovementCycleState, config: OpenMeritProjectConfig): ResultInterpretation;
|
|
36
|
-
export type OrchestrationDecision = {
|
|
37
|
-
readonly action: "issue_intent";
|
|
38
|
-
readonly intentKind: IntentKind;
|
|
39
|
-
readonly reason: string;
|
|
40
|
-
} | {
|
|
41
|
-
readonly action: "await_user";
|
|
42
|
-
readonly reason: string;
|
|
43
|
-
} | {
|
|
44
|
-
readonly action: "observe";
|
|
45
|
-
readonly reason: string;
|
|
46
|
-
};
|
|
47
|
-
export declare function nextImprovementAction(state: ImprovementCycleState): OrchestrationDecision;
|
|
48
|
-
export declare function verifyFrontierSnapshot(profile: TaskProfile, assessments: readonly CandidateAssessment[], snapshot: FrontierSnapshot): VerificationResult;
|
|
49
|
-
export interface IssuedIntent {
|
|
50
|
-
readonly intent: OpenMeritIntent;
|
|
51
|
-
readonly state: OpenMeritProjectState;
|
|
52
|
-
readonly receipt: HarnessDispatchReceipt;
|
|
53
|
-
}
|
|
54
|
-
export interface CoordinatorIssueOptions {
|
|
55
|
-
readonly additionalConstraints?: Readonly<Record<string, JsonValue>> | undefined;
|
|
56
|
-
}
|
|
57
|
-
export interface AcceptedIntentResult {
|
|
58
|
-
readonly intentKind: IntentKind;
|
|
59
|
-
readonly state: OpenMeritProjectState;
|
|
60
|
-
readonly interpretation: ResultInterpretation;
|
|
61
|
-
readonly automation: {
|
|
62
|
-
readonly plan: AutomationPlan;
|
|
63
|
-
readonly receipt?: HarnessDispatchReceipt;
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
export interface ObservationResult {
|
|
67
|
-
readonly state: OpenMeritProjectState;
|
|
68
|
-
readonly assessmentDue: boolean;
|
|
69
|
-
}
|
|
70
|
-
export interface CatalogChangeResult {
|
|
71
|
-
readonly state: OpenMeritProjectState;
|
|
72
|
-
readonly changed: boolean;
|
|
73
|
-
readonly reassessmentDue: boolean;
|
|
74
|
-
}
|
|
75
|
-
export interface AutomationSignalResult {
|
|
76
|
-
readonly state: OpenMeritProjectState;
|
|
77
|
-
readonly duplicate: boolean;
|
|
78
|
-
readonly decision: OrchestrationDecision;
|
|
79
|
-
}
|
|
80
|
-
export declare function buildAutomationPlan(state: OpenMeritProjectState, descriptor: HarnessDescriptor): AutomationPlan;
|
|
81
|
-
/** Harness-neutral durable coordinator. Adapters transport jobs; this class owns lifecycle state. */
|
|
82
|
-
export declare class OpenMeritCoordinator {
|
|
83
|
-
private readonly store;
|
|
84
|
-
private readonly adapter;
|
|
85
|
-
constructor(store: OpenMeritStore, adapter: HarnessAdapter);
|
|
86
|
-
issue(kind: IntentKind, options?: CoordinatorIssueOptions): Promise<IssuedIntent>;
|
|
87
|
-
reconcileAutomation(now?: string): Promise<{
|
|
88
|
-
readonly plan: AutomationPlan;
|
|
89
|
-
readonly receipt?: HarnessDispatchReceipt;
|
|
90
|
-
}>;
|
|
91
|
-
acceptResult(result: AnyIntentResult): Promise<AcceptedIntentResult>;
|
|
92
|
-
recordLifecycle(update: IntentLifecycleUpdate): Promise<void>;
|
|
93
|
-
recordTaskObservation(details?: Readonly<Record<string, JsonValue>>): Promise<ObservationResult>;
|
|
94
|
-
assessBaseline(now?: string): Promise<{
|
|
95
|
-
readonly state: OpenMeritProjectState;
|
|
96
|
-
readonly errors: readonly string[];
|
|
97
|
-
readonly decision: OrchestrationDecision;
|
|
98
|
-
}>;
|
|
99
|
-
recordAutomationSignal(signal: AutomationSignal): Promise<AutomationSignalResult>;
|
|
100
|
-
recordModelCatalogFingerprint(fingerprint: string): Promise<CatalogChangeResult>;
|
|
101
|
-
}
|