openmerit 0.1.11 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/README.md +20 -56
  3. package/dist/pi/scout-extension.mjs +190 -0
  4. package/dist/pi/src/index.d.ts +6 -32
  5. package/dist/pi/src/index.js +49 -787
  6. package/dist/terminal/public/app.js +37 -37
  7. package/dist/terminal/public/styles.css +0 -125
  8. package/dist/terminal/scout-update.json +1 -1
  9. package/dist/terminal/src/cli.mjs +2 -0
  10. package/dist/terminal/src/install.mjs +103 -166
  11. package/dist/terminal/src/scout-credentials.d.ts +12 -0
  12. package/dist/terminal/src/scout-credentials.js +41 -0
  13. package/dist/terminal/src/scout-credentials.ts +44 -0
  14. package/dist/terminal/src/scout-instructions.d.ts +5 -0
  15. package/dist/terminal/src/scout-instructions.js +116 -0
  16. package/dist/terminal/src/scout-instructions.ts +119 -0
  17. package/dist/terminal/src/scout-pi.d.ts +44 -0
  18. package/dist/terminal/src/scout-pi.js +84 -0
  19. package/dist/terminal/src/scout-pi.ts +76 -0
  20. package/dist/terminal/src/scout.d.ts +77 -0
  21. package/dist/terminal/src/scout.js +181 -0
  22. package/dist/terminal/src/scout.ts +164 -0
  23. package/dist/terminal/src/server.mjs +1 -1
  24. package/dist/terminal/src/telemetry.mjs +2 -2
  25. package/docs/commands.md +3 -77
  26. package/docs/getting-started.md +20 -45
  27. package/docs/index.md +9 -14
  28. package/docs/inference-terminal.md +2 -63
  29. package/docs/pi-extension.md +32 -28
  30. package/docs/roadmap.md +6 -18
  31. package/docs/scout.md +39 -255
  32. package/docs/security.md +8 -14
  33. package/docs/testing.md +8 -34
  34. package/docs/troubleshooting.md +23 -28
  35. package/package.json +6 -27
  36. package/dist/core/src/index.d.ts +0 -101
  37. package/dist/core/src/index.js +0 -1649
  38. package/dist/core/src/store.d.ts +0 -35
  39. package/dist/core/src/store.js +0 -102
  40. package/dist/pi/src/scheduler.d.ts +0 -11
  41. package/dist/pi/src/scheduler.js +0 -137
  42. package/dist/pi/src/wakeup.d.ts +0 -2
  43. package/dist/pi/src/wakeup.js +0 -108
  44. package/dist/protocol/src/index.d.ts +0 -484
  45. package/dist/protocol/src/index.js +0 -47
  46. package/dist/protocol/src/schemas.d.ts +0 -576
  47. package/dist/protocol/src/schemas.js +0 -280
  48. package/dist/terminal/public/brands/vllm.ico +0 -0
  49. package/dist/terminal/public/flow.css +0 -1
  50. package/dist/terminal/public/flow.js +0 -770
  51. package/dist/terminal/src/demo-policy.mjs +0 -205
  52. package/dist/terminal/src/enterprise-sample.mjs +0 -151
  53. package/docs/adapter-guide.md +0 -189
  54. package/docs/architecture.md +0 -59
  55. package/docs/automation.md +0 -74
  56. package/docs/budgets.md +0 -37
  57. package/docs/demo-backfill.md +0 -31
  58. package/docs/demo-fieldkit.md +0 -49
  59. package/docs/demo-placement.md +0 -32
  60. package/docs/demo-spam.md +0 -17
  61. package/docs/demo-support.md +0 -44
  62. package/docs/demo.md +0 -74
  63. package/docs/first-trial.md +0 -60
  64. package/docs/interactive-demo.md +0 -118
  65. package/docs/lifecycle.md +0 -30
  66. package/docs/metrics-and-evidence.md +0 -48
  67. package/docs/operations.md +0 -38
  68. package/docs/pareto-spec.md +0 -76
  69. package/docs/saga/a-steadier-compass.md +0 -25
  70. package/docs/saga/defining-a-task-for-ai.md +0 -154
  71. package/docs/saga/the-benchmark-is-not-the-job.md +0 -33
  72. package/docs/ux-reference.md +0 -32
package/docs/security.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # Security
2
2
 
3
+ **Pi adapter — npm 0.1.12 and later:** Pi 1.5.0 uses native MCP plus shared Scout rule/skill sections, not the former evaluation adapter. It never initiates login, changes the host model/tools/session, or provisions background jobs. Browser-approved credentials are independent per client/scope/project, stored privately outside the repository, and passed as MCP HTTP headers, never instructions. Pi's installed connection JSON contains only the private credential path. See [Pi paths and targeted removal](pi-extension.md). The standalone evaluation CLI remains separate.
4
+
3
5
  The [preview agent-evaluation runner](agent-evaluations.md) is a separate execution path: it runs trusted local agents/graders, passes only basic OS variables and explicitly named environment variables, and preserves agent-owned authentication. It sends task descriptions and optional model IDs to Scout. Local metric reports and budget reservations persist; temporary task outputs are removed. Fresh directories are not security sandboxes. Its budget admission controls do not enforce provider billing limits.
4
6
 
5
7
  The [budget selector](agent-evaluations.md#budget-selection) in `0.1.7-preview.2` requires an explicit $10, custom or no-fixed-cap choice after model scope. No fixed cap removes monetary admission limits, including Claude's per-attempt dollar flag, while retaining tasks, repetition bounds, timeouts, native permissions and accounting. Saved choices apply across reruns; a foreground watcher can incur further charges on each change until stopped. Provider limits remain independent.
@@ -8,27 +10,17 @@ The [budget selector](agent-evaluations.md#budget-selection) in `0.1.7-preview.2
8
10
 
9
11
  OpenMerit accepts no model-provider or Jev credential. Credentials belong to the coding harness, its environment, or an external secret store.
10
12
 
11
- Never place secrets in profiles, policies, intent constraints, summaries, evidence URIs, audit events, fixtures, traces, or committed environment files.
12
-
13
- ## Mutation authorization
14
-
15
- Most intents carry `modelMutationAllowed: false`. Only application-target apply-swap and rollback intents can authorize a mutation, limited to the relevant proposal, prior configuration, and confirmed target ID. Pi's model is explicitly excluded. Automatic swaps require explicit policy opt-in and meeting the configured verified-swap threshold, which can be zero. Otherwise a proposal requires approval. Post-swap evaluation depends on the harness performing the checks and reporting their results. OpenMerit is not a sandbox around the harness's tools.
16
-
17
- Persistent wakeup provisioning is a separate authorization boundary. An adapter that needs to install a system scheduler, create a paid cloud agent, or change infrastructure must declare `wakeupProvisioning: "infrastructure_change"`; OpenMerit will not request that provisioning unless the confirmed check policy sets `infrastructureChangesAllowed: true`.
18
-
19
- The macOS Pi adapter installs a user LaunchAgent only under that authorization and persistent execution mode. The plist stores the project path, selected Pi model identifier, and executable paths, but no API key. It verifies that Pi authentication works without shell-only credentials before installing. The agent launches Pi headlessly only for due elapsed-time checks, so the configured model provider may receive project context and incur charges under the confirmed policy. Pause unloads the agent; already-running work is not cancelled.
20
-
21
- The background runner imposes a 15-minute wall-clock limit and records a nonzero status if Pi fails to complete a due intent. This is a safety bound, not a provider-spend cap. Unattended candidate discovery and model changes still need representative product validation.
13
+ Never place secrets in evaluation configuration, task fixtures, Scout project policy, terminal configuration, activity exports, traces, or committed environment files.
22
14
 
23
15
  ## Local state
24
16
 
25
- `.openmerit/` may reveal task names, candidate identifiers, evidence locations, budgets, or model choices. Treat it as sensitive project metadata even though it must not contain credentials.
17
+ `.openmerit/` may hold evaluation configuration, task fixtures, reports and budget usage, Scout project policy, and terminal configuration or activity exports. It may reveal task names, model choices, budgets or endpoint identifiers. Treat it as sensitive project metadata even though it must not contain credentials.
26
18
 
27
- Pi also records OpenMerit requests and results in its session history. Project context and evaluation data may be sent to the model providers, tools, and evidence systems configured in the harness. Local OpenMerit storage does not imply that all processing stays on your machine.
19
+ With native Scout, Pi manages its own session/tool records and provider context; the adapter adds no separate evaluation-history store. Task descriptions sent to hosted Scout follow its data policy. Local OpenMerit storage does not imply that all processing stays on your machine.
28
20
 
29
21
  ## Evidence integrity
30
22
 
31
- OpenMerit validates structure and deterministic Pareto conformance, not the truth of arbitrary external evidence. Use authenticated observability, immutable artifacts, and digests where stronger provenance is required.
23
+ OpenMerit does not establish the truth of imported or external evidence. Use authenticated observability, immutable artifacts, and digests where stronger provenance is required.
32
24
 
33
25
  Do not include credentials, private traces, or proprietary evaluation data in public issues.
34
26
 
@@ -55,6 +47,8 @@ The [coding-agent setup](scout.md#connect-a-coding-agent) in npm 0.1.7 uses host
55
47
 
56
48
  **Automatic instruction updates — version 0.1.10:** updater-enabled setup defaults on with an opt-out, installing asynchronous Claude Code startup/response-end hooks. Checks use npm's public stable registry without credentials, task content or installation IDs. SHA-512 verification checks package bytes against registry metadata; it does not defend against a compromised publisher/registry. The updater reads only a bounded instruction bundle, never executes downloaded code or package scripts, and leaves MCP configuration, keys, privacy settings and project policy unchanged. A supported release contract permits instruction updates; maintainers must change it when permissions or data handling change. This is a publisher assertion, not semantic validation of instructions. Edited files are retained, concurrent writes serialize, and journaled recovery stops on intervening edits. New sessions reliably load new instructions. See [automatic updates](scout.md#automatic-instruction-updates-version-0110) for cadence, offline retries, manual commands and the one-time bootstrap required for older installations. Availability follows npm publication independently of hosted releases.
57
49
 
50
+ These update hooks are Claude Code only. The new Pi adapter installs none; updates require explicit targeted reinstall. Its `login`, `logout`, `status`, `remove` and `doctor` shell commands target the selected client/scope without revoking another installation's independent key.
51
+
58
52
  ### OpenMerit coding-agent reset — 23 October 2026
59
53
 
60
54
  **Historical reset, superseded by npm 0.1.7 and hosted device setup.** The reset temporarily removed the installer and MCP transport. The current [Claude Code setup](scout.md#connect-a-coding-agent) uses browser-approved, device-specific keys scoped to MCP recommendations, not pasted session tokens or agent OAuth. The key is stored locally in mode-600 credentials and Claude Code's plain-text MCP configuration. Setup installs an owned Scout skill; conflicts are not overwritten, and removal revokes the device key. Browser Google/GitHub sign-in remains separate.
package/docs/testing.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # Testing and validation
2
2
 
3
+ **Pi adapter — npm 0.1.12 and later:** the Pi 1.5.0 adapter uses native Scout MCP and shared guidance only. Dated acceptance records below describe the packages tested at the time; some mention package exports that have since been removed. Standalone evaluation tests remain separate.
4
+
3
5
  ## Stable 0.1.7 acceptance — 24 October 2026
4
6
 
5
7
  Published `openmerit@0.1.7` to npm's `latest` channel after full repository verification: 757 passing tests, one existing skip, syntax/type checks, builds and documentation checks. The `preview` tag remains `0.1.7-preview.5`. A fresh registry installation on macOS ARM64 with Node.js 22.23.1 matched the tested 104-file tarball byte-for-byte. Packed core/protocol/Pi exports and the installed terminal runtime smoke checks passed, including passive collection, source removal, runtime outage/recovery and shutdown.
@@ -32,7 +34,7 @@ The final production deployment passed **203/203 VM assertions and 8/8 filter/am
32
34
 
33
35
  The published `0.1.7-preview.4` package passed the full repository verification, including 329 Scout tests and documentation checks. Its packaged runtime is byte-for-byte unchanged from preview.3; version metadata and public guides carry this release. Registry integrity matches the reviewed tarball, the `preview` tag resolves to this version, and stable `latest` remains `0.1.6`. Scout deploys independently and its UI files are unchanged. Broader evidence coverage, task calibration and complete quality/cost/latency validation remain follow-up work.
34
36
 
35
- The hosted beta's public post-deployment suite passed 9/11 cases. Explicit SVG generation returned no shortlist; a Qwen-family code-authoring request admitted tool-calling evidence. Restriction checks passed for creator, family, context and dates, including known matches and impossible-context no-match. The actual bird prompt also displayed five recommendations in a browser. Earlier candidate deadline and interpretation failures are retained in the [Scout validation record](scout.md#task-relevance-corrections-hosted-beta-17-october-2026). These checks do not establish universal semantic reliability or improved task accuracy.
37
+ The hosted beta's public post-deployment suite passed 9/11 cases. Explicit SVG generation returned no shortlist; a Qwen-family code-authoring request admitted tool-calling evidence. Restriction checks passed for creator, family, context and dates, including known matches and impossible-context no-match. The actual bird prompt also displayed five recommendations in a browser. Earlier candidate deadline and interpretation failures were recorded separately at the time. These checks do not establish universal semantic reliability or improved task accuracy.
36
38
 
37
39
  Fresh registry installation in the existing Ubuntu ARM64 Lima VM verified all 101 packaged file hashes, registry integrity and public core/protocol/Pi exports. Seven workloads were sent through both HTTP and the official MCP SDK. Six of those fourteen requests failed their functional expectations: books/posters and invoice HTTP requests returned unavailable responses, while SQL generation and safety classification with rewriting returned empty shortlists through both interfaces. The matching MCP books/posters and invoice requests succeeded; bird identification, label-only classification and unsupported audio behaved as expected. The first harness stopped when it assumed SQL had returned an endpoint. A separate continuation tested exact endpoint scope using the successful bird result, request/protocol errors and installed CLI planning, preserving all original failures. Together the runs recorded 187 passing checks and nine failed assertions, including duplicate agreement failures caused by unavailable responses and an installed-CLI candidate check. Both CLI email and SQL plans were empty. Planning created no execution or spend state, and the existing global VM installation was retained. This is not a clean acceptance pass. The seven first HTTP workload requests had a median wall time of 4.924 seconds and a maximum of 6.229 seconds, including unavailable responses; repeated cached requests were faster.
38
40
 
@@ -58,14 +60,6 @@ npm pack --dry-run
58
60
 
59
61
  `verify` runs workspace tests, syntax checks, strict TypeScript checks and the TypeScript migration ratchet, the package build, demo tests, generated-documentation checks, and whitespace validation. It does not publish a package or deploy the website.
60
62
 
61
- ## Coordinator and Pi coverage
62
-
63
- Automated tests cover protocol contracts, task-specific sampling, application-target binding, setup and post-build target detection, incumbent identity, deterministic baseline sufficiency, durable candidate/assessment handoffs, frontier conformance, approval, swap and rollback requests, persistence, evidence requirements, budget accounting, and audit redaction.
64
-
65
- Automation coverage includes task and elapsed cadence, signal idempotency, durable pending work, catalogue changes, regression thresholds, delayed post-swap checks, scheduler capability gaps, and persistent pause. Pi's own usage and coding checkpoints are not counted as application measurements.
66
-
67
- The disposable-project test in `packages/pi/test/live-wakeup.test.ts` verifies that a due, insufficient baseline is recorded without starting Pi. Scheduler tests validate managed LaunchAgent configuration and authorization boundaries. They do not establish reliable unattended candidate discovery, comparison, or swapping; those require a representative headless product trial.
68
-
69
63
  ## Inference Terminal coverage
70
64
 
71
65
  ```sh
@@ -74,35 +68,15 @@ npm test --workspace=@openmerit/terminal
74
68
 
75
69
  Tests cover metadata projection and redaction, source containment, recursive syntax indexing, phased worker progress, incremental reuse, coverage limits, endpoint classification, authorized loopback probes, supported request-log formats, UTF-8 boundaries, rotation/truncation, source reconciliation, historical ranges, unknown versus zero measurements, and the read-only API.
76
70
 
77
- Inventory and UI coverage includes model/service identity, explicit deployment and host links, shared capacity, stale/missing hardware, path classification, map completeness, search and filtering, preference-dependent demo decisions, and preservation of unknown costs. Runtime fixtures establish which endpoints collectors call; they do not prove compatibility with every external runtime version.
78
-
79
- The three fictional scenarios are separate:
80
-
81
- | Scenario | Agents | Tasks | Models / offers | Requests |
82
- | --- | ---: | ---: | --- | ---: |
83
- | Local sample (`--demo` or sample selector) | 40 | 140 | 21 models | 9,600 over 24 hours |
84
- | Hosted enterprise sample | 200 | 699 | 21 models | 9.6 million over 30 days |
85
- | Hosted before/after comparison | 280 | 980 | 26 reviewed offers across six services; assignments depend on preference | 9.6 million over 30 days |
71
+ Inventory and UI coverage includes model/service identity, explicit deployment and host links, shared capacity, stale/missing hardware, path classification, map completeness, search and filtering, and preservation of unknown costs. Runtime fixtures establish which endpoints collectors call; they do not prove compatibility with every external runtime version.
86
72
 
87
- The comparison tests preserve workload and token volume across all six preferences, enforce service/output compatibility, disclose benchmark/efficiency overrides, and verify modeled costs. Sample tests exercise interface scale and arithmetic, not measured throughput, application quality, or hardware performance.
73
+ The local fictional sample (`--demo` or the sample selector) has 40 agents, 140 tasks, 21 models and 9,600 requests over 24 hours. Sample tests exercise interface scale and arithmetic, not measured throughput, application quality, or hardware performance.
88
74
 
89
- Browser review covers the six views, inspectors, search, activity ranges, map and chart interaction, keyboard navigation, reduced motion, and desktop/narrow layouts. The source light/dark change has local browser verification; it is not yet published to npm or the hosted site. Browser checks are distinct from the committed unit suite.
75
+ Browser review covers the six views, inspectors, search, activity ranges, map and chart interaction, keyboard navigation, reduced motion, and desktop/narrow layouts. Browser checks are distinct from the committed unit suite.
90
76
 
91
77
  ## Pi load and package smoke tests
92
78
 
93
- ```sh
94
- ./node_modules/.bin/pi --no-session --no-extensions -e ./packages/pi/src/index.ts --list-models
95
- ```
96
-
97
- This checks extension loading and catalogue enumeration, not successful provider authentication or a completed product intent. CI also packs the package, installs it in a temporary directory, imports the core/protocol entry points, and loads the installed extension.
98
-
99
- ## Experimental sandbox
100
-
101
- `npm run demo` builds a disposable sentiment-classification application through Pi and runs live provider calls. It requires Pi 0.87 with OpenRouter authentication and an application OpenRouter credential. It is separate from the fictional dashboard comparison and from the recorded website player.
102
-
103
- The runner collects a baseline, interleaves incumbent/challenger calls on the prepared cases, records cost/token/latency evidence, submits results to the coordinator, and exercises a change and fresh checks in the temporary app. Missing usage, invalid output, or exhausted allowance stops the run. The default application-spend threshold is $1, checked after responses; Pi's own coding-model spend is excluded. This is not an in-flight provider cutoff.
104
-
105
- The default models are `openai/gpt-4.1-mini` and `openai/gpt-4o-mini`. The runner supports `OPENMERIT_DEMO_BASELINE_MODEL`, comma-separated `OPENMERIT_DEMO_CANDIDATE_MODELS`, `OPENMERIT_DEMO_PI_MODEL`, `OPENMERIT_DEMO_REPETITIONS` (2–5), `OPENMERIT_DEMO_MAX_SPEND_USD` (up to $5), and `OPENMERIT_DEMO_SEED`. The printed temporary directory retains the app, plan, lifecycle state, per-call artifacts, comparison manifest, and summary. Small prepared examples do not establish representative product reliability.
79
+ CI packs the package, installs it in a temporary directory, imports `openmerit/pi`, smoke-tests the installed terminal command, and loads the installed extension in Pi with `--list-models`. These checks establish packaging and loading, not successful provider authentication or Scout recommendations.
106
80
 
107
81
  ## What evidence establishes
108
82
 
@@ -111,7 +85,7 @@ The default models are `openai/gpt-4.1-mini` and `openai/gpt-4o-mini`. The runne
111
85
  - **Product trial verified:** a representative application completes the workflow with inspected evidence.
112
86
  - **Production verified:** repeated representative runs support the claimed operating conditions and reliability.
113
87
 
114
- Implementation checks, historical demo recordings, and public model benchmarks cannot substitute for target-product validation. See [current limitations](roadmap.md) before relying on unattended changes.
88
+ Implementation checks and public model benchmarks cannot substitute for target-product validation. See [current limitations](roadmap.md).
115
89
 
116
90
 
117
91
  ## Agent-evaluation preview validation — 15 October 2026
@@ -1,55 +1,50 @@
1
1
  # Troubleshooting
2
2
 
3
- Start with the current status and the local audit trail. These show what OpenMerit requested, what the harness reported, and which evidence references were recorded.
3
+ The Pi Scout adapter requires OpenMerit **0.1.12** or newer and targets Pi **1.5.0**. Check the resolved package version when using `@latest`. The removed Pi 0.87 `/openmerit` workflow is not a diagnostic path for the replacement.
4
4
 
5
- ```text
6
- /openmerit status
7
- /openmerit doctor
8
- /openmerit logs
9
- ```
10
-
11
- ## The command is missing
5
+ ## Scout is missing in Pi
12
6
 
13
- Check that OpenMerit is installed and enabled in Pi:
7
+ Run targeted checks in your shell:
14
8
 
15
9
  ```sh
16
- pi list
10
+ npx -y openmerit@latest status --client pi --scope user --check
11
+ npx -y openmerit@latest doctor --client pi --scope user
17
12
  ```
18
13
 
19
- The published extension targets Pi 0.87 and Node.js 22.19 or newer. Follow [getting started](getting-started.md) to install it, then start Pi in your product repository.
20
-
21
- ## Setup did not begin
14
+ Use `--scope project` from the original project root for project setup. Check Pi and Node.js versions against [setup](getting-started.md), complete browser device approval, then start a **new session**. Installing the npm package or reconnecting an old session does not load new extension instructions. For SDK embedding, load Pi's `createMcpExtension()` explicitly alongside `createScoutPiExtension`; the Scout factory alone is not an MCP runtime.
22
15
 
23
- Automatic setup requires an unconfigured project, an interactive Pi UI, and a bounded source-detection match for an application LLM call. Non-interactive runs do not start the confirmation flow, and a project without a detected target remains idle until `/openmerit setup` is requested. Existing project state or active work may also prevent a fresh setup request.
16
+ ## The connection configuration is invalid
24
17
 
25
- Inspect status first. If no intent is active, use `/openmerit setup` to request setup explicitly; this is also the supported correction path when source detection or a prior setup recorded the wrong incumbent. A new setup preserves audit history but replaces the active task profile, candidate/assessment handoffs, frontier, and baseline state after the new result is accepted. Do not delete `.openmerit/` as a routine reset: it contains your policy, lifecycle state, evidence references, and audit history.
18
+ The installed extension reads `openmerit-scout.json` beside its bundle; that file points to a private credential file outside the repository. Do not put the key in the extension or prompt. Check the scope and configuration-directory overrides against [installed paths](pi-extension.md#installed-files-and-credentials). Reinstall explicitly to repair missing owned files:
26
19
 
27
- ## Evidence is missing or incomplete
20
+ ```sh
21
+ npx -y openmerit@latest install --client pi --scope user
22
+ ```
28
23
 
29
- A task profile is not the same as working telemetry. Ask Pi which required metrics are uncovered, where their measurements should come from, and which evaluation artifacts have been run. Normal coding checkpoints do not replace those measurements.
24
+ Expired or revoked credentials require targeted `login` followed by reinstall. The adapter reports a sanitized setup diagnostic; it never starts sign-in on its own. Approve only a browser code you initiated.
30
25
 
31
- The [metrics guide](metrics-and-evidence.md) explains missing and insufficient evidence. More samples or instrumentation may be needed before a candidate can be compared.
26
+ ## Installation reports a conflict
32
27
 
33
- ## A result was rejected
28
+ The installer does not overwrite edited or unowned files. Inspect and preserve your changes before resolving the conflict. Also check for overlapping Pi user/project installations or an existing `openmerit` MCP entry. Remove the unwanted owned installation using its own scope; do not revoke a different client's key as a repair. Disable the old `npm:openmerit` Pi registration when migrating from 0.87.
34
29
 
35
- Read the validation error and audit event. Successful work needs evidence references and the output structure required by that intent. Frontier results must match the task and product revisions, identify their assessments, and satisfy the comparison rules.
30
+ ## The adapter did not update automatically
36
31
 
37
- Have the harness correct the measurement or output problem. Do not mark an intent successful merely to advance the workflow.
32
+ That is expected. Pi has no OpenMerit auto-update hooks. Explicitly rerun `install --client pi` with the original scope, then start a new session. Claude Code's update settings do not configure Pi.
38
33
 
39
- ## Work appears stuck after a restart
34
+ ## `/openmerit` or evaluation tools are missing
40
35
 
41
- Pending or active intent state is durable. Inspect `state.json` and `events.jsonl` using the paths from `/openmerit logs`, and ask the harness to reconcile that work with the current session before requesting another intent. The current adapter may need supervised recovery; automatic completion after a restart is not guaranteed.
36
+ They were removed, not renamed. The Pi adapter now supplies only native Scout MCP and shared guidance. It does not set up evaluations, recover intents, schedule work, swap models, or install a LaunchAgent. Existing legacy background jobs must be removed separately; installing the new adapter is not evidence that they stopped. Preserve old evaluation state and audit files if needed. For new evaluations use [the standalone CLI](agent-evaluations.md).
42
37
 
43
- ## Checks are not running while Pi is closed
38
+ ## Scout was not consulted or its evidence is insufficient
44
39
 
45
- On macOS, check that the confirmed policy requests persistent execution and allows infrastructure changes, that the Mac user is logged in, and that Pi authentication works without shell-only credentials. `/openmerit doctor` reports scheduler diagnostics; `.openmerit/scheduler-status.json` records completed wakeups and failures. An active pause unloads the managed LaunchAgent. Other platforms need an external scheduler. A successful schedule does not establish that model-driven work completed. See [automatic checks](automation.md).
40
+ The rule and skill are model guidance, not a guaranteed tool gate. Confirm they loaded in a new session and that the host permits the MCP tools. When project model policy is missing, ask the user; when Scout is unavailable or lacks suitable evidence, pause the model choice rather than invent a result. Host model/tool/session restrictions remain authoritative. Scout candidates are not measurements of your application's quality.
46
41
 
47
- ## Work continues after pausing
42
+ ## Evaluation or terminal problems
48
43
 
49
- Pause is durable and gates automatic work, but an already-active intent can continue and explicit commands remain available. Inspect the active intent and audit timestamps to distinguish prior work from a new automatic dispatch. `/openmerit cancel` records abandonment; it does not terminate a provider request or harness process. The standalone terminal is independent of Pi pause; stop its CLI with Ctrl+C to stop collection. If a new automatic intent starts while the persisted pause is active, record a sanitized reproduction and your installed version.
44
+ The standalone products are separate from Pi setup. Use [agent-evaluation diagnostics](agent-evaluations.md) for tasks, graders, budgets and execution blocks, or [terminal diagnostics](inference-terminal.md) for missing inventory and request evidence. A populated dashboard is not proof of successful evaluations. Never mark missing or rejected evidence successful merely to advance a workflow.
50
45
 
51
46
  ## Report a reproducible problem
52
47
 
53
- Include the OpenMerit, Pi, and Node.js versions, the triggering command or signal, the lifecycle stage, and a sanitized error. Remove credentials, private traces, and proprietary evaluation data before sharing anything in a public issue.
48
+ Include OpenMerit, Pi and Node.js versions, the client/scope, the triggering command, and a sanitized error. Remove credentials, private paths, prompts and proprietary data before sharing. Do not attach credential or connection files containing private configuration.
54
49
 
55
50
  [Open an issue on GitHub](https://github.com/laz-aslam/openmerit/issues)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "openmerit",
3
- "version": "0.1.11",
3
+ "version": "0.2.0",
4
4
  "description": "Evidence-driven model decisions for coding harnesses.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -11,12 +11,9 @@
11
11
  "pi-package",
12
12
  "pi-extension",
13
13
  "model-routing",
14
- "evaluation",
15
- "pareto-frontier"
14
+ "evaluation"
16
15
  ],
17
16
  "files": [
18
- "dist/core/",
19
- "dist/protocol/",
20
17
  "dist/pi/",
21
18
  "dist/terminal/",
22
19
  "dist/eval/",
@@ -26,18 +23,6 @@
26
23
  "README.md"
27
24
  ],
28
25
  "exports": {
29
- ".": {
30
- "types": "./dist/core/src/index.d.ts",
31
- "default": "./dist/core/src/index.js"
32
- },
33
- "./core": {
34
- "types": "./dist/core/src/index.d.ts",
35
- "default": "./dist/core/src/index.js"
36
- },
37
- "./protocol": {
38
- "types": "./dist/protocol/src/index.d.ts",
39
- "default": "./dist/protocol/src/index.js"
40
- },
41
26
  "./pi": {
42
27
  "types": "./dist/pi/src/index.d.ts",
43
28
  "default": "./dist/pi/src/index.js"
@@ -77,7 +62,6 @@
77
62
  "deploy": "node scripts/deploy.ts",
78
63
  "deploy:status": "node scripts/deploy.ts status",
79
64
  "build": "node scripts/build.mjs",
80
- "demo": "node scripts/demo.mjs",
81
65
  "predash": "node scripts/build-terminal.mjs",
82
66
  "dash": "node packages/terminal/src/cli.mjs dash",
83
67
  "build:docs": "node scripts/build-docs.mjs",
@@ -87,24 +71,19 @@
87
71
  "prepare": "npm run build",
88
72
  "verify": "npm test && npm run check && npm run typecheck && npm run check:typescript && npm run build && npm run check:demo && npm run check:scout && npm run check:docs && npm run build:scout && git diff --check",
89
73
  "prepublishOnly": "npm run verify",
90
- "build:dash-preview": "node scripts/build-terminal-preview.mjs",
91
- "deploy:dash-preview": "node scripts/deploy.ts terminal-preview",
92
74
  "scout": "node scout/dev.mjs",
93
75
  "scout:refresh": "node scout/refresh.mjs",
94
76
  "check:scout": "node --test scout/test/*.test.mjs scout/test/*.test.ts && node --check scout/worker.mjs",
95
77
  "build:scout": "node scripts/build-scout.mjs"
96
78
  },
97
79
  "dependencies": {
98
- "@earendil-works/pi-ai": "0.87.0",
99
80
  "@iarna/toml": "2.2.5",
100
81
  "@lezer/javascript": "1.5.6",
101
82
  "@lezer/python": "1.1.19",
102
- "hyparquet": "1.31.5",
103
- "typebox": "1.3.27",
104
83
  "yaml": "2.9.1"
105
84
  },
106
85
  "peerDependencies": {
107
- "@earendil-works/pi-coding-agent": ">=0.87.0 <0.88.0"
86
+ "@earendil-works/pi-coding-agent": "1.5.0"
108
87
  },
109
88
  "peerDependenciesMeta": {
110
89
  "@earendil-works/pi-coding-agent": {
@@ -112,16 +91,15 @@
112
91
  }
113
92
  },
114
93
  "devDependencies": {
94
+ "@earendil-works/pi-coding-agent": "1.5.0",
115
95
  "@clerk/backend": "3.26.1",
116
96
  "@clerk/shared": "4.41.0",
117
- "@lezer/highlight": "1.2.5",
118
97
  "@resvg/resvg-js": "2.6.2",
119
98
  "@types/markdown-it": "14.1.2",
120
99
  "@types/node": "^22.18.6",
121
- "@xyflow/react": "12.12.0",
122
- "elkjs": "0.12.0",
123
100
  "esbuild": "0.28.2",
124
101
  "fflate": "0.8.2",
102
+ "hyparquet": "1.31.5",
125
103
  "liveline": "0.0.7",
126
104
  "markdown-it": "15.0.2",
127
105
  "miniflare": "5.20260921.0-alpha",
@@ -130,6 +108,7 @@
130
108
  "react": "19.3.0",
131
109
  "react-dom": "19.3.0",
132
110
  "torph": "0.1.3",
111
+ "typebox": "1.3.27",
133
112
  "typescript": "^5.9.3",
134
113
  "web-vitals": "6.2.3",
135
114
  "wrangler": "4.136.3"
@@ -1,101 +0,0 @@
1
- import type { AnyIntentResult, AutomationPlan, AutomationSignal, CandidateAssessment, FrontierSnapshot, HarnessCapability, HarnessDescriptor, HarnessDispatchReceipt, HarnessWakeupRequest, TaskProfile, ImprovementCycleState, IntentLifecycleUpdate, IntentKind, OpenMeritIntent, OpenMeritProjectConfig, OpenMeritProjectState, ObservabilityCoverage, JsonValue, SupervisedGraduationPolicy } from "../../protocol/src/index.ts";
2
- import type { OpenMeritStore } from "./store.ts";
3
- export { OPENMERIT_DIRECTORY, ProjectStore, type AuditEventExporter, type OpenMeritStore, type ProjectStoreOptions, } from "./store.ts";
4
- export interface HarnessAdapter {
5
- readonly descriptor: HarnessDescriptor;
6
- dispatch(intent: OpenMeritIntent): Promise<HarnessDispatchReceipt>;
7
- scheduleWakeup?(request: HarnessWakeupRequest): Promise<HarnessDispatchReceipt>;
8
- }
9
- export interface IntentFactoryOptions {
10
- readonly taskProfileId?: string | undefined;
11
- readonly authorizationPolicyId?: string;
12
- readonly policy?: SupervisedGraduationPolicy | undefined;
13
- readonly id?: string;
14
- readonly requestedAt?: string;
15
- readonly evaluationSpend?: number | undefined;
16
- readonly targetId?: string | undefined;
17
- readonly additionalConstraints?: Readonly<Record<string, JsonValue>>;
18
- }
19
- export declare function createOpenMeritIntent(kind: IntentKind, options?: IntentFactoryOptions): OpenMeritIntent;
20
- export declare function missingHarnessCapabilities(descriptor: HarnessDescriptor, kind: IntentKind): readonly HarnessCapability[];
21
- export declare function validateHarnessDescriptor(descriptor: HarnessDescriptor): VerificationResult;
22
- export declare function dispatchToHarness(adapter: HarnessAdapter, intent: OpenMeritIntent): Promise<HarnessDispatchReceipt>;
23
- export declare function startedStage(kind: IntentKind): ImprovementCycleState["stage"];
24
- export declare function completedStage(kind: IntentKind, succeeded: boolean): ImprovementCycleState["stage"];
25
- export interface VerificationResult {
26
- readonly valid: boolean;
27
- readonly errors: readonly string[];
28
- }
29
- export interface ResultInterpretation extends VerificationResult {
30
- readonly nextStage?: ImprovementCycleState["stage"];
31
- readonly config?: OpenMeritProjectConfig | undefined;
32
- readonly cyclePatch?: Partial<ImprovementCycleState> | undefined;
33
- }
34
- export declare function validateObservabilityCoverage(profile: TaskProfile, coverage: ObservabilityCoverage, requireReady: boolean): readonly string[];
35
- export declare function interpretIntentResult(intent: OpenMeritIntent, result: AnyIntentResult, cycle: ImprovementCycleState, config: OpenMeritProjectConfig): ResultInterpretation;
36
- export type OrchestrationDecision = {
37
- readonly action: "issue_intent";
38
- readonly intentKind: IntentKind;
39
- readonly reason: string;
40
- } | {
41
- readonly action: "await_user";
42
- readonly reason: string;
43
- } | {
44
- readonly action: "observe";
45
- readonly reason: string;
46
- };
47
- export declare function nextImprovementAction(state: ImprovementCycleState): OrchestrationDecision;
48
- export declare function verifyFrontierSnapshot(profile: TaskProfile, assessments: readonly CandidateAssessment[], snapshot: FrontierSnapshot): VerificationResult;
49
- export interface IssuedIntent {
50
- readonly intent: OpenMeritIntent;
51
- readonly state: OpenMeritProjectState;
52
- readonly receipt: HarnessDispatchReceipt;
53
- }
54
- export interface CoordinatorIssueOptions {
55
- readonly additionalConstraints?: Readonly<Record<string, JsonValue>> | undefined;
56
- }
57
- export interface AcceptedIntentResult {
58
- readonly intentKind: IntentKind;
59
- readonly state: OpenMeritProjectState;
60
- readonly interpretation: ResultInterpretation;
61
- readonly automation: {
62
- readonly plan: AutomationPlan;
63
- readonly receipt?: HarnessDispatchReceipt;
64
- };
65
- }
66
- export interface ObservationResult {
67
- readonly state: OpenMeritProjectState;
68
- readonly assessmentDue: boolean;
69
- }
70
- export interface CatalogChangeResult {
71
- readonly state: OpenMeritProjectState;
72
- readonly changed: boolean;
73
- readonly reassessmentDue: boolean;
74
- }
75
- export interface AutomationSignalResult {
76
- readonly state: OpenMeritProjectState;
77
- readonly duplicate: boolean;
78
- readonly decision: OrchestrationDecision;
79
- }
80
- export declare function buildAutomationPlan(state: OpenMeritProjectState, descriptor: HarnessDescriptor): AutomationPlan;
81
- /** Harness-neutral durable coordinator. Adapters transport jobs; this class owns lifecycle state. */
82
- export declare class OpenMeritCoordinator {
83
- private readonly store;
84
- private readonly adapter;
85
- constructor(store: OpenMeritStore, adapter: HarnessAdapter);
86
- issue(kind: IntentKind, options?: CoordinatorIssueOptions): Promise<IssuedIntent>;
87
- reconcileAutomation(now?: string): Promise<{
88
- readonly plan: AutomationPlan;
89
- readonly receipt?: HarnessDispatchReceipt;
90
- }>;
91
- acceptResult(result: AnyIntentResult): Promise<AcceptedIntentResult>;
92
- recordLifecycle(update: IntentLifecycleUpdate): Promise<void>;
93
- recordTaskObservation(details?: Readonly<Record<string, JsonValue>>): Promise<ObservationResult>;
94
- assessBaseline(now?: string): Promise<{
95
- readonly state: OpenMeritProjectState;
96
- readonly errors: readonly string[];
97
- readonly decision: OrchestrationDecision;
98
- }>;
99
- recordAutomationSignal(signal: AutomationSignal): Promise<AutomationSignalResult>;
100
- recordModelCatalogFingerprint(fingerprint: string): Promise<CatalogChangeResult>;
101
- }