openmerit 0.1.12 → 0.3.0

package/dist/terminal/src/telemetry.mjs

@@ -5,12 +5,8 @@ export let clientVersion;
 for (const path of ['../../../package.json', '../../../../package.json']) {
   try { const pkg = JSON.parse(readFileSync(new URL(path, import.meta.url), 'utf8')); if (pkg.name === 'openmerit') { clientVersion = pkg.version; break; } } catch {}
 }
-export const frameFiles = Object.freeze(['cli.mjs', 'server.mjs', 'collect.mjs', 'repository-index.mjs', 'install.mjs', 'scout.mjs', 'config.mjs', 'runner.mjs', 'adapters.mjs', 'process.mjs', 'report.mjs', 'app.js']);
-const applicationURLs = frameFiles.flatMap(file => ['terminal/src/', 'eval/src/'].map(dir => new URL(`../../${dir}${file}`, import.meta.url).href));
-const diagnostics = new WeakMap();
-export function scoutDiagnostic(error, trace, code) {
-  if (error && typeof error === 'object' && ['scout_network_error', 'scout_http_error', 'scout_response_error', 'scout_timeout'].includes(code)) diagnostics.set(error, { operation: 'scout_recommend', code, ...(typeof trace === 'string' && /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(trace) ? { trace_id: trace } : {}) });
-}
+const frameFiles = Object.freeze(['cli.mjs', 'install.mjs']);
+const applicationURLs = frameFiles.map(file => new URL(`../../terminal/src/${file}`, import.meta.url).href);
 export function safeFrames(error) {
   try {
     const stack = error?.stack;
@@ -24,7 +20,7 @@ export function safeFrames(error) {
   } catch { return []; }
 }
 export function errorProperties(error) {
-  try { return { error_type: errorType(error), frames: safeFrames(error), ...(diagnostics.get(error) ?? {}) }; }
+  try { return { error_type: errorType(error), frames: safeFrames(error) }; }
   catch { return { error_type: 'Error', frames: [] }; }
 }
 const environment = () => ({ ...(clientVersion ? { client_version: clientVersion } : {}), platform: ['darwin', 'linux', 'win32'].includes(process.platform) ? process.platform : 'unknown', runtime_major: Number(process.versions.node.split('.')[0]) });
@@ -99,65 +95,17 @@ export async function sendInstallationEvent(input) {
   } catch { /* Never affect setup, even if the transport ignores abort. */ }
   finally { clearTimeout(timer); }
 }
-export const errorType = error => ['Error', 'TypeError', 'RangeError', 'SyntaxError', 'ReferenceError', 'URIError', 'EvalError', 'AggregateError', 'AbortError'].includes(error?.name) ? error.name : 'Error';
-export function terminalProperties(event, input) {
-  if (!input || typeof input !== 'object' || Array.isArray(input)) return null;
-  const out = {};
-  const select = (key, values) => { if (!values.includes(input[key])) return false; out[key] = input[key]; return true; };
-  if (event === 'terminal_error') {
-    if (!select('code', ['collection_error', 'snapshot_error', 'render_error', 'uncaught_error']) || !select('error_type', ['Error', 'TypeError', 'RangeError', 'SyntaxError', 'ReferenceError', 'URIError', 'EvalError', 'AggregateError', 'AbortError'])) return null;
-    if (Array.isArray(input.frames)) out.frames = input.frames.slice(0, 8).filter(f => f && frameFiles.includes(f.filename) && Number.isInteger(f.lineno) && f.lineno > 0 && f.lineno <= 1000000 && Number.isInteger(f.colno) && f.colno > 0 && f.colno <= 1000000).map(f => ({ filename: f.filename, lineno: f.lineno, colno: f.colno }));
-    return out;
-  }
-  if (!select('mode', ['sample', 'live'])) return null;
-  if (event === 'terminal_visited') return out;
-  if (event === 'terminal_page_viewed') return select('page', ['overview', 'models', 'services', 'activity', 'compute', 'sources', 'connections']) ? out : null;
-  if (event === 'terminal_interaction') return select('action', ['inspect', 'filter', 'search', 'map_expand', 'mode', 'command']) && select('kind', ['model', 'agent', 'task', 'service', 'source', 'host', 'deployment', 'pool', 'record', 'path', 'other']) ? out : null;
-  if (event !== 'terminal_snapshot') return null;
-  for (const key of ['model_count', 'record_count', 'source_count', 'host_count', 'deployment_count', 'agent_count']) {
-    if (!Number.isInteger(input[key]) || input[key] < 0 || input[key] > 1000000000) return null;
-    out[key] = input[key];
-  }
-  return out;
-}
-// Shared, process-wide budget; no queue and at most four in-flight requests.
-let windowStart = 0, count = 0, active = 0;
-export function sendTerminalEvent(event, input) {
-  if (!telemetryEnabled()) return;
-  const properties = terminalProperties(event, input);
-  if (!properties) return;
-  if (Date.now() - windowStart >= 60000) { windowStart = Date.now(); count = 0; }
-  if (count >= 60 || active >= 4) return;
-  count++; active++;
-  void sendAnalytics(event, properties).finally(() => active--);
-}
-async function sendAnalytics(event, properties) {
-  const controller = new AbortController();
-  const body = JSON.stringify({ event, event_id: randomUUID(), anonymousId, properties: { ...properties, ...environment() } });
-  let timer;
-  try {
-    await Promise.race([
-      Promise.resolve().then(() => fetch('https://openmerit.ai/scout/api/analytics', {
-        method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Scout-Client': 'cli', 'X-Scout-Analytics': 'on', ...(process.env.OPENMERIT_ANALYTICS_ACCOUNT === 'off' ? { 'X-Scout-Analytics-Account': 'off' } : {}) },
-        body, signal: controller.signal, redirect: 'error',
-      })).then(response => response.body?.cancel()),
-      new Promise(resolve => { timer = setTimeout(() => { controller.abort(); resolve(); }, 750); timer.unref(); }),
-    ]);
-  } catch { /* Never affect product behavior. */ }
-  finally { clearTimeout(timer); }
-}
-
-// Consent is independent of terminal discovery. Command identity is ephemeral;
-// the installer separately manages the owned installation's correlation IDs.
+const errorType = error => ['Error', 'TypeError', 'RangeError', 'SyntaxError', 'ReferenceError', 'URIError', 'EvalError', 'AggregateError', 'AbortError'].includes(error?.name) ? error.name : 'Error';
+// Command identity is ephemeral; the installer separately manages the owned
+// installation's correlation IDs.
 export function commandTelemetry(args) {
   const setup = ['install', 'status', 'remove', 'login', 'logout', 'project', 'doctor'];
   const enabled = () => telemetryEnabled(setup.includes(args[0]));
   if (!enabled()) return async () => {};
   const started = performance.now();
   const command_id = randomUUID();
-  const actions = ['init', 'doctor', 'plan', 'run', 'watch', 'report'];
-  const command = !args.length || args.includes('--help') || args.includes('-h') || args[0] === 'eval' && args.length === 1
-    ? 'help' : args[0] === 'dash' ? 'dash' : setup.includes(args[0]) ? args[0] : args[0] === 'eval' && actions.includes(args[1]) ? `eval_${args[1]}` : 'unknown';
+  const command = !args.length || args.includes('--help') || args.includes('-h')
+    ? 'help' : setup.includes(args[0]) ? args[0] : 'unknown';
   async function send(event, properties) {
     if (!enabled()) return;
     const controller = new AbortController();

package/docs/agent-evaluations.md

@@ -1,155 +1,15 @@
-# Evaluate CLI changes across coding agents
+# Agent-evaluation CLI retirement
 
-**Stable npm `0.1.7`.** Install from the `latest` tag. This evaluates the coding agent with your CLI and documentation. The [Pi workflow](getting-started.md) still evaluates application inference; the [Inference Terminal](inference-terminal.md) remains read-only. Scout's hosted API and MCP service deploy independently; npm installation does not start or bundle the hourly research service.
+**npm 0.3.0, 4 November 2026.** The standalone `openmerit eval` CLI is retired in full, including `init`, `doctor`, `plan`, `run`, `watch` and `report`. Older installed packages keep it until they are upgraded. The old Scout JSON recommendation API is retired in unreleased source and goes offline only with a separate Scout deployment.
 
-Compared with `0.1.7-preview.1`, this version adds [corrected cost accounting and blocked-execution handling](#accounting-and-blocked-executions), plus [budget selection after model scope](#budget-selection). It does not claim reconciled provider billing or a guaranteed exact model order.
+After an approved Scout release, `/scout/api/recommend` and its legacy alpha/versioned paths return 404, regardless of authentication. There is no replacement frozen-plan protocol. Use the [Scout conversational agent or MCP](scout.md) for provisional model guidance and your own representative task checks for evaluation. The Inference Terminal is [retired as well](inference-terminal.md); the Pi adapter remains a native Scout MCP integration, not an evaluation runner.
 
-One local runner supports Claude Code, Codex and OpenCode. Scout supplies candidates; local task checks measure each model/agent/CLI combination. Installing npm alone does not install or authenticate agents, modify project instructions, or start paid evaluations.
+## Existing local artifacts
 
-## Optional CLI product analytics
+Upgrading does not delete `.openmerit/eval.json`, reports, budget ledgers, task fixtures or grader files. Review and retain or remove them yourself. Stop any old foreground watcher before upgrading. Instructions previously appended to `AGENTS.md` or `CLAUDE.md` are not automatically rewritten; remove obsolete evaluation instructions deliberately, preserving unrelated project guidance.
 
-Npm 0.1.7 supports the [opt-in CLI analytics contract](inference-terminal.md#optional-cli-product-analytics). Only `OPENMERIT_TELEMETRY=1` enables command start/completion events; `DO_NOT_TRACK=1` overrides it. A process-only random ID accompanies command categories, completion outcomes and bounded durations, never task descriptions, model IDs, arguments, paths, error text or configuration. No project key is distributed. Transport failures cannot change JSON output or exit status.
+Historical reports remain evidence only for the versions and tasks actually tested. Unknown costs do not become known, and past budget reservations are not reconciled provider bills. Retirement does not cancel external agent/provider work, erase historical Scout analytics, change provider retention or enforce billing limits.
 
-`eval watch` completes only when its loop returns, not after each iteration; forced exits can leave a start without a completion. An `ok` command outcome is not evidence of model quality or reconciled charges. Direct imports of the evaluation module do not emit CLI lifecycle events.
+## Preserved capabilities
 
-**Instrumentation in 0.1.7:** opt-in diagnostics additionally include sanitized package version, platform/runtime categories, random command correlation and up to eight allowlisted application basename/line/column frames. Scout failure categories and validated request traces can correlate planning failures without raw error text, task content or absolute paths. See the [full CLI contract](inference-terminal.md#optional-cli-product-analytics). Prompt-content/retention controls are unchanged. Negative feedback may inform only [reviewed sanitized evaluation cases](site-scout-architecture.md#reviewed-feedback-and-future-experiments), never automatic private-content copying or expanded retention. Future controlled ranking experiments require an offline baseline first; no experiment is active.
-
-Scout planning requests carry `X-Scout-Client: cli` and `X-Scout-Analytics: off` unless explicitly opted in without Do Not Track. They always include `X-Scout-Analytics-Content: off`, so task descriptions required for planning are not opted into richer product-analytics content collection even when command analytics is enabled. The server owns search-completion counts; the CLI does not emit another search event. This opt-out affects optional PostHog product events, not task processing or historical private Cloudflare prompt records. Planning still sends the task descriptions and model scope required by its protocol. Evaluation planning requires a current Clerk session token; the separate device key from coding-agent setup grants MCP recommendations only, not JSON API access.
-
-## Set up
-
-Install the stable release and initialize it inside the project you want to evaluate:
-
-```sh
-npm install -g openmerit@latest
-openmerit eval init
-openmerit eval doctor
-```
-
-Init creates `.openmerit/eval.json`, a tiny extraction fixture, a tool and a standalone grader. It appends marked instructions to `AGENTS.md` and `CLAUDE.md`, preserving existing contents. Codex/OpenCode can read `AGENTS.md`; Claude Code reads `CLAUDE.md`. The instructions ask the agent to configure representative tasks, choose scope, obtain the user's budget, and rerun after CLI changes. These are instructions the host must follow, not guaranteed hooks. There are no install-time hooks or background services. A second init refuses to overwrite the suite.
-
-Replace the starter before drawing product conclusions. Have your agent author representative fixtures and graders with you, and reserve fresh cases for final validation.
-
-## Choose scope and connections
-
-```sh
-openmerit eval plan
-```
-
-The first interactive plan asks: **existing project connections or all models Scout indexes?** It then asks for a **$10, custom or no-fixed-cap budget**. Both choices are saved. Noninteractive agents must ask the user and pass `--scope existing` or `--scope all` together with `--budget <USD|unlimited>` on first setup; missing input never chooses scope or spending automatically.
-
-Add explicit routes to `connections` in the config, for example:
-
-```json
-{
-  "id": "claude-project",
-  "agent": "claude",
-  "scoutId": "anthropic/claude-haiku-4.5",
-  "model": "claude-haiku-4-5-20251001",
-  "env": ["ANTHROPIC_API_KEY"]
-}
-```
-
-`scoutId` is Scout's catalogue ID. `model` is the native identifier your authenticated agent accepts. `command` optionally selects another executable path. `env` lists environment variable **names**, never values. Only basic OS variables and explicitly listed names pass to child agents. Saved authentication remains available through the normal home directory. No dotenv files are loaded automatically. Configure endpoint details through the native agent or named environment variables.
-
-An SDK dependency or key-variable name does not prove access. The agent should inspect the project's intended connections and declare the models they can serve. This preview does not automatically enumerate endpoint catalogues. Multiple connections can test one Scout model through different agents/routes. Prefer pinned model versions; aliases can change.
-
-Existing scope sends IDs and requires Scout to acknowledge filtering **before ranking five**. Fewer than five eligible models remain fewer. All scope searches the index, but missing authenticated routes block execution until configured and replanned. Inaccessible candidates are not silently replaced. Unranked Hugging Face specialists are not executed automatically.
-
-Scout receives only each task's `description`, plus IDs for existing scope. Scout sends descriptions to OpenAI Decisions. Fixtures, execution prompts, endpoint URLs and credentials stay out of this request. The shortlist, warnings and ranking context are frozen locally. Scout scores are screening evidence, not measured success with your harness.
-
-## Tasks and variants
-
-Each task needs `id`, `description` (for Scout, at most 4,000 characters), `prompt` (for execution), `fixture` (a project-relative directory) and `grader` (a standalone Node `.mjs` file).
-
-Each variant needs `id`, `files` and `instructions`. The starter compares `bare` against `cli`. Bare omits your CLI/docs; native agent tools remain available. Across agents you are comparing different harnesses, with different permission systems. A no-native-tools experiment needs a separately controlled configuration.
-
-Selected files, including uncommitted changes, are snapshotted before execution. Variant paths must not overwrite fixture inputs and retain their project-relative locations; fixture contents go at the workspace root. Include self-contained CLI artifacts and lean docs explicitly. Dependencies are not installed automatically; tools must be bundled or have their runtime requirements already available. Symlinks, parent-directory paths, `.env*`, `.git`, `.openmerit` and `node_modules` are rejected. Each selected snapshot is limited to 25 MB. Do not select credentials stored under other filenames.
-
-The grader reads `{ "workspace": "/temporary/path", "output": "final response" }` from stdin and prints exactly `{ "quality": 0.0 }`, with a finite score from 0 to 1. It can check generated files using executable assertions. Keep it self-contained: imports beside it are not copied automatically. Graders are copied outside agent workspaces; never also include them or answer keys in fixtures/variant files. Test graders against correct and incorrect answers. A broken grader stops the experiment with unknown quality. Ordinary agent failures/timeouts count as quality zero; recognized budget, environment and connection blocks instead stop with unknown quality. See [failure classification](#accounting-and-blocked-executions).
-
-## Run after changes
-
-The starter's `budget.maxUsd` is zero. After the user's scope and budget choices, for example:
-
-```sh
-openmerit eval plan --scope existing --budget 10
-openmerit eval run
-# Edit your CLI or its documentation.
-openmerit eval run
-```
-
-For repeated local development:
-
-```sh
-openmerit eval watch
-```
-
-Watch runs initially, then after selected source/docs/fixture/grader/config bytes change and remain stable for two polls. Result writes do not trigger it. Runs never overlap; edits during execution become the next revision. Ctrl+C stops the watcher and active child process. No OS schedule or daemon is installed. `OPENMERIT_EVAL=1` tells cooperating agent instructions not to recurse into another evaluation.
-
-Every attempt uses a new workspace and agent session. Each task runs balanced rounds across all selected connections and variants; order rotates and reverses. The default is 3–10 repetitions. After three, rounds continue while any group's quality range exceeds 5 percentage points, an attempt failed, or cost, input-token or latency variation has a coefficient of variation above 0.25. Ten is the maximum. Missing metrics cannot establish stability. Set `repeats.min` equal to `repeats.max` for fixed repetitions, useful for holdouts. This heuristic is not a significance test, and repetition is not task diversity.
-
-The frozen plan survives CLI changes. Changes to descriptions, scope or connections require replanning. Previous-revision comparisons require matching prompts, fixtures, graders, routes, repeat settings, selected candidates, variant IDs, timeouts, agent versions, runner versions, invocation settings and reported model identities. CLI/docs bytes and variant instructions are the interventions. Keep external services and native agent settings stable too; they are not hermetically captured.
-
-## Metrics
-
-```sh
-openmerit eval report
-openmerit eval report --json
-```
-
-Reports show quality mean/range, failures, mean input/output tokens, mean reported dollars and median full-process seconds per task/connection/variant. They retain Scout ranks but do not force a weighted winner. Lower cost with lower quality is a tradeoff. Matching complete revisions show deltas; partial/invalid experiments do not.
-
-Claude provides aggregate and per-model usage and cost estimates, reconciled without counting the same usage twice. Codex JSON events provide tokens but not billed dollars, so cost stays unknown. OpenCode per-step usage/cost is summed across the episode, including tools; missing or zero step pricing leaves full cost unknown and preserves partial estimates. Input totals include separately reported cache reads/writes; cached tokens are retained separately without counting them twice. OpenCode output totals include separately reported reasoning tokens. Agent accounting and subscriptions differ. Reported costs are not reconciled provider bills; zero pricing is treated as unknown. Catalogue prices never replace unknown measured costs.
-
-Preflight requires supported noninteractive JSON interfaces: Claude print JSON, Codex `exec --json --ephemeral --sandbox workspace-write`, and OpenCode `run --format json --auto`. OpenCode's auto option approves tool requests that are not explicitly denied, avoiding an interactive approval stall. Claude uses an explicit local tool allowlist without inherited MCP configuration; Codex and OpenCode retain native global settings. Recognized permission failures stop as connection problems, not model-quality evidence. OpenMerit does not add a sandbox around Claude/OpenCode. The old 2025 Codex CLI is unsupported. Native references: [Claude Code](https://code.claude.com/docs/en/headless), [Codex](https://developers.openai.com/codex/noninteractive), [OpenCode](https://opencode.ai/docs/cli/).
-
-## Budget and data
-
-`budget.maxUsd` applies cumulatively across runs and restarts. Before each attempt the runner persists a reservation. It retains the greater of that reservation and the available full or partial cost estimate; estimates never release reserved allowance. Partial events never count as a complete bill. Insufficient remaining allowance stops a partial run. Raising maxUsd extends the allowance, not resets it. Do not delete unknown spend from the ledger. This is admission control, not a billing cutoff: one attempt may exceed its reservation. Finite budgets also use Claude's native maximum-budget flag. Use provider-side limits for hard monetary caps. Each attempt also has a timeout and 4 MB process-output limit. See [budget selection](#budget-selection) for the three choices and automatic reservations.
-
-`.openmerit/eval/` holds `plan.json`, `budget.json`, `latest.json`, `runs/*.json` and `lock/pid`. Init ignores it in Git. Reports contain identities, metrics, hashes and status, not full transcripts or final outputs. Temporary files are removed afterward. Native agents/providers may retain their own data; Claude/Codex receive ephemeral flags, while OpenCode uses native storage. `.openmerit/eval.json` can be versioned without credentials.
-
-Fresh workspaces are not security sandboxes. Trusted agents and graders execute with native permissions and user-level resources. Hard kills may leave temporary directories. After a crash, inspect the lock's PID and remove it only if that process is gone. This preview does not change Pi approvals, swap production models or upload results into Scout rankings.
-
-
-## VM acceptance limits
-
-The published `0.1.7-preview.1` package was tested on an Ubuntu ARM64 VM with Node 22.23.1 on 15 October 2026. Claude Code 2.1.300 and OpenCode 1.19.0 executed a small CSV CLI task before and after a parser change. A separate OpenCode watcher completed three attempts per revision and emitted comparison deltas. Codex 0.162.1 passed interface preflight but could not execute filesystem tools because that VM denied the required Linux user namespaces. CLI flag checks do not establish that the native sandbox works on a host.
-
-In that earlier `0.1.7-preview.1` test, a native budget stop could appear as a generic agent failure, and a successful final response explaining a host permission problem could receive zero from the task grader. These did not establish poor model quality; the corrections below address recognized native failures. The combined VM run also stopped before its final repetition because conservative reservations and agent cost estimates exhausted its local allowance; partial reports correctly omitted automatic comparison. Agent-reported cost differed substantially from OpenRouter account usage. This was one integration fixture, not validation of general task quality, exact model order, or billing accuracy.
-
-## Accounting and blocked executions
-
-Since `0.1.7-preview.2`, recognized budget, sandbox, permission, authentication and model-access blocks are separate from task failures. A blocked attempt has `quality: null`, stops the run and cannot produce a revision comparison. Reports distinguish scored attempts, blocked attempts and task-check failures. Ordinary wrong outputs still receive their grader score; ordinary agent failures and configured timeouts still count as failures. Cancellation, a missing executable and output-limit interruption are unscored. Error classification uses native failure fields and failed-tool diagnostics, not arbitrary task text or model commentary. Unrecognized host failures can still require manual review.
-
-On Linux, Codex preflight also runs a local sandbox probe before any model request or budget reservation. It uses the configured executable and the workspace-write policy; it does not weaken sandbox permissions. A failed probe produces `environment_blocked` with no task score. Doctor performs this same probe. Other operating systems still have interface checks only.
-
-Claude's per-model totals recover usage omitted from aggregate budget-stop responses, without adding the same aggregate and model counters twice. Costs carry `costBasis` and remain native estimates. Zero or missing pricing is unknown; incomplete OpenCode step pricing stays partial. Partial costs survive into the row, summary and ledger, including failed attempts. No provider bill is reconstructed from these estimates, and no account-wide spend delta is assigned to individual attempts.
-
-Budget admission now retains the greater of each reservation and its available full or partial estimate, including old ledger entries. Estimates never release a reservation. This may exhaust an existing allowance earlier; increasing it requires an explicit budget decision. The displayed reserved allowance is not billed spend, and a provider-side limit is still needed for a hard cap. Report version 2 prevents comparisons with the earlier accounting rules. There is no new provider integration, proxy, background service or model-specific rule.
-
-## Budget selection
-
-After choosing existing connections or all indexed models, the first interactive `eval plan` asks:
-
-1. **$10** — a cumulative evaluation allowance for this project.
-2. **Custom amount** — asks how much in USD.
-3. **No fixed cap** — run the configured suite without a monetary admission limit.
-
-Nothing is preselected. Blank, invalid or cancelled input cannot enable spending. Init still starts at zero. Newly generated project instructions tell coding agents to ask the same follow-up and pass the answer explicitly, for example:
-
-```sh
-openmerit eval plan --scope existing --budget 10
-# Alternatives after the user's choice:
-openmerit eval plan --budget 25.50
-openmerit eval plan --budget unlimited
-```
-
-`--budget` belongs to `plan`, accepts positive USD amounts with at most two decimal places or the exact word `unlimited`, and never starts evaluation by itself. Noninteractive planning requires explicit flags when scope or budget has not been chosen. Existing positive budgets are reused. The choices are saved even if Scout subsequently fails, so a retry does not ask again. Init does not rewrite instructions in already-initialized projects; an agent in such a project can use these flags directly.
-
-The allowance is cumulative across runs, watched edits, restarts and replanning. Changing from $10 to $20 raises the total allowance to $20; it does not add $20 or erase earlier reservations. Changing to or from no fixed cap also preserves the ledger. A normal `eval plan` reuses the saved choice. To change it, pass a new `--budget` value; setting `budget.maxUsd` back to zero in the config disables paid execution and prompts again on the next interactive plan.
-
-Reservations are internal. New suites and budget choices use `reservationUsd: "auto"`: the smaller of $0.25 and the total allowance divided across the minimum planned attempts (tasks × selected connections × variants × minimum repetitions). No fixed cap uses $0.25 for ledger bookkeeping. This is a conservative starting allowance, not a prediction of an attempt's price or a guarantee the suite will finish. Higher reported charges still consume more allowance. Existing explicitly configured numeric reservations remain supported. Finite budgets retain Claude's native per-attempt budget flag; as before, an attempt can exceed its reservation, and provider limits are needed for a hard billing cap.
-
-No fixed cap is stored explicitly as `budget.maxUsd: "unlimited"`; missing, null or invalid values never mean unlimited. It also removes Claude's per-attempt dollar limit, while keeping the same task count, 3–10 repetition bounds, timeouts and native permissions. Provider/account limits still apply. Usage and cost estimates remain recorded. `eval watch` can incur further charges after each selected change until stopped; Ctrl+C stops it and the active attempt. No background service or automatic production model change is added.
+Scout's agent, MCP, coding-agent setup, catalogue refresh/publication, status and account/privacy controls remain. Shared evidence preparation and model projections remain available to those consumers. See [commands](commands.md), [security](security.md) and [testing](testing.md) for the remaining interfaces and verification boundary.

package/docs/commands.md

@@ -1,91 +1,9 @@
 # Commands
 
-**Pi replacement — unreleased, unpublished source:** Pi 1.5.0 uses only native Scout MCP and shared guidance. Run `npx -y openmerit@latest install --client pi [--scope user|project]` after publication and approve in the browser. Shell `login`, `logout`, `status`, `remove`, and `doctor` accept the same target flags. Updates require explicit reinstall; no Pi update hooks or scheduler are installed. See [Pi setup](pi-extension.md).
+**Pi adapter — npm 0.1.12 and later:** Pi 1.5.0 uses only native Scout MCP and shared guidance. Run `npx -y openmerit@latest install --client pi [--scope user|project]` and approve in the browser. Shell `login`, `logout`, `status`, `remove`, and `doctor` accept the same target flags. Updates require explicit reinstall; no Pi update hooks or scheduler are installed. See [Pi setup](pi-extension.md).
 
-The standalone `openmerit dash` and `openmerit eval` commands are unchanged. All `/openmerit` slash commands below are **historical Pi 0.87 references**, not commands in the new Scout adapter. Core lifecycle APIs remain available for custom harnesses.
+The Pi 0.87 `/openmerit` slash commands were removed with that adapter and have no replacement.
 
-Stable npm `openmerit@0.1.7` provides `openmerit eval init`, `doctor`, `plan`, `run`, `watch` and `report`. See [agent evaluations](agent-evaluations.md) for scope selection, repeated task checks and budgets. These are shell commands, separate from Pi commands. The same release includes [Scout setup commands](scout.md#commands), beginning with `npx -y openmerit@latest install --client claude-code`.
+**Retired in npm 0.3.0:** `openmerit dash` ([details](inference-terminal.md)) and all `openmerit eval` commands, including `plan`. Existing local files are untouched. Unreleased Scout source also retires `/scout/api/recommend`; use Scout's conversational agent or MCP. Hosted Workers change only through separate approved deployments. See [retirement details](agent-evaluations.md).
 
-Since `0.1.7-preview.2`, `openmerit eval plan [--scope existing|all] [--budget <USD|unlimited>]` asks for a budget after scope. Choose $10, a custom amount, or no fixed cap; saved choices and cumulative usage persist. See [budget selection](agent-evaluations.md#budget-selection).
-
-## Inference Terminal
-
-Install with `npm install -g openmerit` to use `openmerit dash [path]` for a passive local view of models, connections, usage, and compute. From this checkout, use `npm run dash -- [path]`. It is independent of Pi setup and does not run model comparisons or inference.
-
-See the [Inference Terminal guide](inference-terminal.md) for local installation, automatic discovery, `--no-discovery`, `--json`, `--demo`, collection intervals, and data limitations. Discovery is part of `dash` and is scoped to application evidence. Coding-harness histories are not collected.
-
-## Inspect status
-
-```text
-/openmerit
-/openmerit status
-```
-
-Both show the task profile, metric readiness, evaluation status, active work, next assessment, lifecycle stage, incumbent candidate, candidate and assessment counts, challenger spend, proposed candidate, and pause flag. A readiness label is based on the harness’s structured reports; inspect the evidence for the underlying measurements.
-
-## Start or recover setup
-
-```text
-/openmerit setup
-```
-
-Requests setup explicitly or reopens setup to correct stale target/incumbent state. Accepted replacement setup clears prior candidate, assessment, frontier, and baseline handoffs while retaining the audit log.
-
-Request setup explicitly only when recovering, rerunning, or configuring a custom integration. An unconfigured project starts setup when bounded source detection finds an application LLM call at session start or after a settled Pi build turn. OpenMerit refuses another intent while work is active.
-
-## Cancel interrupted work
-
-```text
-/openmerit cancel
-```
-
-Record the active intent as cancelled after an interrupted or abandoned harness run. This clears durable active-work state without claiming success and releases any interactive request that was queued behind setup.
-
-## Request an assessment
-
-```text
-/openmerit assess
-```
-
-Ask OpenMerit core to check stored baseline metric windows now. It reports missing or immature required metrics without starting Pi, or starts candidate discovery when they pass. It does not create missing measurements or override sample requirements.
-
-## Request a frontier
-
-```text
-/openmerit frontier
-```
-
-Ask the harness to calculate a model comparison now. OpenMerit validates the returned frontier against the confirmed profile and supplied assessments. The command does not itself authorize a model change.
-
-## Approve a proposal
-
-```text
-/openmerit approve
-```
-
-Approve the current proposal after its frontier result has passed validation. If there is no proposal in the expected lifecycle stage, the command reports that no verified proposal is awaiting approval. See [comparisons and swaps](lifecycle.md) for automatic-swap policy.
-
-## Pause and resume
-
-```text
-/openmerit pause
-/openmerit resume
-```
-
-Pause is persisted in `.openmerit/state.json`, blocks automatic due decisions and lifecycle follow-ups, and unloads the project's managed macOS LaunchAgent. Incoming signals can still be recorded. Resume clears pause and restores scheduling only when the confirmed policy authorizes it. Neither command cancels an already-active intent; explicit commands remain available. The standalone Inference Terminal is independent and keeps collecting until its CLI process exits.
-
-## Locate logs and evidence
-
-```text
-/openmerit logs
-```
-
-Show the local audit, exporter-error, evidence-manifest, and state paths. See [project files and logs](operations.md) for their contents.
-
-## Check the installation
-
-```text
-/openmerit doctor
-```
-
-Show adapter diagnostics, UI availability, and scheduling support. This checks extension wiring; it does not prove provider authentication, successful evaluations, or a completed model swap.
+The [Scout setup commands](scout.md#commands) remain available, beginning with `npx -y openmerit@latest install --client claude-code`.

package/docs/getting-started.md

@@ -1,20 +1,6 @@
 # Getting started
 
-OpenMerit provides a passive Inference Terminal, Scout model-selection guidance, and a separate [agent-evaluation CLI](agent-evaluations.md). Pi Scout setup requires OpenMerit **0.1.12** or newer and Pi **1.5.0**. Package publication and website deployment are independent releases.
-
-## Open your inference environment
-
-With Node.js 22.19 or newer and npm installed, run:
-
-```sh
-npm install -g openmerit
-cd /path/to/your/application
-openmerit dash
-```
-
-The Inference Terminal opens a local, read-only view of the selected application's services, models, recorded requests, and linked compute. It does not require Pi or a provider API key, make inference calls, or change model assignments. Supported code/configuration declarations identify inventory; request history requires existing supported logs or exports. See [the terminal guide](inference-terminal.md) for collection, privacy, configuration, and diagnostics.
-
-To inspect fictional data without installation, open the [hosted dashboard](https://dash.openmerit.ai/) or its [before/after comparison](https://dash.openmerit.ai/demo). Neither connects to your machine.
+OpenMerit provides Scout model-selection guidance. The Inference Terminal (`openmerit dash`) is [retired in npm 0.3.0](inference-terminal.md). Pi Scout setup requires OpenMerit **0.1.12** or newer and Pi **1.5.0**. Package publication and website deployment are independent releases.
 
 ## Connect Scout to a coding agent
 
@@ -48,4 +34,4 @@ See [the Pi extension guide](pi-extension.md) for installed paths, migration fro
 
 ## Evaluate application changes
 
-The standalone `openmerit eval` CLI is unchanged by the new Pi adapter. It evaluates Claude Code, Codex and OpenCode through explicit task, scope, budget and grader configuration; see [agent evaluations](agent-evaluations.md). The core evaluation coordinator remains available to custom harness integrations, but the removed Pi 0.87 lifecycle adapter is not the way to invoke it.
+The standalone evaluation CLI is retired in npm 0.3.0, and the legacy JSON recommendation API in unreleased Scout source. Scout guidance is not measured application quality; use your own task checks and provider budget controls. See [retirement details](agent-evaluations.md).

package/docs/index.md

@@ -1,24 +1,10 @@
 # OpenMerit documentation
 
-These guides distinguish current source work from published behavior. OpenMerit **0.1.12** adds native Scout support for Pi **1.5.0**, replacing the old Pi 0.87 evaluation extension. The Inference Terminal and standalone agent-evaluation CLI are separate products and are unchanged by that replacement. The website, hosted samples, Scout service and npm package are deployed independently; dated receipts describe the versions actually tested.
-
-## See your inference environment
-
-With Node.js 22.19 or newer:
-
-```sh
-npm install -g openmerit
-cd /path/to/your/application
-openmerit dash
-```
-
-The terminal shows application services, model declarations, recorded requests, and linked compute. It reads supported local evidence, needs no Pi installation or provider API key, and makes no inference calls or model changes. Configuration can identify declared services; actual activity requires existing supported logs or exports.
-
-Start with [installation and setup](getting-started.md) or the [Inference Terminal guide](inference-terminal.md). Explore the [hosted sample](https://dash.openmerit.ai/) and [before/after dashboard demo](https://dash.openmerit.ai/demo) without connecting your machine. The [demo guide](demo.md) explains their fictional data and comparison limits.
+These maintained Markdown guides distinguish current source work from published behavior. OpenMerit **0.1.12** added native Scout support for Pi **1.5.0**, replacing the old Pi 0.87 evaluation extension. OpenMerit **0.3.0** retires the Inference Terminal (`openmerit dash`) and standalone evaluation CLI. Unreleased source also retires the legacy recommendation API and the unpublished docs website shell. The policy, protected memo, Scout service and npm package are deployed independently; dated receipts describe the versions actually tested.
 
 ## Discover model candidates
 
-For a shortlist before building an integration, the separate [Scout beta](scout.md) accepts a task description and returns up to five evidence-backed model candidates. It is available at [openmerit.ai/scout/](https://openmerit.ai/scout/), with a prompt-only JSON API and MCP tool for coding agents.
+Start with [installation and setup](getting-started.md). The [Scout beta](scout.md) provides conversational application-model guidance at [openmerit.ai/scout/](https://openmerit.ai/scout/) and through MCP for coding agents. The legacy prompt-only JSON recommendation endpoint is retired in source.
 
 ## Connect Scout to Pi
 
@@ -28,17 +14,8 @@ Follow [setup](getting-started.md#connect-scout-to-a-coding-agent) and [the Pi g
 
 ## Evaluate application changes
 
-Use [the standalone evaluation CLI](agent-evaluations.md) for Claude Code, Codex and OpenCode. Custom harnesses may use the core coordinator. The following guides retain core concepts and historical Pi 0.87 workflows; they do not describe the new Scout adapter:
-
-- [Metrics and evidence](metrics-and-evidence.md): measurements, sample requirements, and readiness.
-- [Comparisons and swaps](lifecycle.md): baseline, challenger trials, approval, verification, and rollback.
-- [Budgets and permissions](budgets.md): challenger accounting and provider-spend limits.
-- [Automatic checks](automation.md): durable signals, cadence, and authorized macOS scheduling.
+Use your own representative task checks to evaluate model candidates. The [standalone evaluation CLI is retired in npm 0.3.0](agent-evaluations.md); existing local artifacts are not deleted.
 
 ## Operate and troubleshoot
 
-[Troubleshooting](troubleshooting.md), [security and data](security.md), and [current limitations](roadmap.md) explain current Pi setup and its boundaries. [Commands](commands.md) and [project files and logs](operations.md) also retain explicitly historical Pi lifecycle references alongside standalone/core information; their old pause and recovery commands are not part of the new adapter.
-
-## Build an integration
-
-Pi now supplies a native Scout connection, not an evaluation `HarnessAdapter`. Custom evaluation harnesses can implement the protocol and coordinator boundary described in [architecture](architecture.md) and [the adapter guide](adapter-guide.md). The [Pareto specification](pareto-spec.md) defines comparison rules; [testing and validation](testing.md) distinguishes automated coverage from product evidence.
+[Commands](commands.md) lists the Scout and setup commands. The [Inference Terminal is retired in npm 0.3.0](inference-terminal.md); existing local files are not deleted. [Troubleshooting](troubleshooting.md), [security and data](security.md), and [current limitations](roadmap.md) explain setup failures and product boundaries. [Testing and validation](testing.md) distinguishes automated coverage from product evidence.