openmerit 0.1.7-preview.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +7 -3
- package/dist/eval/src/scout.mjs +12 -1
- package/dist/terminal/public/app.js +33 -32
- package/dist/terminal/src/browser.mjs +7 -6
- package/dist/terminal/src/cli.mjs +10 -1
- package/dist/terminal/src/install.mjs +399 -0
- package/dist/terminal/src/server.mjs +28 -3
- package/dist/terminal/src/telemetry.mjs +118 -0
- package/docs/agent-evaluations.md +13 -3
- package/docs/commands.md +1 -1
- package/docs/getting-started.md +2 -2
- package/docs/index.md +1 -1
- package/docs/inference-terminal.md +19 -3
- package/docs/scout-data-policy.md +77 -0
- package/docs/scout.md +302 -30
- package/docs/security.md +38 -2
- package/docs/testing.md +6 -2
- package/package.json +6 -2
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
|
|
4
|
+
let clientVersion;
|
|
5
|
+
for (const path of ['../../../package.json', '../../../../package.json']) {
|
|
6
|
+
try { const pkg = JSON.parse(readFileSync(new URL(path, import.meta.url), 'utf8')); if (pkg.name === 'openmerit') { clientVersion = pkg.version; break; } } catch {}
|
|
7
|
+
}
|
|
8
|
+
export const frameFiles = Object.freeze(['cli.mjs', 'server.mjs', 'collect.mjs', 'repository-index.mjs', 'install.mjs', 'scout.mjs', 'config.mjs', 'runner.mjs', 'adapters.mjs', 'process.mjs', 'report.mjs', 'app.js']);
|
|
9
|
+
const applicationURLs = frameFiles.flatMap(file => ['terminal/src/', 'eval/src/'].map(dir => new URL(`../../${dir}${file}`, import.meta.url).href));
|
|
10
|
+
const diagnostics = new WeakMap();
|
|
11
|
+
export function scoutDiagnostic(error, trace, code) {
|
|
12
|
+
if (error && typeof error === 'object' && ['scout_network_error', 'scout_http_error', 'scout_response_error', 'scout_timeout'].includes(code)) diagnostics.set(error, { operation: 'scout_recommend', code, ...(typeof trace === 'string' && /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(trace) ? { trace_id: trace } : {}) });
|
|
13
|
+
}
|
|
14
|
+
export function safeFrames(error) {
|
|
15
|
+
try {
|
|
16
|
+
const stack = error?.stack;
|
|
17
|
+
if (typeof stack !== 'string') return [];
|
|
18
|
+
return stack.slice(0, 8000).split('\n').slice(1, 40).flatMap(line => {
|
|
19
|
+
const match = line.match(/(?:\(|\s)(file:\/\/[^\s)]+):(\d+):(\d+)\)?$/);
|
|
20
|
+
if (!match || !applicationURLs.includes(match[1])) return [];
|
|
21
|
+
const lineNumber = Number(match[2]), column = Number(match[3]);
|
|
22
|
+
return lineNumber > 0 && lineNumber <= 1000000 && column > 0 && column <= 1000000 ? [{ filename: match[1].split('/').at(-1), lineno: lineNumber, colno: column }] : [];
|
|
23
|
+
}).slice(0, 8);
|
|
24
|
+
} catch { return []; }
|
|
25
|
+
}
|
|
26
|
+
export function errorProperties(error) {
|
|
27
|
+
try { return { error_type: errorType(error), frames: safeFrames(error), ...(diagnostics.get(error) ?? {}) }; }
|
|
28
|
+
catch { return { error_type: 'Error', frames: [] }; }
|
|
29
|
+
}
|
|
30
|
+
const environment = () => ({ ...(clientVersion ? { client_version: clientVersion } : {}), platform: ['darwin', 'linux', 'win32'].includes(process.platform) ? process.platform : 'unknown', runtime_major: Number(process.versions.node.split('.')[0]) });
|
|
31
|
+
|
|
32
|
+
const anonymousId = randomUUID();
|
|
33
|
+
export const telemetryEnabled = () => process.env.OPENMERIT_TELEMETRY === '1' && process.env.DO_NOT_TRACK !== '1';
|
|
34
|
+
export const errorType = error => ['Error', 'TypeError', 'RangeError', 'SyntaxError', 'ReferenceError', 'URIError', 'EvalError', 'AggregateError', 'AbortError'].includes(error?.name) ? error.name : 'Error';
|
|
35
|
+
export function terminalProperties(event, input) {
|
|
36
|
+
if (!input || typeof input !== 'object' || Array.isArray(input)) return null;
|
|
37
|
+
const out = {};
|
|
38
|
+
const select = (key, values) => { if (!values.includes(input[key])) return false; out[key] = input[key]; return true; };
|
|
39
|
+
if (event === 'terminal_error') {
|
|
40
|
+
if (!select('code', ['collection_error', 'snapshot_error', 'render_error', 'uncaught_error']) || !select('error_type', ['Error', 'TypeError', 'RangeError', 'SyntaxError', 'ReferenceError', 'URIError', 'EvalError', 'AggregateError', 'AbortError'])) return null;
|
|
41
|
+
if (Array.isArray(input.frames)) out.frames = input.frames.slice(0, 8).filter(f => f && frameFiles.includes(f.filename) && Number.isInteger(f.lineno) && f.lineno > 0 && f.lineno <= 1000000 && Number.isInteger(f.colno) && f.colno > 0 && f.colno <= 1000000).map(f => ({ filename: f.filename, lineno: f.lineno, colno: f.colno }));
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
if (!select('mode', ['sample', 'live'])) return null;
|
|
45
|
+
if (event === 'terminal_visited') return out;
|
|
46
|
+
if (event === 'terminal_page_viewed') return select('page', ['overview', 'models', 'services', 'activity', 'compute', 'sources', 'connections']) ? out : null;
|
|
47
|
+
if (event === 'terminal_interaction') return select('action', ['inspect', 'filter', 'search', 'map_expand', 'mode', 'command']) && select('kind', ['model', 'agent', 'task', 'service', 'source', 'host', 'deployment', 'pool', 'record', 'path', 'other']) ? out : null;
|
|
48
|
+
if (event !== 'terminal_snapshot') return null;
|
|
49
|
+
for (const key of ['model_count', 'record_count', 'source_count', 'host_count', 'deployment_count', 'agent_count']) {
|
|
50
|
+
if (!Number.isInteger(input[key]) || input[key] < 0 || input[key] > 1000000000) return null;
|
|
51
|
+
out[key] = input[key];
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
// Shared, process-wide budget; no queue and at most four in-flight requests.
|
|
56
|
+
let windowStart = 0, count = 0, active = 0;
|
|
57
|
+
export function sendTerminalEvent(event, input) {
|
|
58
|
+
if (!telemetryEnabled()) return;
|
|
59
|
+
const properties = terminalProperties(event, input);
|
|
60
|
+
if (!properties) return;
|
|
61
|
+
if (Date.now() - windowStart >= 60000) { windowStart = Date.now(); count = 0; }
|
|
62
|
+
if (count >= 60 || active >= 4) return;
|
|
63
|
+
count++; active++;
|
|
64
|
+
void sendAnalytics(event, properties).finally(() => active--);
|
|
65
|
+
}
|
|
66
|
+
async function sendAnalytics(event, properties) {
|
|
67
|
+
const controller = new AbortController();
|
|
68
|
+
let timer;
|
|
69
|
+
try {
|
|
70
|
+
await Promise.race([
|
|
71
|
+
Promise.resolve().then(() => fetch('https://openmerit.ai/scout/api/analytics', {
|
|
72
|
+
method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Scout-Client': 'cli', 'X-Scout-Analytics': 'on' },
|
|
73
|
+
body: JSON.stringify({ event, anonymousId, properties: { ...properties, ...environment() } }), signal: controller.signal, redirect: 'error',
|
|
74
|
+
})).then(response => response.body?.cancel()),
|
|
75
|
+
new Promise(resolve => { timer = setTimeout(() => { controller.abort(); resolve(); }, 750); timer.unref(); }),
|
|
76
|
+
]);
|
|
77
|
+
} catch { /* Never affect product behavior. */ }
|
|
78
|
+
finally { clearTimeout(timer); }
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Consent is independent of terminal discovery. Never persist an identity.
|
|
82
|
+
export function commandTelemetry(args) {
|
|
83
|
+
if (process.env.OPENMERIT_TELEMETRY !== '1' || process.env.DO_NOT_TRACK === '1') return async () => {};
|
|
84
|
+
const started = performance.now();
|
|
85
|
+
const command_id = randomUUID();
|
|
86
|
+
const actions = ['init', 'doctor', 'plan', 'run', 'watch', 'report'];
|
|
87
|
+
const setup = ['install', 'status', 'remove', 'login', 'logout', 'project', 'doctor'];
|
|
88
|
+
const command = !args.length || args.includes('--help') || args.includes('-h') || args[0] === 'eval' && args.length === 1
|
|
89
|
+
? 'help' : args[0] === 'dash' ? 'dash' : setup.includes(args[0]) ? args[0] : args[0] === 'eval' && actions.includes(args[1]) ? `eval_${args[1]}` : 'unknown';
|
|
90
|
+
async function send(event, properties) {
|
|
91
|
+
if (!telemetryEnabled()) return;
|
|
92
|
+
const controller = new AbortController();
|
|
93
|
+
let timer;
|
|
94
|
+
try {
|
|
95
|
+
await Promise.race([
|
|
96
|
+
Promise.resolve().then(() => fetch('https://openmerit.ai/scout/api/analytics', {
|
|
97
|
+
method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Scout-Client': 'cli', 'X-Scout-Analytics': 'on' },
|
|
98
|
+
body: JSON.stringify({ event, anonymousId, properties: { ...properties, ...environment(), command_id } }),
|
|
99
|
+
signal: controller.signal, redirect: 'error',
|
|
100
|
+
})).then(response => response.body?.cancel()),
|
|
101
|
+
new Promise(resolve => { timer = setTimeout(() => { controller.abort(); resolve(); }, 750); }),
|
|
102
|
+
]);
|
|
103
|
+
} catch { /* Analytics must never affect command behavior. */ }
|
|
104
|
+
finally { clearTimeout(timer); }
|
|
105
|
+
}
|
|
106
|
+
const pending = send('cli_command_started', { command });
|
|
107
|
+
let completed = false;
|
|
108
|
+
// code: a content-free setup outcome such as approval_pending; never a message.
|
|
109
|
+
return async (outcome, error, code) => {
|
|
110
|
+
if (completed) return;
|
|
111
|
+
completed = true;
|
|
112
|
+
await Promise.all([pending, send('cli_command_completed', {
|
|
113
|
+
command, outcome: outcome === 'ok' ? 'ok' : 'error',
|
|
114
|
+
...(outcome === 'ok' ? {} : { code: code ?? 'command_error', ...(error ? errorProperties(error) : {}) }),
|
|
115
|
+
duration_ms: Math.min(86400000, Math.max(0, Math.round(performance.now() - started))),
|
|
116
|
+
})]);
|
|
117
|
+
};
|
|
118
|
+
}
|
|
@@ -1,17 +1,27 @@
|
|
|
1
1
|
# Evaluate CLI changes across coding agents
|
|
2
2
|
|
|
3
|
-
**npm
|
|
3
|
+
**Stable npm `0.1.7`.** Install from the `latest` tag. This evaluates the coding agent with your CLI and documentation. The [Pi workflow](getting-started.md) still evaluates application inference; the [Inference Terminal](inference-terminal.md) remains read-only. Scout's hosted API and MCP service deploy independently; npm installation does not start or bundle the hourly research service.
|
|
4
4
|
|
|
5
5
|
Compared with `0.1.7-preview.1`, this version adds [corrected cost accounting and blocked-execution handling](#accounting-and-blocked-executions), plus [budget selection after model scope](#budget-selection). It does not claim reconciled provider billing or a guaranteed exact model order.
|
|
6
6
|
|
|
7
7
|
One local runner supports Claude Code, Codex and OpenCode. Scout supplies candidates; local task checks measure each model/agent/CLI combination. Installing npm alone does not install or authenticate agents, modify project instructions, or start paid evaluations.
|
|
8
8
|
|
|
9
|
+
## Optional CLI product analytics
|
|
10
|
+
|
|
11
|
+
Npm 0.1.7 supports the [opt-in CLI analytics contract](inference-terminal.md#optional-cli-product-analytics). Only `OPENMERIT_TELEMETRY=1` enables command start/completion events; `DO_NOT_TRACK=1` overrides it. A process-only random ID accompanies command categories, completion outcomes and bounded durations, never task descriptions, model IDs, arguments, paths, error text or configuration. No project key is distributed. Transport failures cannot change JSON output or exit status.
|
|
12
|
+
|
|
13
|
+
`eval watch` completes only when its loop returns, not after each iteration; forced exits can leave a start without a completion. An `ok` command outcome is not evidence of model quality or reconciled charges. Direct imports of the evaluation module do not emit CLI lifecycle events.
|
|
14
|
+
|
|
15
|
+
**Instrumentation in 0.1.7:** opt-in diagnostics additionally include sanitized package version, platform/runtime categories, random command correlation and up to eight allowlisted application basename/line/column frames. Scout failure categories and validated request traces can correlate planning failures without raw error text, task content or absolute paths. See the [full CLI contract](inference-terminal.md#optional-cli-product-analytics). Prompt-content/retention controls are unchanged. Negative feedback may inform only [reviewed sanitized evaluation cases](site-scout-architecture.md#reviewed-feedback-and-future-experiments), never automatic private-content copying or expanded retention. Future controlled ranking experiments require an offline baseline first; no experiment is active.
|
|
16
|
+
|
|
17
|
+
Scout planning requests carry `X-Scout-Client: cli` and `X-Scout-Analytics: off` unless explicitly opted in without Do Not Track. They always include `X-Scout-Analytics-Content: off`, so task descriptions required for planning are not opted into richer product-analytics content collection even when command analytics is enabled. The server owns search-completion counts; the CLI does not emit another search event. This opt-out affects optional PostHog product events, not task processing or historical private Cloudflare prompt records. Planning still sends the task descriptions and model scope required by its protocol. Evaluation planning requires a current Clerk session token; the separate device key from coding-agent setup grants MCP recommendations only, not JSON API access.
|
|
18
|
+
|
|
9
19
|
## Set up
|
|
10
20
|
|
|
11
|
-
Install the
|
|
21
|
+
Install the stable release and initialize it inside the project you want to evaluate:
|
|
12
22
|
|
|
13
23
|
```sh
|
|
14
|
-
npm install -g openmerit@
|
|
24
|
+
npm install -g openmerit@latest
|
|
15
25
|
openmerit eval init
|
|
16
26
|
openmerit eval doctor
|
|
17
27
|
```
|
package/docs/commands.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The standalone `openmerit dash` command runs in your shell. The `/openmerit` slash commands run inside Pi with the extension loaded.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Stable npm `openmerit@0.1.7` provides `openmerit eval init`, `doctor`, `plan`, `run`, `watch` and `report`. See [agent evaluations](agent-evaluations.md) for scope selection, repeated task checks and budgets. These are shell commands, separate from Pi commands. The same release includes [Scout setup commands](scout.md#commands), beginning with `npx -y openmerit@latest install --client claude-code`.
|
|
6
6
|
|
|
7
7
|
Since `0.1.7-preview.2`, `openmerit eval plan [--scope existing|all] [--budget <USD|unlimited>]` asks for a budget after scope. Choose $10, a custom amount, or no fixed cap; saved choices and cumulative usage persist. See [budget selection](agent-evaluations.md#budget-selection).
|
|
8
8
|
|
package/docs/getting-started.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Getting started
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Stable npm 0.1.7 includes a separate [agent-evaluation CLI-change loop](agent-evaluations.md) for Claude Code, Codex and OpenCode, plus [Scout device setup for Claude Code](scout.md#connect-a-coding-agent). Neither is part of the Pi setup below.
|
|
4
4
|
|
|
5
5
|
## Open your inference environment
|
|
6
6
|
|
|
@@ -12,7 +12,7 @@ cd /path/to/your/application
|
|
|
12
12
|
openmerit dash
|
|
13
13
|
```
|
|
14
14
|
|
|
15
|
-
The Inference Terminal is included in stable `openmerit@0.1.
|
|
15
|
+
The Inference Terminal is included in stable `openmerit@0.1.7`. It opens a local, read-only view of the selected application's services, models, recorded requests, and linked compute. It does not require Pi or a provider API key, make inference calls, or change model assignments. Supported code/configuration declarations identify inventory; request history requires existing supported logs or exports. See [the terminal guide](inference-terminal.md) for collection, privacy, configuration, and diagnostics.
|
|
16
16
|
|
|
17
17
|
To inspect fictional data without installation, open the [hosted dashboard](https://dash.openmerit.ai/) or its [before/after comparison](https://dash.openmerit.ai/demo). Neither connects to your machine.
|
|
18
18
|
|
package/docs/index.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# OpenMerit documentation
|
|
2
2
|
|
|
3
|
-
These guides describe the implementation on **main**.
|
|
3
|
+
These guides describe the implementation on **main**. Stable npm `openmerit@0.1.7` includes the Inference Terminal, agent-evaluation CLI and Scout device setup for Claude Code. The website, hosted samples, Scout service and npm package are deployed independently; dated receipts describe the versions actually tested.
|
|
4
4
|
|
|
5
5
|
## See your inference environment
|
|
6
6
|
|
|
@@ -4,9 +4,25 @@ The terminal gives people and agents a local overview of an application's models
|
|
|
4
4
|
|
|
5
5
|
It reads existing evidence independently of inference. It does not call a model, change a route, run evaluations, load or unload models, or retain prompts and responses. Closing it does not stop your workloads. Collection uses local CPU, memory, and authorized runtime API capacity.
|
|
6
6
|
|
|
7
|
+
## Optional CLI product analytics
|
|
8
|
+
|
|
9
|
+
Available in npm 0.1.7. CLI product analytics are off unless `OPENMERIT_TELEMETRY=1`; `DO_NOT_TRACK=1` always disables them. Terminal `discovery` configuration and `--no-discovery` are unrelated to analytics consent.
|
|
10
|
+
|
|
11
|
+
When explicitly enabled, the CLI posts `cli_command_started` and `cli_command_completed` to `https://openmerit.ai/scout/api/analytics`, with a fresh `crypto.randomUUID()` identity for that process only. No identity file, cookie, or installation ID is stored. Properties contain only a command category (`help`, `dash`, `eval_init`, `eval_doctor`, `eval_plan`, `eval_run`, `eval_watch`, `eval_report`, or `unknown`), plus completion outcome (`ok` or `error`) and integer elapsed milliseconds capped at one day. Paths, arguments, prompts, configuration, model IDs, error text, environment contents, machine names, and email are not included.
|
|
12
|
+
|
|
13
|
+
Transport is best-effort, silent, has no retries, and is bounded to about 750 ms per event. It does not change command exit status or JSON output. For a long-running `dash`, completion means startup readiness and the browser-opening attempt finished, not that the server session ended or collection succeeded. JSON and diagnostics modes complete after producing their output. Abrupt termination can lose events. See [evaluation analytics notes](agent-evaluations.md#optional-cli-product-analytics) for watch and Scout request behavior.
|
|
14
|
+
|
|
15
|
+
**Instrumentation in 0.1.7:** the opt-in schema additionally sends a sanitized package version, platform (`darwin`, `linux`, `win32`, `unknown`), supported Node major and random command ID. Errors can include at most eight allowlisted application-basename/line/column frames, never arbitrary stack text, function names, absolute paths or source context. Scout failures use fixed network/HTTP/response/timeout codes and a validated request trace when available; trace correlation is not authentication. These additions qualify the earlier minimal-property descriptions here; they do not include environment contents, change content-sharing/retention controls or enable replay.
|
|
16
|
+
|
|
17
|
+
The same opt-in enables local terminal events using that process-only UUID: `terminal_visited` (sample/live mode), `terminal_page_viewed` (fixed page category, including the connections map), `terminal_interaction` (inspect, filter, search, map expansion, mode or command action and entity kind only), `terminal_snapshot` (retained model, record, source, host, deployment and distinct observed-agent counts), and `terminal_error` (fixed collection/snapshot/render/uncaught error code and standard error class only). Counts are integers from 0 through 1,000,000,000; they describe the retained view, not complete environment totals. No entity keys, names, search values, source paths, raw source, prompts, credentials, hostname, arbitrary error messages or stacks are transmitted. No session replay or login is added.
|
|
18
|
+
|
|
19
|
+
This instrumentation covers the local terminal, not the hosted fictional preview at `dash.openmerit.ai`. The hosted preview remains uninstrumented and deploys separately.
|
|
20
|
+
|
|
21
|
+
`GET /api/info` exposes the boolean `telemetry` capability. Browser events additionally respect browser Do Not Track, Global Privacy Control and `localStorage['scout-analytics-opt-out'] === '1'`. They go only to the same-origin `POST /api/analytics` relay; the browser CSP still permits connections only to self. The loopback Host/Origin/cross-site guards apply to the relay, which projects every accepted event onto a fixed allowlist, limits JSON bodies to 2 KiB and body arrival to one second. Forwarding has no queue or retries, permits at most 60 terminal events per minute across the process and four in flight, and times out after 750 ms. Browser submission is separately capped at 60 events per minute and two in flight with a one-second timeout. Events can be dropped without affecting collection or UI requests. Collection errors follow process consent independently of browser controls. Outbound analytics carries `X-Scout-Client: cli` and `X-Scout-Analytics: on`.
|
|
22
|
+
|
|
7
23
|
## Install
|
|
8
24
|
|
|
9
|
-
Requires **Node.js 22.19 or newer** and npm. Stable `openmerit@0.1.
|
|
25
|
+
Requires **Node.js 22.19 or newer** and npm. Stable `openmerit@0.1.7` includes the terminal:
|
|
10
26
|
|
|
11
27
|
```sh
|
|
12
28
|
npm install -g openmerit
|
|
@@ -20,7 +36,7 @@ The CLI starts a server bound to `127.0.0.1` and opens the browser before initia
|
|
|
20
36
|
|
|
21
37
|
If npm reports `EACCES`, use a user-owned Node installation/npm prefix rather than `sudo`. If the command is not found after installation, reopen the shell and check that npm's global executable directory is on `PATH`.
|
|
22
38
|
|
|
23
|
-
The **
|
|
39
|
+
The **terminal and hosted dashboard** follow the device's light/dark preference, including charts and inspectors, and update when that preference changes. There is no theme control or saved override. The npm terminal includes this appearance behavior in 0.1.7.
|
|
24
40
|
|
|
25
41
|
Keep the CLI running while using the dashboard. Files unreadable by your user are reported as unavailable. Configuration can identify services but cannot supply missing request history.
|
|
26
42
|
|
|
@@ -642,6 +658,6 @@ node packages/terminal/src/cli.mjs dash /path/to/project --json
|
|
|
642
658
|
|
|
643
659
|
While the server is running, `GET /api/snapshot` returns the cached JSON object. Schema version 1 contains `mode`, `observedAt`, `interval`, `project`, `models`, `inventory`, `records`, `taskSummaries`, `sources`, `runtimeStats`, `deployments`, `capacityPools`, `hosts`, `machine`, and `discovery`. The optional `discovery` object reports whether discovery is enabled, its last check time, the 60-second configuration discovery interval, candidate count, contributing request sources, and reconciliation coverage. Request records can include `sourceIds`, a supported log `evidenceFormat`, and `conflictingFields`. Colliding local IDs from demonstrably distinct records receive a stable namespaced ID while retaining `originalId`. The host, deployment, and pool fields are optional version-1 snapshot fields; snapshots without them show unknown coverage. `machine` remains the current host for compatibility; `hosts` is the retained project-linked hardware inventory. Sources affected by inventory limits expose `truncated: true` and a `limits` array of `{ collection, limit, omitted }`; their state is `partial`. The UI shows an inventory-coverage notice and per-source omitted counts. Task summaries never establish request coverage. Unknown status, cost basis, and currency are represented by `"unknown"`. Fields containing unsupported measurements use `null` or are absent. `GET /api/sample` returns only fictional data. `GET /api/info` describes the process's mode and interval.
|
|
644
660
|
|
|
645
|
-
The server binds to `127.0.0.1`, accepts read-only GET requests, checks Host and Origin, rejects cross-site browser requests, and serves only its bundled assets and explicit API routes. It provides no general file-serving endpoint. Local processes and other users with access to this machine may still reach the loopback service; this is not a multi-user authentication boundary.
|
|
661
|
+
The server binds to `127.0.0.1`, accepts read-only GET requests plus the optional analytics-only POST relay described above, checks Host and Origin, rejects cross-site browser requests, and serves only its bundled assets and explicit API routes. The relay does not change project files or inference state. It provides no general file-serving endpoint. Local processes and other users with access to this machine may still reach the loopback service; this is not a multi-user authentication boundary.
|
|
646
662
|
|
|
647
663
|
See [security and data](security.md) for the separate OpenMerit orchestration and harness data boundaries.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# Scout beta data policy
|
|
2
|
+
|
|
3
|
+
Updated 23 October 2026.
|
|
4
|
+
|
|
5
|
+
**Coding-agent reset, 23 October 2026:** this build removes Scout's coding-agent backend and setup/connection tracking. The Claude Code panel is a web-only placeholder with no installation or authentication flow. MCP and agent OAuth metadata routes return HTTP 404 in this build; the website, browser sign-in and JSON API remain. Earlier MCP references below describe the policy before this reset. Existing analytics records and remote grants are not deleted by this reset. A source-only replacement adds device sign-in: Scout stores the approving account, device name you chose (the host name by default), client, scope, timestamps and hashes of device codes and keys, but never a key itself. It is not deployed yet.
|
|
6
|
+
|
|
7
|
+
**Version-5 diagnostics:** Scout serving builds collect page views/leaves, bounded engagement, allowlisted acquisition/UTM and browser/device categories, and sampled browser performance observations where supported and usage analytics is enabled. Native URL properties use only reconstructed public paths, never raw visitor URLs, queries or fragments. Linked activity can join same-tab visits, signup and searches; unlinking retains request-only server identity, not a cross-request funnel. Feedback records a useful/not-useful rating and optional fixed reason, with linked trace/retry correlation, not free text. Server diagnostics include bounded stage/provider failure codes and actual pipeline attempt tokens when reported, not invented charges or executions of recommended models. See the [technical contract](scout.md#version-5-instrumentation). Main-site/docs collection and CLI publication have separate release boundaries. No replay or ranking experiment is enabled, and none of the prompt-content or retention controls below change.
|
|
8
|
+
|
|
9
|
+
Scout is operated by OpenMerit. This policy covers the hosted Scout website, recommendation API and MCP endpoint. It describes current data handling, not a promise that beta data is anonymous or exempt from privacy requirements. External coding agents, the local Inference Terminal and other OpenMerit services have separate data flows.
|
|
10
|
+
|
|
11
|
+
## Help improve Scout during beta
|
|
12
|
+
|
|
13
|
+
Scout processes your submitted task to generate recommendations. With usage analytics enabled, we record recommendations and usage diagnostics for product evaluation, improvement and debugging, not foundation model training. Our service providers process data on our behalf.
|
|
14
|
+
|
|
15
|
+
**Beta revision, effective 22 October 2026.** Browser prompt sharing defaults on only where no content choice is stored; a saved off choice remains off. You can turn sharing off before submitting a task, or choose not to link this browser's activity to your account. Browser privacy signals still take precedence. A beta label or this notice does not establish consent or another legal basis.
|
|
16
|
+
|
|
17
|
+
**Don't submit confidential or sensitive information.** Describe the work rather than pasting credentials, private customer records or real documents. Automated redaction cannot recognize every secret or personal detail.
|
|
18
|
+
|
|
19
|
+
## What we process and why
|
|
20
|
+
|
|
21
|
+
| Data | Purpose and collection |
|
|
22
|
+
| --- | --- |
|
|
23
|
+
| Submitted task and model restrictions | Required to interpret the task, find suitable models and return recommendations. Scout sends task information to OpenAI even when analytics is off. |
|
|
24
|
+
| Recommendations and search diagnostics | When usage analytics is enabled, PostHog receives task categories, model IDs, ranks, available quality/cost/latency evidence, timings, cache status and outcomes. These describe recommendations, not execution of those models on your data. |
|
|
25
|
+
| Product interactions and errors | Enabled analytics includes visits, preset selections, sign-in activity, results viewed, evidence opened, copies, provider-link clicks and coding-agent setup activity. API/MCP events include method, outcome and confirmed connections. Error reports use bounded categories and application stack locations, not raw error messages. |
|
|
26
|
+
| Account details | Clerk handles Google/GitHub login. Enabled account analytics can synchronize your name, verified primary email, signup time and connected providers to PostHog. Account and session lifecycle events help us understand onboarding and returning use. |
|
|
27
|
+
| Optional prompt content | Only when prompt sharing is enabled, PostHog also receives the submitted prompt and written task breakdown, for at most three tasks. Prompt text is limited to 4,000 characters, with up to 1,000 characters per excerpt or five-layer field. Common sensitive patterns are removed where detected; this is not guaranteed anonymization. All such text is confined to `$ai_trace.$ai_input_state={prompt,tasks}`; new ordinary `scout_search_details` events contain no prompt, excerpt or layer prose. |
|
|
28
|
+
|
|
29
|
+
Scout does not use session replay, record keystrokes, or automatically capture every page element. Our analytics code does not forward bearer tokens, cookies, raw Clerk identifiers or visitor IP addresses to PostHog. Hosting and authentication providers still process the connection and security information needed to run their services.
|
|
30
|
+
|
|
31
|
+
## Your choices and account linkage
|
|
32
|
+
|
|
33
|
+
Open **Privacy & analytics** in the [Scout footer](/scout/#analytics-settings):
|
|
34
|
+
|
|
35
|
+
- **Share usage and account analytics** controls optional product events and account synchronization. It is on by default unless you opt out or a supported browser privacy signal disables it.
|
|
36
|
+
- **Share my task prompts and breakdowns for product improvement** defaults on only without a stored content choice. Saved off, general opt-out, DNT/GPC and unreadable preference storage keep content off; usage analytics must also be enabled.
|
|
37
|
+
- **Don’t link this browser’s activity to my account** applies to this browser/request only, not your account everywhere. It sends `X-Scout-Analytics-Account: off`.
|
|
38
|
+
- Turning sharing off stops future analytics collection. It does not stop necessary task processing, undo in-flight work or delete information already stored.
|
|
39
|
+
|
|
40
|
+
Scout respects Do Not Track and Global Privacy Control for these analytics. An authenticated browser also communicates its usage preference to the server; the latest received account preference controls background synchronization. A browser that is offline or not signed in cannot update that account preference until it reconnects and authenticates.
|
|
41
|
+
|
|
42
|
+
API/MCP clients can send `X-Scout-Analytics: off` to disable analytics. Prompt text still requires explicit `X-Scout-Analytics-Content: on`; general opt-out and supported privacy signals take precedence. CLI content capture is always off. Copied agent setup instructions disable prompt-content sharing and do not inherit this browser's account-linkage setting. See the [technical analytics guide](scout.md#product-analytics) for the full contract.
|
|
43
|
+
|
|
44
|
+
Enabled signed-in analytics normally uses a stable account identifier linked to the profile above; hashing does not make it anonymous, and a pre-sign-in visit can be joined after login. In unlinking mode the server uses fresh unrelated random identifiers, sends no profile/identify operation, ignores previously linked browser identifiers and supplied trace IDs, and does not return an analytics trace ID for later interactions to reuse. Account opt-out and deletion gates still apply before sending. Clerk profile/lifecycle analytics can continue separately under general usage consent. Past records remain linked; signing out or selecting this option does not unlink them. Unlinked request events cannot support cross-request funnels or automatic account-based historical deletion. Prompt text itself may identify a person: unlinking is not anonymization.
|
|
45
|
+
|
|
46
|
+
## Service providers
|
|
47
|
+
|
|
48
|
+
| Provider | Role |
|
|
49
|
+
| --- | --- |
|
|
50
|
+
| Cloudflare | Hosts Scout, handles requests and rate limits, holds the public model catalogue, and coordinates analytics preferences, webhook receipts and deletion state. |
|
|
51
|
+
| OpenAI | Processes task descriptions and related public evidence for interpretation, matching and query embeddings. Its API retention policies apply; Scout does not promise zero retention by OpenAI. |
|
|
52
|
+
| Clerk, Google and GitHub | Provide authentication and account/session handling. Google or GitHub is contacted according to the sign-in method you choose. |
|
|
53
|
+
| PostHog | Stores optional product/account analytics, recommendation diagnostics and separately enabled prompt content. Scout uses PostHog's EU ingestion endpoint; this is not a claim that every provider processes data only in the EU. |
|
|
54
|
+
|
|
55
|
+
Scout's hourly research uses Parallel and public model/benchmark sources. Visitor prompts are not sent to Parallel, benchmark publishers or the recommended-model providers as part of that research. Opening a provider link takes you to a separate service under its own policies.
|
|
56
|
+
|
|
57
|
+
## Storage and retention
|
|
58
|
+
|
|
59
|
+
Submitting a task is not a zero-storage operation, even with analytics disabled:
|
|
60
|
+
|
|
61
|
+
- **In your browser:** Scout temporarily keeps a prompt draft in tab storage during sign-in. Analytics preferences use local storage; anonymous visit/session-transition markers use tab storage when analytics is enabled. Clerk manages authentication storage separately.
|
|
62
|
+
- **Search cache:** Scout temporarily keeps exact tasks and results in server memory for reuse. Cached results are eligible for reuse for 30 seconds. Expired entries are removed on subsequent cache access or when the instance ends; this is not a guarantee of physical erasure at exactly 30 seconds.
|
|
63
|
+
- **PostHog:** Content uses the provider's [AI event retention policy](https://posthog.com/docs/ai-observability/data-retention): large AI properties including `$ai_input_state` are stored in `posthog.ai_events` for 30 days and stripped from normal events at ingestion. Metadata remains under normal event retention. Storage separation and the 30-day setting have been checked with synthetic data; actual expiry and backup erasure have not been independently verified. Historical v3 `scout_search_details` may still contain content outside those AI properties. There is no custom retention cron or content export, and no claim of retroactive deletion. Other events and profiles remain subject to applicable retention/deletion settings. Disabling sharing is not a deletion request.
|
|
64
|
+
- **Account synchronization:** Cloudflare retains preference state and pseudonymous delivery/deletion records, not a second copy of your name, email or prompts. Pending lifecycle events without a known preference are bounded and expire after seven days. Deletion tombstones are retained to prevent delayed events from recreating an account's analytics.
|
|
65
|
+
- **Historical records:** Before the 22 October 2026 consolidation, Scout stored prompts and recommendation evidence in a private Cloudflare Analytics Engine archive. New writes have stopped. That archive follows Cloudflare's three-month retention, may contain unredacted personal information, and is not automatically erased by the PostHog account-deletion flow.
|
|
66
|
+
|
|
67
|
+
OpenAI, Clerk and other providers have their own retention and security policies. A change to Scout's analytics settings does not change those policies. Restrict analytics access to authorized personnel with a product evaluation/debugging need; Scout does not expose a public search-history endpoint. The processors above receive data: this is not a “no third party” service.
|
|
68
|
+
|
|
69
|
+
## Data questions and deletion requests
|
|
70
|
+
|
|
71
|
+
Contact [lazim@openmerit.ai](mailto:lazim@openmerit.ai) for data questions or to request access, correction or deletion. Include the account email and approximate dates, but do not send passwords, session tokens or sensitive prompt content. We may need to verify account ownership before acting. The rights and obligations that apply depend on your location and circumstances.
|
|
72
|
+
|
|
73
|
+
Deleting a Clerk account triggers a request to delete its linked PostHog profile and events. Deletion is asynchronous, and Scout repeats reconciliation to catch delayed deliveries; an accepted request is not proof that erasure has finished. Unlinked anonymous events and the historical Cloudflare archive are not automatically covered. Clearing a current profile field also does not remove it from older events. Contact us when a request needs review beyond the account-deletion flow.
|
|
74
|
+
|
|
75
|
+
## Changes during beta
|
|
76
|
+
|
|
77
|
+
We will update this page when data handling changes and identify the revision date. This notice does not itself override an existing opt-out or guarantee a fixed erasure deadline. Privacy settings remain available before and after submitting a task.
|