openmerit 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,312 @@
1
+ // Standalone updater copied into Claude's configuration directory. Only Node
2
+ // built-ins: it must survive the npx cache disappearing, without installing or
3
+ // executing code from the release it downloads.
4
+ import { createHash, randomUUID } from 'node:crypto';
5
+ import { mkdir, open, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
6
+ import { dirname, join, resolve } from 'node:path';
7
+ import { fileURLToPath } from 'node:url';
8
+ import { gunzipSync } from 'node:zlib';
9
+
10
+ type ObjectValue = Record<string, unknown>;
11
+ interface Bundle { format: number; contract: string; version: string; skill: string; rule: string }
12
+ interface UpdateRecord { enabled: boolean; version: string; runnerHash: string; hook: ObjectValue }
13
+ export const updateContract = 'scout-recommend-v1';
14
+ const registry = 'https://registry.npmjs.org/openmerit/latest';
15
+ const day = 24 * 60 * 60 * 1000;
16
+ const retry = 15 * 60 * 1000;
17
+ const digest = (text: string | Buffer) => createHash('sha256').update(text).digest('hex');
18
+ const isObject = (value: unknown): value is ObjectValue => !!value && typeof value === 'object' && !Array.isArray(value);
19
+ const parsed = (text: string): ObjectValue => {
20
+ const value: unknown = JSON.parse(text);
21
+ if (!isObject(value)) throw new Error('Expected a JSON object.');
22
+ return value;
23
+ };
24
+ async function optional(path: string): Promise<string | null> {
25
+ try { return await readFile(path, 'utf8'); }
26
+ catch (error) { if (isObject(error) && error.code === 'ENOENT') return null; throw error; }
27
+ }
28
+ async function object(path: string): Promise<ObjectValue> { return parsed(await optional(path) ?? '{}'); }
29
+ async function atomic(path: string, value: unknown) {
30
+ await mkdir(dirname(path), { recursive: true, mode: 0o700 });
31
+ const temp = `${path}.${randomUUID()}.tmp`;
32
+ try {
33
+ await writeFile(temp, typeof value === 'string' ? value : `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
34
+ await rename(temp, path);
35
+ } finally { await rm(temp, { force: true }); }
36
+ }
37
+ export function updatePaths(base: string, scope: string) {
38
+ return {
39
+ record: join(base, 'openmerit-install.json'), runner: join(base, 'openmerit-updater.mts'),
40
+ settings: join(base, scope === 'user' ? 'settings.json' : 'settings.local.json'),
41
+ state: join(base, 'openmerit-updates.json'), backup: join(base, 'openmerit-update-backup.json'),
42
+ skill: join(base, 'skills/openmerit-scout/SKILL.md'), rule: join(base, 'rules/openmerit-scout.md'),
43
+ };
44
+ }
45
+ function updateRecord(record: ObjectValue): UpdateRecord | null {
46
+ const value = record.updates;
47
+ if (!isObject(value) || typeof value.enabled !== 'boolean' || typeof value.version !== 'string'
48
+ || typeof value.runnerHash !== 'string' || !isObject(value.hook)) return null;
49
+ return value as unknown as UpdateRecord;
50
+ }
51
+
52
+ // Serialize installs, removals and hooks. A dead process's lock is recoverable;
53
+ // an active login is not mistaken for a stale updater just because it is slow.
54
+ export async function withUpdateLock<T>(base: string, work: () => Promise<T>): Promise<T> {
55
+ await mkdir(base, { recursive: true, mode: 0o700 });
56
+ const path = join(base, 'openmerit-update.lock');
57
+ let handle;
58
+ for (let attempt = 0; attempt < 2; attempt++) {
59
+ try { handle = await open(path, 'wx', 0o600); break; }
60
+ catch (error) {
61
+ if (!isObject(error) || error.code !== 'EEXIST') throw error;
62
+ // Only one process may reap a dead lock. Without this guard, two stale
63
+ // observers can unlink each other's newly acquired live lock.
64
+ const recoveryPath = `${path}.recovery`;
65
+ let recovery;
66
+ try { recovery = await open(recoveryPath, 'wx', 0o600); }
67
+ catch { throw new Error('Another OpenMerit lock recovery is running. If it was interrupted, review openmerit-update.lock.recovery before removing it.'); }
68
+ try {
69
+ await recovery.writeFile(JSON.stringify({ pid: process.pid }));
70
+ const text = await optional(path);
71
+ if (text === null) continue;
72
+ let alive = true;
73
+ try {
74
+ const pid = parsed(text).pid;
75
+ if (typeof pid === 'number' && pid > 0) {
76
+ try { process.kill(pid, 0); }
77
+ catch (failure) { if (isObject(failure) && failure.code === 'ESRCH') alive = false; }
78
+ } else alive = Date.now() - (await stat(path)).mtimeMs < 10 * 60_000;
79
+ } catch { alive = Date.now() - (await stat(path)).mtimeMs < 10 * 60_000; }
80
+ if (alive) throw new Error('Another OpenMerit install or update is running. Retry shortly.');
81
+ await rm(path, { force: true });
82
+ } finally { await recovery.close(); await rm(recoveryPath, { force: true }); }
83
+ }
84
+ }
85
+ if (!handle) throw new Error('Another OpenMerit update is running.');
86
+ try { await handle.writeFile(JSON.stringify({ pid: process.pid })); return await work(); }
87
+ finally { await handle.close(); await rm(path, { force: true }); }
88
+ }
89
+
90
+ function groups(settings: ObjectValue): ObjectValue {
91
+ if (settings.hooks === undefined) return {};
92
+ if (!isObject(settings.hooks)) throw new Error('Claude settings.hooks is not an object; nothing changed.');
93
+ return settings.hooks;
94
+ }
95
+ function eventGroups(hooks: ObjectValue, event: string): unknown[] {
96
+ const value = hooks[event];
97
+ if (value === undefined) return [];
98
+ if (!Array.isArray(value)) throw new Error(`Claude ${event} hooks are not an array; nothing changed.`);
99
+ return value;
100
+ }
101
+ function usesRunner(group: unknown, runner: string): boolean {
102
+ return isObject(group) && Array.isArray(group.hooks) && group.hooks.some(hook =>
103
+ isObject(hook) && typeof hook.command === 'string' && (hook.command.includes(runner) || hook.command.includes(quote(runner))));
104
+ }
105
+ export async function inspectUpdates(base: string, scope: string, record: ObjectValue) {
106
+ const paths = updatePaths(base, scope), saved = updateRecord(record);
107
+ const pending = updateRecord({ updates: record.pendingUpdates });
108
+ const owners = [saved, pending].filter(value => value !== null);
109
+ const runner = await optional(paths.runner), settings = await object(paths.settings), hooks = groups(settings);
110
+ const conflicts: string[] = [];
111
+ if (runner !== null && !owners.some(owner => digest(runner) === owner.runnerHash)) conflicts.push('OpenMerit updater was edited or is not owned; nothing changed.');
112
+ for (const event of ['SessionStart', 'Stop']) {
113
+ for (const group of eventGroups(hooks, event)) {
114
+ if (usesRunner(group, paths.runner) && !owners.some(owner => JSON.stringify(group) === JSON.stringify(owner.hook))) {
115
+ conflicts.push(`OpenMerit ${event} hook was edited or is not owned; nothing changed.`);
116
+ }
117
+ }
118
+ }
119
+ const active = runner !== null && ['SessionStart', 'Stop'].every(event =>
120
+ eventGroups(hooks, event).some(group => owners.some(owner => JSON.stringify(group) === JSON.stringify(owner.hook))));
121
+ return { paths, saved: saved ?? pending, runner, settings, hooks, conflicts, active };
122
+ }
123
+ // Claude Code uses a shell for command hooks, including Git Bash on Windows.
124
+ const quote = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`;
125
+ export async function configureUpdates(base: string, scope: string, record: ObjectValue, version: string, enabled: boolean) {
126
+ const state = await inspectUpdates(base, scope, record);
127
+ if (state.conflicts.length) throw new Error(state.conflicts.join('\n'));
128
+ const source = await readFile(new URL('./updates.ts', import.meta.url), 'utf8');
129
+ const hook = { hooks: [{ type: 'command', command: `${quote(process.execPath)} ${quote(state.paths.runner)} --base ${quote(base)} --scope ${scope} --hook`, async: true, timeout: 150 }] };
130
+ const hooks = { ...state.hooks };
131
+ for (const event of ['SessionStart', 'Stop']) {
132
+ const kept = eventGroups(hooks, event).filter(group => !usesRunner(group, state.paths.runner));
133
+ if (enabled) kept.push(hook);
134
+ if (kept.length) hooks[event] = kept; else delete hooks[event];
135
+ }
136
+ // Avoid clobbering settings that changed while the installer was signing in.
137
+ const latest = await object(state.paths.settings);
138
+ if (JSON.stringify(latest) !== JSON.stringify(state.settings)) throw new Error('Claude settings changed during setup. Retry.');
139
+ const next = { enabled, version, runnerHash: digest(source), hook };
140
+ await atomic(state.paths.record, { ...await object(state.paths.record), pendingUpdates: next });
141
+ await atomic(state.paths.runner, source);
142
+ await atomic(state.paths.settings, { ...state.settings, hooks });
143
+ return next;
144
+ }
145
+ export async function removeUpdates(base: string, scope: string, record: ObjectValue) {
146
+ const state = await inspectUpdates(base, scope, record);
147
+ const hooks = { ...state.hooks };
148
+ for (const event of ['SessionStart', 'Stop']) {
149
+ const kept = eventGroups(hooks, event).filter(group => !state.saved || JSON.stringify(group) !== JSON.stringify(state.saved.hook));
150
+ if (kept.length) hooks[event] = kept; else delete hooks[event];
151
+ }
152
+ if (state.saved) await atomic(state.paths.settings, { ...state.settings, hooks });
153
+ const stillReferenced = ['SessionStart', 'Stop'].some(event => eventGroups(hooks, event).some(group => usesRunner(group, state.paths.runner)));
154
+ if (!stillReferenced && state.runner !== null && state.saved && digest(state.runner) === state.saved.runnerHash) await rm(state.paths.runner, { force: true });
155
+ await rm(state.paths.state, { force: true });
156
+ }
157
+
158
+ // Read one fixed regular file in memory, never extract archive paths onto disk.
159
+ function instructionBundle(bytes: Buffer): ObjectValue | null {
160
+ const tar = gunzipSync(bytes, { maxOutputLength: 32 * 1024 * 1024 });
161
+ let found: ObjectValue | null = null;
162
+ for (let offset = 0; offset + 512 <= tar.length;) {
163
+ const header = tar.subarray(offset, offset + 512);
164
+ if (header.every(byte => byte === 0)) break;
165
+ const name = header.subarray(0, 100).toString().replace(/\0.*$/s, '');
166
+ const rawSize = header.subarray(124, 136).toString().replace(/\0.*$/s, '').trim();
167
+ if (!/^[0-7]+$/.test(rawSize)) throw new Error('Invalid package archive.');
168
+ const size = Number.parseInt(rawSize, 8), start = offset + 512;
169
+ if (start + size > tar.length) throw new Error('Truncated package archive.');
170
+ if (name === 'package/dist/terminal/scout-update.json') {
171
+ if (found || size > 256 * 1024 || ![0, 48].includes(header[156] ?? -1)) throw new Error('Invalid instruction bundle.');
172
+ found = parsed(tar.subarray(start, start + size).toString('utf8'));
173
+ }
174
+ offset = start + Math.ceil(size / 512) * 512;
175
+ }
176
+ return found;
177
+ }
178
+ const stableVersion = (version: string) => /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/.test(version);
179
+ function newer(next: string, installed: string) {
180
+ if (!stableVersion(next) || !stableVersion(installed)) return false;
181
+ const a = next.split('.').map(Number), b = installed.split('.').map(Number);
182
+ for (let i = 0; i < 3; i++) { if (a[i] !== b[i]) return a[i]! > b[i]!; }
183
+ return false;
184
+ }
185
+ async function boundedFetch(url: string, max: number, fetcher: typeof fetch, timeout = 12_000): Promise<Buffer> {
186
+ const controller = new AbortController();
187
+ let reader: ReadableStreamDefaultReader<Uint8Array> | undefined;
188
+ let timer: ReturnType<typeof setTimeout> | undefined;
189
+ const deadline = new Promise<never>((_, reject) => {
190
+ timer = setTimeout(() => { controller.abort(); reject(new Error('Update download timed out.')); }, timeout);
191
+ });
192
+ const download = async () => {
193
+ const response = await fetcher(url, { redirect: 'error', signal: controller.signal, headers: { Accept: 'application/json' } });
194
+ if (!response.ok || !response.body || controller.signal.aborted) throw new Error('Update server unavailable.');
195
+ const parts: Uint8Array[] = []; let size = 0;
196
+ reader = response.body.getReader();
197
+ for (;;) {
198
+ const { value, done } = await reader.read();
199
+ if (done) break;
200
+ size += value.length;
201
+ if (size > max) throw new Error('Update download exceeded its size limit.');
202
+ parts.push(value);
203
+ }
204
+ return Buffer.concat(parts);
205
+ };
206
+ try {
207
+ return await Promise.race([download(), deadline]);
208
+ } finally {
209
+ clearTimeout(timer); controller.abort();
210
+ // Cancellation itself must not keep a timed-out hook alive.
211
+ void reader?.cancel().catch(() => {});
212
+ }
213
+ }
214
+ async function recover(base: string, scope: string) {
215
+ const paths = updatePaths(base, scope), text = await optional(paths.backup);
216
+ if (!text) return;
217
+ const backup = parsed(text);
218
+ if (typeof backup.skill !== 'string' || typeof backup.rule !== 'string' || !isObject(backup.record)) throw new Error('Update recovery needs manual review.');
219
+ for (const field of ['skill', 'rule'] as const) {
220
+ const current = await optional(paths[field]);
221
+ if (current !== null && digest(current) !== digest(backup[field] as string) && digest(current) !== backup[`${field}Hash`]) throw new Error('An interrupted update has user edits; nothing changed.');
222
+ }
223
+ // Recover the previous complete pair before any retry. Atomic per-file writes
224
+ // plus a durable journal cover failure between the two renames.
225
+ await atomic(paths.skill, backup.skill); await atomic(paths.rule, backup.rule);
226
+ await atomic(paths.record, backup.record); await rm(paths.backup, { force: true });
227
+ }
228
+ export async function recoverUpdates(base: string, scope: string) { await recover(base, scope); }
229
+
230
+ export interface UpdateResult { status: string; message: string; version?: string }
231
+ export async function runUpdate(base: string, scope: string, options: { hook?: boolean; enabled?: boolean; fetcher?: typeof fetch; now?: number } = {}): Promise<UpdateResult> {
232
+ return withUpdateLock(base, async () => {
233
+ await recover(base, scope);
234
+ const paths = updatePaths(base, scope), record = await object(paths.record), saved = updateRecord(record);
235
+ if (record.version !== 3 || record.scope !== scope || !saved) return { status: 'manual_required', message: 'Run npx -y openmerit@latest install --client claude-code to bootstrap updates in this scope.' };
236
+ if (options.hook && (!saved.enabled || process.env.OPENMERIT_AUTO_UPDATE === 'off')) return { status: 'disabled', message: 'Automatic updates disabled.' };
237
+ const now = options.now ?? Date.now(), state = options.enabled === undefined ? await object(paths.state) : {};
238
+ if (options.hook && typeof state.nextCheckAt === 'number' && now < state.nextCheckAt) return { status: 'not_due', message: 'Update check not due.' };
239
+ const support = await inspectUpdates(base, scope, record);
240
+ if (support.conflicts.length) {
241
+ if (options.hook) await atomic(paths.state, { nextCheckAt: now + day, status: 'conflict' });
242
+ return { status: 'conflict', message: `${support.conflicts.join(' ')} Say “update openmerit” after resolving the conflict.` };
243
+ }
244
+ if (options.enabled !== undefined) {
245
+ record.updates = await configureUpdates(base, scope, record, saved.version, options.enabled);
246
+ delete record.pendingUpdates;
247
+ await atomic(paths.record, record);
248
+ return { status: options.enabled ? 'enabled' : 'disabled', message: `Automatic Scout instruction updates ${options.enabled ? 'enabled' : 'disabled'}. Start a new Claude Code session to reload hooks.` };
249
+ }
250
+ // Record the attempt before networking so a crashed hook cannot hammer npm.
251
+ await atomic(paths.state, { ...state, nextCheckAt: now + retry });
252
+ try {
253
+ const fetcher = options.fetcher ?? fetch;
254
+ const release = parsed((await boundedFetch(registry, 512 * 1024, fetcher)).toString('utf8'));
255
+ const version = release.version;
256
+ if (release.name !== 'openmerit' || typeof version !== 'string' || !stableVersion(version)) throw new Error('Invalid stable release metadata.');
257
+ if (!newer(version, saved.version)) {
258
+ await atomic(paths.state, { checkedAt: now, nextCheckAt: now + day, status: 'current' });
259
+ return { status: 'current', version: saved.version, message: `Scout instructions ${saved.version} are current.` };
260
+ }
261
+ const dist = release.dist;
262
+ const tarball = `https://registry.npmjs.org/openmerit/-/openmerit-${version}.tgz`;
263
+ if (!isObject(dist) || dist.tarball !== tarball || typeof dist.integrity !== 'string' || !/^sha512-[A-Za-z0-9+/]+={0,2}$/.test(dist.integrity)) throw new Error('Release has no trusted SHA-512 package integrity.');
264
+ const bytes = await boundedFetch(tarball, 16 * 1024 * 1024, fetcher, 120_000);
265
+ if (`sha512-${createHash('sha512').update(bytes).digest('base64')}` !== dist.integrity) throw new Error('Package integrity verification failed.');
266
+ const bundle = instructionBundle(bytes);
267
+ if (!bundle || bundle.format !== 1 || bundle.contract !== updateContract || bundle.version !== version) {
268
+ await atomic(paths.state, { checkedAt: now, nextCheckAt: now + day, status: 'manual_required' });
269
+ return { status: 'manual_required', version, message: `Release ${version} needs a reviewed installer upgrade. Review its changes, then run npx -y openmerit@latest install --client claude-code --scope ${scope}. Do not bypass permissions or rerun login automatically.` };
270
+ }
271
+ if (typeof bundle.skill !== 'string' || typeof bundle.rule !== 'string' || !bundle.skill.startsWith('---\nname: openmerit-scout\n') || !bundle.rule.trim()) throw new Error('Invalid Scout instructions.');
272
+ const instructions = bundle as unknown as Bundle;
273
+ const skill = await optional(paths.skill), rule = await optional(paths.rule);
274
+ if (skill === null || rule === null || digest(skill) !== record.skillHash || digest(rule) !== record.ruleHash) {
275
+ await atomic(paths.state, { checkedAt: now, nextCheckAt: now + day, status: 'conflict' });
276
+ return { status: 'conflict', message: 'Scout instructions are missing or edited. Nothing changed. Say “update openmerit” after reviewing the files.' };
277
+ }
278
+ const skillHash = digest(instructions.skill), ruleHash = digest(instructions.rule);
279
+ await atomic(paths.backup, { record, skill, rule, skillHash, ruleHash });
280
+ try {
281
+ // Do not overwrite edits made during the download or journal write.
282
+ if (await optional(paths.skill) !== skill || await optional(paths.rule) !== rule) throw new Error('Scout instructions changed during the update.');
283
+ await atomic(paths.skill, instructions.skill); await atomic(paths.rule, instructions.rule);
284
+ await atomic(paths.record, { ...record, skillHash, ruleHash, updates: { ...saved, version } });
285
+ await rm(paths.backup, { force: true });
286
+ } catch (error) {
287
+ try { await recover(base, scope); }
288
+ catch { return { status: 'conflict', message: 'An interrupted Scout update could not be recovered safely. Review the installed files and openmerit-update-backup.json; no user edits were overwritten.' }; }
289
+ throw error;
290
+ }
291
+ // A failed schedule write must not misreport an already applied update.
292
+ await atomic(paths.state, { checkedAt: now, nextCheckAt: now + day, status: 'updated', version }).catch(() => {});
293
+ return { status: 'updated', version, message: `Scout instructions updated to ${version}. Start a new Claude Code session to load them; the running session may still use its previous instructions.` };
294
+ } catch {
295
+ await atomic(paths.state, { nextCheckAt: now + retry, status: 'retry' });
296
+ return { status: 'retry', message: 'Could not safely update Scout. The installed version was retained. Automatic checks retry after 15 minutes while Claude Code is in use; say “update openmerit” to retry now.' };
297
+ }
298
+ });
299
+ }
300
+
301
+ // Hooks deliberately never ask Claude to continue, read transcripts or emit
302
+ // blocking decisions. Offline and concurrent-hook failures exit successfully.
303
+ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
304
+ const args = process.argv.slice(2), base = args[args.indexOf('--base') + 1], scope = args[args.indexOf('--scope') + 1];
305
+ if (base && ['user', 'project'].includes(scope ?? '')) {
306
+ process.stdin.resume();
307
+ try {
308
+ const result = await runUpdate(resolve(base), scope!, { hook: args.includes('--hook') });
309
+ if (['updated', 'conflict', 'manual_required', 'retry'].includes(result.status)) console.log(JSON.stringify({ systemMessage: result.message }));
310
+ } catch { /* Maintenance cannot block the user's task. */ }
311
+ }
312
+ }
@@ -6,6 +6,10 @@ It reads existing evidence independently of inference. It does not call a model,
6
6
 
7
7
  ## Optional CLI product analytics
8
8
 
9
+ **Version 0.1.10:** Scout setup commands default analytics on without a prompt prefix, with `OPENMERIT_TELEMETRY=0` and `DO_NOT_TRACK=1` opt-outs. Terminal and evaluation commands retain the opt-in behavior below. See the [setup default contract](scout.md#scout-setup-analytics-default-version-0110).
10
+
11
+ **Coding-agent setup telemetry — npm 0.1.9:** opted-in setup adds stage/outcome/component/connection diagnostics and random setup/installation correlation, described in the [Scout lifecycle contract](scout.md#coding-agent-lifecycle-instrumentation-npm-019). Unlike the process-only terminal identity below, an opted-in setup stores its installation UUID in the existing ownership record and installed MCP headers for later calls. No paths, credentials, device names or task content are exported. The copied website prompt explicitly carries usage/unlink preferences; standalone commands still require `OPENMERIT_TELEMETRY=1`, and `DO_NOT_TRACK=1` overrides it. Plans and status checks do not create an installation identity. Older npm 0.1.8 installations need an installer rerun; hosted browser/server collection requires the matching independent Scout deployment.
12
+
9
13
  Available in npm 0.1.7. CLI product analytics are off unless `OPENMERIT_TELEMETRY=1`; `DO_NOT_TRACK=1` always disables them. Terminal `discovery` configuration and `--no-discovery` are unrelated to analytics consent.
10
14
 
11
15
  When explicitly enabled, the CLI posts `cli_command_started` and `cli_command_completed` to `https://openmerit.ai/scout/api/analytics`, with a fresh `crypto.randomUUID()` identity for that process only. No identity file, cookie, or installation ID is stored. Properties contain only a command category (`help`, `dash`, `eval_init`, `eval_doctor`, `eval_plan`, `eval_run`, `eval_watch`, `eval_report`, or `unknown`), plus completion outcome (`ok` or `error`) and integer elapsed milliseconds capped at one day. Paths, arguments, prompts, configuration, model IDs, error text, environment contents, machine names, and email are not included.
@@ -1,10 +1,14 @@
1
1
  # Scout beta data policy
2
2
 
3
- Updated 23 October 2026.
3
+ Updated 25 October 2026.
4
4
 
5
- **Coding-agent reset, 23 October 2026:** this build removes Scout's coding-agent backend and setup/connection tracking. The Claude Code panel is a web-only placeholder with no installation or authentication flow. MCP and agent OAuth metadata routes return HTTP 404 in this build; the website, browser sign-in and JSON API remain. Earlier MCP references below describe the policy before this reset. Existing analytics records and remote grants are not deleted by this reset. A source-only replacement adds device sign-in: Scout stores the approving account, device name you chose (the host name by default), client, scope, timestamps and hashes of device codes and keys, but never a key itself. It is not deployed yet.
5
+ **Source only, not released:** Scout setup usage/installation analytics now defaults on without an analytics-enabling prefix in the normal copied command. `OPENMERIT_TELEMETRY=0` or `DO_NOT_TRACK=1` disables it and propagates to device approval and installed MCP headers. Scout's browser usage/unlink choices and account opt-outs remain effective; MCP prompt-content capture stays off. Terminal/evaluation analytics remain opt-in. These defaults require an updated npm release before prompt deployment; npm 0.1.9 still has the opt-in setup behavior described below.
6
6
 
7
- **Version-5 diagnostics:** Scout serving builds collect page views/leaves, bounded engagement, allowlisted acquisition/UTM and browser/device categories, and sampled browser performance observations where supported and usage analytics is enabled. Native URL properties use only reconstructed public paths, never raw visitor URLs, queries or fragments. Linked activity can join same-tab visits, signup and searches; unlinking retains request-only server identity, not a cross-request funnel. Feedback records a useful/not-useful rating and optional fixed reason, with linked trace/retry correlation, not free text. Server diagnostics include bounded stage/provider failure codes and actual pipeline attempt tokens when reported, not invented charges or executions of recommended models. See the [technical contract](scout.md#version-5-instrumentation). Main-site/docs collection and CLI publication have separate release boundaries. No replay or ranking experiment is enabled, and none of the prompt-content or retention controls below change.
7
+ **Coding-agent setup, 24 October 2026:** hosted Scout supports device sign-in and MCP; stable npm 0.1.8 installs Claude Code's MCP configuration, startup rule and Scout skill after approval. This supersedes the 23 October web-placeholder reset. Scout stores the approving account, device name you chose (the host name by default), client, scope, timestamps and hashes of device codes and keys, but never a plaintext key. Local setup stores the key in private configuration; external coding agents have their own data handling. Installation and startup guidance do not guarantee an agent will invoke Scout.
8
+
9
+ **Lifecycle telemetry — npm 0.1.9 and matching Scout deployment:** this revision adds consent-gated setup-click, clipboard/launch-request, device-authorization, installation/component-verification and MCP-invocation diagnostics. Linked setup uses random setup and installation UUIDs to correlate those milestones; the local ownership record and installed MCP headers retain them across sessions/upgrades. Unlinked mode omits these IDs. Copied setup prompts carry usage/unlink preferences into installation, while standalone CLI telemetry remains opt-in and DNT overrides it. Installed MCP prompt-content capture remains off. The revision also adds bounded delivery retries and sanitized Cloudflare delivery counters, without identity, content, URLs, device codes or credentials in those counters. Existing installations need an installer rerun; these changes do not backfill earlier activity. Release acceptance is recorded separately.
10
+
11
+ **Version-5 diagnostics:** Scout serving builds collect page views/leaves, bounded engagement, allowlisted acquisition/UTM and browser/device categories, and sampled browser performance observations where supported and usage analytics is enabled. Native URL properties use only reconstructed public paths, never raw visitor URLs, queries or fragments. Linked activity can join same-tab visits, signup and searches; unlinking retains request-only server identity, not a cross-request funnel. Feedback records a useful/not-useful rating and optional fixed reason, with linked trace/retry correlation, not free text. Server diagnostics include bounded stage/provider failure codes and actual pipeline attempt tokens when reported, not invented charges or executions of recommended models. Main-site/docs collection and CLI publication have separate release boundaries. No replay or ranking experiment is enabled, and none of the prompt-content or retention controls below change.
8
12
 
9
13
  Scout is operated by OpenMerit. This policy covers the hosted Scout website, recommendation API and MCP endpoint. It describes current data handling, not a promise that beta data is anonymous or exempt from privacy requirements. External coding agents, the local Inference Terminal and other OpenMerit services have separate data flows.
10
14
 
@@ -22,7 +26,7 @@ Scout processes your submitted task to generate recommendations. With usage anal
22
26
  | --- | --- |
23
27
  | Submitted task and model restrictions | Required to interpret the task, find suitable models and return recommendations. Scout sends task information to OpenAI even when analytics is off. |
24
28
  | Recommendations and search diagnostics | When usage analytics is enabled, PostHog receives task categories, model IDs, ranks, available quality/cost/latency evidence, timings, cache status and outcomes. These describe recommendations, not execution of those models on your data. |
25
- | Product interactions and errors | Enabled analytics includes visits, preset selections, sign-in activity, results viewed, evidence opened, copies, provider-link clicks and coding-agent setup activity. API/MCP events include method, outcome and confirmed connections. Error reports use bounded categories and application stack locations, not raw error messages. |
29
+ | Product interactions and errors | Enabled analytics includes visits, preset selections, sign-in activity, results viewed, evidence opened, copies, provider-link clicks and coding-agent setup activity. API/MCP events record request outcomes. Lifecycle-enabled builds add allowlisted method/tool, reported agent/harness and version, setup/installation correlation and verified-connection diagnostics. A launch request is not proof an app opened, and key authorization is not proof of installation. Error reports use bounded categories and application stack locations, not raw error messages. |
26
30
  | Account details | Clerk handles Google/GitHub login. Enabled account analytics can synchronize your name, verified primary email, signup time and connected providers to PostHog. Account and session lifecycle events help us understand onboarding and returning use. |
27
31
  | Optional prompt content | Only when prompt sharing is enabled, PostHog also receives the submitted prompt and written task breakdown, for at most three tasks. Prompt text is limited to 4,000 characters, with up to 1,000 characters per excerpt or five-layer field. Common sensitive patterns are removed where detected; this is not guaranteed anonymization. All such text is confined to `$ai_trace.$ai_input_state={prompt,tasks}`; new ordinary `scout_search_details` events contain no prompt, excerpt or layer prose. |
28
32
 
@@ -39,7 +43,7 @@ Open **Privacy & analytics** in the [Scout footer](/scout/#analytics-settings):
39
43
 
40
44
  Scout respects Do Not Track and Global Privacy Control for these analytics. An authenticated browser also communicates its usage preference to the server; the latest received account preference controls background synchronization. A browser that is offline or not signed in cannot update that account preference until it reconnects and authenticates.
41
45
 
42
- API/MCP clients can send `X-Scout-Analytics: off` to disable analytics. Prompt text still requires explicit `X-Scout-Analytics-Content: on`; general opt-out and supported privacy signals take precedence. CLI content capture is always off. Copied agent setup instructions disable prompt-content sharing and do not inherit this browser's account-linkage setting. See the [technical analytics guide](scout.md#product-analytics) for the full contract.
46
+ API/MCP clients can send `X-Scout-Analytics: off` to disable analytics. Prompt text still requires explicit `X-Scout-Analytics-Content: on`; general opt-out and supported privacy signals take precedence. CLI content capture is always off. Older 0.1.8 setup does not inherit this browser's account-linkage setting; the 0.1.9 lifecycle revision explicitly carries usage and unlink preferences into the command and installed headers. Later browser changes do not rewrite existing agent configuration; rerun setup to update it. Request-only refusals in that revision do not overwrite saved account-wide preferences.
43
47
 
44
48
  Enabled signed-in analytics normally uses a stable account identifier linked to the profile above; hashing does not make it anonymous, and a pre-sign-in visit can be joined after login. In unlinking mode the server uses fresh unrelated random identifiers, sends no profile/identify operation, ignores previously linked browser identifiers and supplied trace IDs, and does not return an analytics trace ID for later interactions to reuse. Account opt-out and deletion gates still apply before sending. Clerk profile/lifecycle analytics can continue separately under general usage consent. Past records remain linked; signing out or selecting this option does not unlink them. Unlinked request events cannot support cross-request funnels or automatic account-based historical deletion. Prompt text itself may identify a person: unlinking is not anonymization.
45
49
 
@@ -52,7 +56,7 @@ Enabled signed-in analytics normally uses a stable account identifier linked to
52
56
  | Clerk, Google and GitHub | Provide authentication and account/session handling. Google or GitHub is contacted according to the sign-in method you choose. |
53
57
  | PostHog | Stores optional product/account analytics, recommendation diagnostics and separately enabled prompt content. Scout uses PostHog's EU ingestion endpoint; this is not a claim that every provider processes data only in the EU. |
54
58
 
55
- Scout's hourly research uses Parallel and public model/benchmark sources. Visitor prompts are not sent to Parallel, benchmark publishers or the recommended-model providers as part of that research. Opening a provider link takes you to a separate service under its own policies.
59
+ Scout's twice-daily research uses Parallel and public model/benchmark sources. Visitor prompts are not sent to Parallel, benchmark publishers or the recommended-model providers as part of that research. Opening a provider link takes you to a separate service under its own policies.
56
60
 
57
61
  ## Storage and retention
58
62
 
package/docs/scout.md CHANGED
@@ -4,8 +4,14 @@ Describe what you need a model to do. Scout returns up to five model-and-endpoin
4
4
 
5
5
  Scout is a separate hosted beta product. Npm 0.1.7 includes optional Claude Code setup for it; the hosted service does not change the Pi coordinator or Inference Terminal. Recommendations are evidence-based candidates for your own evaluation, not a guarantee of task accuracy or an independently validated state-of-the-art ranking.
6
6
 
7
+ **Website rebuild — published 24 October 2026:** the rebuild preserves the existing landing page at `/` and Scout at `/scout/`, and takes all hosted `/docs/` pages offline, including this guide. `/connect/`, the JSON API/MCP and the standalone `/scout/data-policy/` remain available. The guide remains maintained in Markdown; this does not remove the local CLI or its documentation from the package. See the [cutover notes](site-domain-migration.md) for publication and verification details.
8
+
7
9
  **Coding-agent setup — 24 October 2026:** stable npm 0.1.7 includes [device sign-in setup](#connect-a-coding-agent). Hosted Scout serves `/connect` and `/scout/mcp`; the button opens Claude Desktop with a setup prompt. This supersedes the 23 October web-placeholder reset. Existing older client configuration and authorization grants are not automatically removed or revoked. Scout serving, main-site documentation and npm publication remain separate releases; dated receipts below describe their original versions, not acceptance of every later build.
8
10
 
11
+ ## Twice-daily catalogue refresh — deployed, 25 October 2026
12
+
13
+ The hosted catalogue publisher refreshes twice a day, at 00:07 and 12:07 UTC (05:37 and 17:37 IST), instead of hourly. Each run is unchanged: the same feeds, discovery pages, up to 24 Parallel research jobs and the same publication checks. Daily paid research therefore drops from at most 576 to at most 48 jobs, and Hugging Face discovery pages advance twelve times more slowly. Public feeds are reported stale 13 hours after their last successful check, which is one missed run plus an hour's grace. A single failed run can therefore leave up to roughly a day between source checks. The local `npm run scout` preview still refreshes its bundled feeds hourly. The owner-approved publisher and serving deployments went live on 25 October 2026 at 17:00 UTC.
14
+
9
15
  ## Beta evaluation logging — 22 October 2026 rollout
10
16
 
11
17
  The beta default records submitted prompts alongside recommendations for matching evaluation and debugging, not foundation-model training. The serving configuration enables `SCOUT_BETA_EVALUATION_LOGGING=true`, alongside `POSTHOG_CAPTURE_CONTENT=true` and complete account-sync/deletion configuration. The [beta data policy](/scout/data-policy/) is generated from Markdown and published with Scout, independently of the main website. This rollout supersedes the dated default-off contract below. The operator remains responsible for the applicable privacy basis and restricted PostHog access; a beta label does not establish consent or another lawful basis.
@@ -24,6 +30,34 @@ Examples go through the same interpretation and ranking as typed prompts, with n
24
30
 
25
31
  ## Product analytics
26
32
 
33
+ ### Scout setup analytics default — version 0.1.10
34
+
35
+ Version 0.1.10 enables usage and installation diagnostics by default for Scout's `install`, `login`, `status`, `remove`, `logout`, `project` and `doctor` commands. The normal homepage and Scout handoff prompts use `npx -y openmerit@latest install --client claude-code` without an analytics-enabling environment prefix. Terminal and evaluation analytics remain opt-in. npm 0.1.9 uses the opt-in setup behavior below; publish 0.1.10 before deploying these prompts.
36
+
37
+ `OPENMERIT_TELEMETRY=0` or `DO_NOT_TRACK=1` disables setup analytics, strips outgoing correlation IDs and carries the refusal into the approval URL and installed MCP headers. Scout's usage-disabled handoff still includes `DO_NOT_TRACK=1`, and its unlinked handoff includes `OPENMERIT_ANALYTICS_ACCOUNT=off`. Browser DNT/GPC, saved browser choices and server-side account opt-outs remain effective. Analytics-enabled, linked installations retain the random installation UUID described below. The homepage creates no browser setup ID. MCP prompt-content capture stays off; beta status does not establish consent or another legal basis. Package and hosted publication are independent; these changes alone do not establish live event ingestion.
38
+
39
+ ### Coding-agent lifecycle instrumentation — npm 0.1.9
40
+
41
+ Npm 0.1.9 includes these installer changes; browser/server instrumentation requires the matching independent Scout deployment. Release acceptance is recorded separately. Rerun the installer to update existing 0.1.8 installations; nothing backfills historical events. This contract supersedes the no-retry and process-only setup-identity descriptions below for lifecycle-enabled builds. Homepage/docs deployment remains separate.
42
+
43
+ | Touchpoint | Event and meaning |
44
+ |---|---|
45
+ | Add to coding agent | `scout_agent_setup_started` records the click. `scout_agent_launch_requested` records an attempted Claude deep link, **not proof the app opened**. `scout_agent_setup_copied` separately records clipboard success/failure. |
46
+ | Device authorization | Server-owned `scout_connect_step` records device creation, approval-page requests, pending/slow polling, approval or denial, expiry, key collection, device listing and revocation. Expected pending polls are not exceptions. Browser failures use `scout_connect_client_error` with a fixed stage/code. |
47
+ | Installation | Opt-in `scout_installation_step` records preflight, login/browser opening, configuration, verification, status, removal and completion. `applied`, component states and `connection` distinguish a plan, incomplete installation and a verified connection. A command exit of zero alone does not establish installation success. |
48
+ | MCP connection | `scout_request_completed` records every handled MCP request, including initialization/discovery and authentication failures. Allowlisted `mcp_method`, `mcp_tool`, reported agent and package version distinguish verifier probes from agent calls. Unrecognized names become `other`/`unknown`, never raw strings. |
49
+ | Each invocation/task | `scout_tool_invocation_started` and `scout_tool_invocation_completed` bracket authenticated tool dispatch. `scout_search_completed` counts an accepted recommendation attempt; `scout_search_details` retains each task's shortlist. All share the request trace. Protocol/tool errors are failures even with HTTP 200. |
50
+
51
+ When browser usage analytics is enabled, the copied setup prompt explicitly sets `OPENMERIT_TELEMETRY=1`; usage-disabled prompts instead set `DO_NOT_TRACK=1`. Account-unlinked prompts also set `OPENMERIT_ANALYTICS_ACCOUNT=off`. A linked setup gets a random `--setup-id` UUID that joins the click, resumed login, installation and later invocations. An opted-in installation stores a separate random `installation_id` in its existing ownership record and MCP headers; upgrades reuse it. These identifiers contain no account, hostname or project path. They are correlation hints, not authentication. Plans and status checks do not create an installation identity. Standalone CLI telemetry remains default-off, and `DO_NOT_TRACK=1` wins. Installed MCP content capture remains off.
52
+
53
+ The approval page initializes the same browser privacy controls and authenticated account preference synchronization as Scout, without inventing a Scout pageview. Installer URL refusals (`analytics=off`, `analytics_account=off`) remain effective without changing the browser's stored settings; a request-only refusal does not overwrite the account preference. A collected key is labelled **authorized**, not a verified installation. Account-wide opt-out still requires the authenticated preference endpoint and continues to gate linked and unlinked account requests. Unlinking removes setup/installation identifiers as well as browser/trace linkage. A privacy change in a browser does not rewrite an already installed agent's headers; rerun setup with the desired settings. Existing account opt-outs are not automatically reset.
54
+
55
+ First trigger and first task mean **first observed** tool call and task for that installation/account. They are derived from the same event stream, not additional search counts. Discovery is not a trigger. Recommendation completion is not downstream model execution or application-task success. No transcript is read, and this instrumentation cannot prove a startup rule loaded, a model decision occurred without calling Scout, or a native app actually opened. Missing consent, storage, connectivity or abrupt termination can leave a funnel incomplete; missing events must not be presented as a verified zero or a proven skipped call.
56
+
57
+ Initialization can record an allowlisted harness name and semver-shaped reported harness version; the installed headers report the npm installer version, not Claude's current version or invoking model. The stateless service cannot infer the coding agent's model from Scout's own provider generation events. Installer milestones include a fixed `command` category so status/doctor/project completion is never counted as a new installation. Setup milestone submission allows one transient retry within a hard 1.5-second total deadline; general CLI/terminal delivery retains the separately documented behavior below.
58
+
59
+ Browser event submission retries one transient failure with a stable event UUID. Server delivery retries transport failures, HTTP 429 and 5xx up to three total attempts, with the same event UUIDs/timestamps and a 1.5-second timeout per attempt. Permanent 4xx responses do not retry. Product responses do not wait for delivery. There is no durable product-event queue or exactly-once guarantee; bounded retries can still exhaust. The serving Worker configuration enables sanitized Cloudflare console counters (`scout_analytics_delivery`: delivered, failed, request-suppressed or account-suppressed), with query redaction and invocation logs disabled. Counters contain no identity, trace, device code, URL, content or credential and remain available independently of PostHog delivery. They are operational diagnostics, not PostHog service-log ingestion or session replay. `X-Scout-Analytics-Status` reports only whether the request permits capture, never a receipt of ingestion.
60
+
27
61
  ### Version-5 instrumentation
28
62
 
29
63
  This contract applies to version-5 Scout serving builds. The serving rollout is independent of main-site publication and the npm package; npm 0.1.7 includes the opt-in CLI additions, while the website collector remains source-only. Prompt-content choices, redaction, retention and account deletion controls are unchanged. Main-site publication remains blocked by [source divergence](site-documentation.md#website-analytics-source-only-unreleased). Earlier dated receipts below describe earlier releases, not acceptance of version 5.
@@ -457,6 +491,32 @@ Claude can load the `openmerit-scout` skill when choosing an application inferen
457
491
 
458
492
  The agent compares model and endpoint, evidence, reference cost/time and unknowns, and helps configure the chosen route. New providers, credentials, spending and deployments need separate approval. A skill is a default, not enforcement. The MCP tool, `scout_models`, is read-only and does not execute models.
459
493
 
494
+ ### Automatic instruction updates — version 0.1.10
495
+
496
+ Version 0.1.10 adds default-on updates for the installed Scout skill and startup rule. **npm 0.1.9 does not contain the updater.** After 0.1.10 is published, existing installations need one explicit installer rerun in their original scope to acquire it. A website or Scout deployment cannot bootstrap files on someone else's machine. Compatible hosted Scout changes already reach agents through their next MCP request.
497
+
498
+ Setup discloses automatic updates and installs asynchronous `SessionStart` and `Stop` command hooks. They check npm's stable `latest` release at most once per 24 hours per installation. Offline or failed checks retain the installed version and retry at the next eligible hook after 15 minutes. `Stop` means Claude finished responding, not that the user's task succeeded. No background service watches connectivity while Claude Code is closed, and checks never ask Claude to keep working. A one-shot `claude -p` process can exit before an asynchronous Stop hook finishes; startup checks and later sessions provide the retry path. Managed policies or disabled hooks can prevent automatic checks; an explicit manual update remains available.
499
+
500
+ Completed failed checks emit a non-blocking Claude Code notice: the installed version was retained, automatic checks retry after 15 minutes during use, and the user can say **“update openmerit”** to retry manually. Hooks during that backoff stay silent; another eligible failed check emits the notice again. Current-version checks and disabled automatic checks stay silent. Delivery depends on Claude Code processing the completed asynchronous hook output; an exiting one-shot process may not show the notice.
501
+
502
+ The updater lives beside the install record, independent of the npx cache. User scope adds hooks to Claude's `settings.json`; project scope uses `.claude/settings.local.json`. Existing settings and other hooks are preserved. It downloads only from the public npm registry, verifies the package's SHA-512 integrity against registry metadata, and reads a fixed instruction JSON bundle in memory. It does not install the downloaded package or execute its scripts. npm sees the normal network request, but receives no device key, installation ID, task, project path or transcript. Registry integrity protects against mismatched bytes, not a compromised publisher or registry.
503
+
504
+ Only bundles declaring the supported instruction/permission contract update automatically. Releases that change provider permissions, authentication, analytics, hook code or MCP configuration need a reviewed installer upgrade; the updater leaves those components alone. The contract is a release-maintainer assertion, not an automated security review of prose. Maintainers must change the contract when those boundaries change. Updates preserve credentials, project policy and user-edited instructions, hooks and updater files. Writes are serialized and journaled so an interrupted instruction update can recover the previous complete pair; intervening user edits stop recovery instead of being overwritten.
505
+
506
+ Start a **new Claude Code session** after an update to reliably load the new instructions. A running session may retain earlier instructions; reconnecting `/mcp` does not reload the startup rule. The installed rule tells Claude how to handle “update openmerit”; this natural-language trigger is guidance, not an enforced command.
507
+
508
+ Commands in version 0.1.10 and later:
509
+
510
+ ```sh
511
+ openmerit update # check immediately, even if auto updates are off
512
+ openmerit update --auto off # persistently disable automatic updates
513
+ openmerit update --auto on # enable them again
514
+ openmerit update --scope project --json # update this project's installation
515
+ openmerit install --client claude-code --auto-update off # opt out during setup
516
+ ```
517
+
518
+ Use the same scope as installation. `OPENMERIT_AUTO_UPDATE=off` suppresses automatic checks in that process without disabling an explicit manual update. Installer reruns preserve a saved opt-out. `status` includes the automatic-update preference, hook readiness and instruction version; `remove` removes unchanged owned hooks/updater files and retains personal settings and edited files. Manual updates report conflicts, retryable failures or a required installer upgrade instead of claiming success. If lock recovery itself was interrupted, inspect the PID in `openmerit-update.lock.recovery` beside the install record and confirm that process has stopped before removing that recovery lock and retrying.
519
+
460
520
  ### Startup model-decision rule — npm 0.1.8
461
521
 
462
522
  The 0.1.8 installer adds `rules/openmerit-scout.md` alongside the skill. Unlike an on-demand skill, an unscoped [Claude Code rule](https://code.claude.com/docs/en/memory#organize-rules-with-claude/rules/) loads at session startup. The rule requires loading the skill and consulting Scout before any application-model choice, even when the user only asks for a feature. It explicitly includes pretrained/local/browser models, weight downloads, SDK defaults, embeddings, OCR, speech, classification, detection and segmentation. An explicit user model choice must be assessed, not silently replaced. Unchanged approved models and ordinary inference calls do not require another search.
@@ -503,25 +563,26 @@ Scout recommends models; it does not process an uploaded invoice, execute a task
503
563
 
504
564
  ## Real sources and their limits
505
565
 
506
- The hourly updater fetches eight independent public sources, including Models.dev and fourteen bounded Hugging Face pipeline queries. Two additional versioned provider-contract sources have explicit manual review dates. It prepares the index before requests arrive. Queries use batched Decisions questions for intent classification, a bounded verification pass for weak answers, and relevant specialist compatibility checks; they do not browse benchmark sites or ask a model to invent rankings.
566
+ The twice-daily updater fetches eight independent public sources, including Models.dev and fourteen bounded Hugging Face pipeline queries, and adds hosted routes and models from OpenMerit's merged model catalogue. Two additional versioned provider-contract sources have explicit manual review dates. It prepares the index before requests arrive. Queries use batched Decisions questions for intent classification, a bounded verification pass for weak answers, and relevant specialist compatibility checks; they do not browse benchmark sites or ask a model to invent rankings.
507
567
 
508
568
  | Feed | Used for |
509
569
  | --- | --- |
510
570
  | [Epoch AI](https://epoch.ai/benchmarks) | Benchmark definitions, published results, source links, and evaluation configurations. |
511
- | [Artificial Analysis](https://artificialanalysis.ai/leaderboards/models) | Benchmark results, measured first-answer delay and reference generation speed. Archived evaluations remain labelled when their models are still available. |
571
+ | [Artificial Analysis](https://artificialanalysis.ai/leaderboards/models) | Benchmark results for 27 evaluations, measured first-answer delay and reference generation speed. Elo-style ratings (GDPval-AA and AA-3D) become the expected win rate against the median-rated model, as for Arena; AA-Briefcase is not read. Archived evaluations remain labelled when their models are still available. |
512
572
  | [OmniDocBench](https://github.com/opendatalab/OmniDocBench#end-to-end-evaluation) | Document parsing scores from the explicitly labelled `v1.6_full` leaderboard, including newer document models than the OpenVLM snapshot. |
513
573
  | [IDP Leaderboard](https://www.idp-leaderboard.org/) | Nanonets' MIT-licensed document processing leaderboard: OlmOCR, OmniDocBench v1.5 and IDP Core (key information extraction, handwriting, tables, visual QA) component scores. The overall average is not imported; per-model dates are not supplied. |
514
574
  | [OpenCompass / VLMEvalKit](https://huggingface.co/spaces/opencompass/open_vlm_leaderboard) | Visual understanding and OCR benchmark results. Its feed currently identifies itself as 17 September 2025. |
515
575
  | [Berkeley Function Calling Leaderboard (BFCL v4)](https://gorilla.cs.berkeley.edu/leaderboard) | Native function-calling single-turn, live, and multi-turn scores; separate memory scores for key-value, vector, and recursive-summarization stores. |
516
576
  | [OpenRouter](https://openrouter.ai/api/v1/models) | Available model IDs, input modalities, declared tools and structured-output support, context limits, and advertised token prices. |
517
577
  | [Models.dev](https://models.dev) | Independent native model IDs, declared capabilities and prices via ten direct-provider adapters: OpenAI, Anthropic, Google, DeepSeek, Mistral, xAI, Cohere, Moonshot, MiniMax and Z.AI. Not a performance benchmark or account-availability check. |
578
+ | OpenMerit model catalogue | Additional hosted chat routes and token prices, merged from [Models.dev](https://models.dev), [LiteLLM](https://github.com/BerriAI/litellm), OpenRouter, [Vercel AI Gateway](https://vercel.com/ai-gateway/models) and [genai-prices](https://github.com/pydantic/genai-prices). Only reviewed inference hosts and model makers' own OpenAI-compatible APIs with positive per-token prices are included; resellers, routers, subscription plans and account-specific cloud endpoints are not. A route is attached only through an exact model identity. Chat models with such routes that no other source lists become catalogue models, so benchmark results naming them are kept; a catalogue model whose name another listed model already uses is left out. Listings are not verified access in your account. |
518
579
  | [OpenAI Decisions](https://developers.openai.com/api/docs/guides/decisions) | Reviewed typed-choice, predicate and score endpoint contract, inputs and endpoint-specific rates. No general chat timing is inherited. |
519
580
  | [TypeSafe Jev](https://typesafe.ai/blog/jev-1-14) | Reviewed System One contract. Image and single-box output are experimental; headers, version and incomplete total pricing are disclosed. Jev is a candidate for compatible tasks, not Scout’s ranking engine. |
520
581
  | [Hugging Face](https://huggingface.co/models) | Public repository metadata, model-card results and structured evaluation records for specialist discovery. Separate from performance ranking. |
521
582
 
522
- Over 100 benchmark **series** are indexed, including source-specific implementations and versions. This is not 100 independent publishers, and coverage is not uniformly current. Some definitions have no results matched to an available model. Both counts are exposed. Missing metrics and ambiguous or unmatched model versions are omitted. Model discovery merges OpenRouter with independent native-provider records before matching public benchmark results. An OpenRouter outage does not prevent an independent catalogue from bootstrapping. Distinct endpoints retain prices and parameters; duplicate hosting entries do not create additional model votes or consume multiple places in the five. Endpoint listings are metadata, not verified access in the caller’s account.
583
+ Over 100 benchmark **series** are indexed, including source-specific implementations and versions. This is not 100 independent publishers, and coverage is not uniformly current. Some definitions have no results matched to an available model. Both counts are exposed. Missing metrics and ambiguous or unmatched model versions are omitted. Model discovery merges OpenRouter with independent native-provider records and catalogue host routes before matching public benchmark results. An OpenRouter outage does not prevent an independent catalogue from bootstrapping. Distinct endpoints retain prices and parameters; duplicate hosting entries do not create additional model votes or consume multiple places in the five. Endpoint listings are metadata, not verified access in the caller’s account.
523
584
 
524
- Collection time is not measurement time. A successful hourly check does not make an old result new. Some feeds omit measurement dates; those remain unknown. Model release dates are never substituted. OmniDocBench's repository version and leaderboard version differ; Scout uses the version actually printed on the score table. Document parsing is a proxy for invoice extraction, not an evaluation of your exact six fields. New models without comparable evidence may be absent from a task's shortlist.
585
+ Collection time is not measurement time. A successful refresh does not make an old result new. Some feeds omit measurement dates; those remain unknown. Model release dates are never substituted. OmniDocBench's repository version and leaderboard version differ; Scout uses the version actually printed on the score table. Document parsing is a proxy for invoice extraction, not an evaluation of your exact six fields. New models without comparable evidence may be absent from a task's shortlist.
525
586
 
526
587
  Epoch AI attribution: “Capabilities & benchmarking,” published at epoch.ai, normalized and matched by Scout under the source's [CC BY 4.0 terms](https://creativecommons.org/licenses/by/4.0/). Other sources are attributed in the source panel and individual evidence links. These are data-source attributions, not a statement about OpenMerit's licensing.
527
588
 
@@ -529,7 +590,7 @@ Epoch AI attribution: “Capabilities & benchmarking,” published at epoch.ai,
529
590
 
530
591
  Some specialist models have useful evaluations outside general LLM leaderboards. Scout reads both legacy model-card `model-index` results and the Hub’s [structured evaluation records](https://huggingface.co/docs/hub/eval-results). It retains the original dataset, task or metric, units, configuration, split, reported date, setup notes and source link when supplied. Scores are not averaged across unrelated datasets. A repository revision pins the metadata collected; it does not establish which checkpoint produced every reported score. Reported dates can also default to a commit date. Scout does not validate verification tokens or independently reproduce these scores.
531
592
 
532
- The hourly discovery sample takes up to 30 repositories sorted by downloads for each of fourteen pipelines: document question answering, token classification, text classification, sentence similarity, text-to-text generation, text generation, image-to-text, image-text-to-text, image classification, zero-shot image classification, object detection, zero-shot object detection, image segmentation and mask generation. The two image-to-text queries remain OCR-filtered. Bounded pinned configs add label vocabularies for up to six repositories in each fixed-label vision pipeline. Private/disabled repositories, adapters and quantization packages are excluded; fine-tunes remain separate. Downloads select coverage only, never quality. This bounded sample can miss capable low-download specialists.
593
+ Each refresh's discovery sample takes up to 30 repositories sorted by downloads for each of fourteen pipelines: document question answering, token classification, text classification, sentence similarity, text-to-text generation, text generation, image-to-text, image-text-to-text, image classification, zero-shot image classification, object detection, zero-shot object detection, image segmentation and mask generation. The two image-to-text queries remain OCR-filtered. Bounded pinned configs add label vocabularies for up to six repositories in each fixed-label vision pipeline. Private/disabled repositories, adapters and quantization packages are excluded; fine-tunes remain separate. Downloads select coverage only, never quality. This bounded sample can miss capable low-download specialists.
533
594
 
534
595
  For document, code and retrieval components, **Specialist models** expands beneath the five recommendations when compatible prepared profiles match. For classification, detection and segmentation, compatible native pipelines may enter the five **Candidates to evaluate** instead. Fixed-label pipelines need complete semantic coverage of the requested categories; one overlapping label is insufficient. Zero-shot pipelines accept supplied labels; masks can require points or boxes. Pipeline metadata is not an accuracy guarantee. Up to three components are ordered by metadata completeness and component diversity after compatibility checks. Domain and language requirements come from Decisions, with no prompt keyword dictionary. Tool execution and persistent-memory tasks do not receive unvalidated specialist matches. Entity taggers, OCR and embedding models are components requiring integration, not interchangeable chat APIs. Repeated repository names are de-emphasized for display diversity without merging their identities. See the release status in [semantic specialist matching](#semantic-specialist-matching-hosted-beta-15-october-2026).
535
596
 
@@ -547,7 +608,7 @@ Image tasks require declared image input and relevant visual evidence in perform
547
608
 
548
609
  Classification and judgment tasks first use applicable benchmark evidence. With no supported benchmark candidates, supplied-label classification and judgments retain the explicitly labelled **task-fit** fallback. Boxes and masks use their compatible task-fit path. Documented operation/output fit, deployment readiness and known reference costs/times determine unmeasured shortlists; quality stays unknown. Open or uncertain identification instead needs name-generating endpoints and relevant evidence; fixed-choice APIs cannot satisfy it. Specialist compatibility uses hourly-prepared public profiles. Unknown compatibility omits a repository with a warning, rather than assuming task accuracy.
549
610
 
550
- For performance ranking, a query embedding retrieves benchmark methodologies, supplemented by their hourly capability memberships. Bounded independent Decisions predicates verify relevance and a deliberately scored required ability before a methodology contributes proxy evidence. Similarity never supplies a quality score. Rejected, unknown and unavailable evidence is withheld. If some checks fail, verified evidence can still be used with an explicit warning; if no applicability can be verified because checks are unavailable, the search returns unavailable rather than reporting a false no-match.
611
+ For performance ranking, a query embedding retrieves benchmark methodologies, supplemented by capability memberships prepared at each refresh. Bounded independent Decisions predicates verify relevance and a deliberately scored required ability before a methodology contributes proxy evidence. Similarity never supplies a quality score. Rejected, unknown and unavailable evidence is withheld. If some checks fail, verified evidence can still be used with an explicit warning; if no applicability can be verified because checks are unavailable, the search returns unavailable rather than reporting a false no-match.
551
612
 
552
613
  Quality comparisons use shared benchmark results, grouped by correlated family and task capability. Different cohorts are not compared through unrelated percentiles. BFCL uses native function-calling configurations with their original labels; correlated series are one family. Missing scores remain unknown and actual zero remains a score. Model/version matching remains conservative. A relative quality signal is not predicted accuracy or a statistical confidence interval.
553
614
 
@@ -622,7 +683,7 @@ Cloudflare [retains Analytics Engine events for three months](https://developers
622
683
 
623
684
  Each query starts with batched Decisions questions and can make one verification pass for weak answers, followed by relevant specialist compatibility batches. Task interpretation and specialist comparison share a four-second request budget; catalogue reads and ranking add their own time. Optional verification has at most 750 milliseconds and leaves weak answers uncertain if unavailable. Independent specialist batches run in parallel, with at most 100 questions per call. Transient connection, timeout, rate-limit and server failures can receive one retry within the remaining deadline; valid `Retry-After` delays are honored. Authentication, exhausted-credit and quota failures are not retried. Failed interpretation returns an explicit unavailable response; failed specialist checks omit unverified specialist candidates. No keyword fallback is used. These bounds reduce avoidable waits and failures; they do not guarantee a successful search or a particular latency. In-memory admission allows 32 active requests and 600 starts per minute per Worker isolate; a separate edge limiter allows 60 POST requests per minute per IP. Excess requests fail promptly instead of queuing. These are operational limits, not an account-level billing cap or a throughput guarantee. Credentials remain server-side.
624
685
 
625
- Hourly sources refresh independently; reviewed typed-endpoint contracts retain their manual review dates. Freshness follows each source's prepared policy: public feeds have a two-hour freshness allowance, while incremental research uses the configured seven-day recheck interval. A source does not become stale merely because it follows a longer research interval. Missing or invalid check dates remain overdue, and actual collection or preparation failures remain degraded. A failed download, changed parser format, or unexpected loss of coverage preserves that source's previous evidence and timestamp while healthy sources and prices continue updating. Fresh catalogue removals still remove unavailable models. All fourteen Hub queries and required semantic preparation must succeed for its discovery snapshot to replace the previous one. A complete outage preserves the previous index. Failure and stale status are exposed in both interfaces.
686
+ Public sources refresh twice daily and independently; reviewed typed-endpoint contracts retain their manual review dates. Freshness follows each source's prepared policy: public feeds have a 13-hour freshness allowance (one twelve-hour cycle plus an hour), while incremental research uses the configured seven-day recheck interval. A source does not become stale merely because it follows a longer research interval. Missing or invalid check dates remain overdue, and actual collection or preparation failures remain degraded. A failed download, changed parser format, or unexpected loss of coverage preserves that source's previous evidence and timestamp while healthy sources and prices continue updating. Fresh catalogue removals still remove unavailable models. All fourteen Hub queries and required semantic preparation must succeed for its discovery snapshot to replace the previous one. A complete outage preserves the previous index. Failure and stale status are exposed in both interfaces.
626
687
 
627
688
  Source content is parsed as data, never executed. Downloads, decompression, and request bodies are size-bounded; source downloads time out. Links are restricted to HTTP(S). The publisher's OpenVLM data URL uses HTTP and lacks transport authentication; the remaining feeds use HTTPS. Scout does not independently authenticate benchmark submissions.
628
689
 
@@ -637,11 +698,11 @@ npm run scout
637
698
 
638
699
  Open `http://127.0.0.1:45833/scout/`. This lightweight local preview uses the shared interpreter and matcher with bundled/source-feed data; it does not run the hosted Parallel research workflow or R2 publication system. Local API and MCP endpoints use that same origin. `SCOUT_PORT` changes the port. The server immediately uses the newer of its bundled index and `.openmerit/scout/snapshot.json` cache, refreshes when older than an hour, and continues hourly while running. Stopping the process stops its local schedule.
639
700
 
640
- `npm run scout:refresh` fetches the eight hourly public sources and atomically updates `scout/snapshot.json` after validation. Successful sources replace their previous data; failed sources retain labelled prior data when available. Source downloads need no provider inference, but preparing new or changed specialist profiles and benchmark interpretations requires `OPENAI_API_KEY` and incurs Decisions usage. Unchanged metadata reuses cached analysis. Without a usable key, required specialist preparation retains the previous specialist snapshot; affected new or changed benchmark definitions remain unclassified. Both failures are reported on the affected source. It does not run model evaluations or update the separate calibration file. Calibration refreshes are an explicit offline evaluation and deployment; no paid hourly evaluation job is installed. Recommendation queries require the server key and incur OpenAI interpretation/embedding usage. The local source-refresh command updates the bundled feeds; the hosted hourly Workflow additionally prepares and publishes its D1/Vectorize serving generation.
701
+ `npm run scout:refresh` fetches the eight public sources and atomically updates `scout/snapshot.json` after validation. Successful sources replace their previous data; failed sources retain labelled prior data when available. Source downloads need no provider inference, but preparing new or changed specialist profiles and benchmark interpretations requires `OPENAI_API_KEY` and incurs Decisions usage. Unchanged metadata reuses cached analysis. Without a usable key, required specialist preparation retains the previous specialist snapshot; affected new or changed benchmark definitions remain unclassified. Both failures are reported on the affected source. It does not run model evaluations or update the separate calibration file. Calibration refreshes are an explicit offline evaluation and deployment; no paid scheduled evaluation job is installed. Recommendation queries require the server key and incur OpenAI interpretation/embedding usage. The local source-refresh command updates the bundled feeds; the hosted twice-daily Workflow additionally prepares and publishes its D1/Vectorize serving generation.
641
702
 
642
703
  ## Hosting and verification
643
704
 
644
- `wrangler.scout.jsonc` defines the Scout Worker, a private R2 catalogue, an hourly Workflow and a SQLite Durable Object coordinating publication. The cron runs at minute 7 UTC. Configure `OPENAI_API_KEY` and `PARALLEL_API_KEY` as Worker secrets. Refreshes build immutable catalogue versions and prepared candidate pools; searches read those pools directly and never pass through the publication coordinator. A running refresh does not block searches against the active catalogue. The first hosted publication must be validated before serving searches; a missing catalogue fails explicitly. The Worker is `openmerit-scout`, the private catalogue bucket is `openmerit-scout-catalogue`, and the Workflow is `openmerit-scout-refresh`. Both API keys are stored as Cloudflare Worker secrets and read from server-side bindings; they are not shipped to the browser. Separate workers.dev and preview URLs are disabled.
705
+ `wrangler.scout.jsonc` defines the Scout Worker, a private R2 catalogue, a twice-daily Workflow and a SQLite Durable Object coordinating publication. The cron runs at 00:07 and 12:07 UTC (`7 0,12 * * *`). Configure `OPENAI_API_KEY` and `PARALLEL_API_KEY` as Worker secrets. Refreshes build immutable catalogue versions and prepared candidate pools; searches read those pools directly and never pass through the publication coordinator. A running refresh does not block searches against the active catalogue. The first hosted publication must be validated before serving searches; a missing catalogue fails explicitly. The Worker is `openmerit-scout`, the private catalogue bucket is `openmerit-scout-catalogue`, and the Workflow is `openmerit-scout-refresh`. Both API keys are stored as Cloudflare Worker secrets and read from server-side bindings; they are not shipped to the browser. Separate workers.dev and preview URLs are disabled.
645
706
 
646
707
  Publication records a durable completion receipt after updating the active pointer, pending-research queue and health status. Interrupted publication resumes unfinished work; a lost success reply can be acknowledged again after the lease is released. Replaying a completed publication cannot roll back a newer catalogue or clear a later refresh's lease. Completed research receipts leave the queue only when publication commits; unfinished research retains its original paid job for a later hourly run.
647
708
 
@@ -651,7 +712,7 @@ The 17 October post-deployment recovery reused all 24 saved research jobs and pu
651
712
 
652
713
  Prepared capability pools can overlap. Searches use their prepared ID lists to choose a smaller covering set of pools, preserving the selected candidates and complete evidence while avoiding repeated downloads of the same model records. Explicit model and endpoint inventories retain their existing scope checks. These metadata reads do not fetch source pages or redo hourly research.
653
714
 
654
- `demo/domain-router.mjs` routes Scout paths to this service while retaining the existing website, protected memo, authentication, and terminal services. There is no Scout admin/report route. The public guide is generated and published with the main documentation. Do not deploy the older root `wrangler.jsonc` to publish Scout. `npm run build:scout` packages only Scout, public documentation, and its shared assets; private memo and demo applications are excluded.
715
+ `demo/domain-router.mjs` routes the landing page, Scout and `/connect/` to the Scout serving service. Hosted docs, demos, engineering articles and terminal pages/APIs are offline during the rebuild. The separately protected memo was restored through the router on 25 October 2026; see the [cutover record](site-domain-migration.md). There is no Scout admin/report route. Do not deploy the older root `wrangler.jsonc` to publish Scout. `npm run build:scout` packages the landing page, Scout, `/connect/`, the standalone Scout data policy and required shared assets; generated docs, memo and demo applications are excluded.
655
716
 
656
717
  ```sh
657
718
  npm run check:scout