openmerit 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/README.md +12 -25
  3. package/dist/terminal/scout-update.json +1 -1
  4. package/dist/terminal/src/browser.mjs +2 -4
  5. package/dist/terminal/src/cli.mjs +4 -56
  6. package/dist/terminal/src/install.mjs +2 -2
  7. package/dist/terminal/src/scout-pi.js +1 -1
  8. package/dist/terminal/src/scout.js +3 -3
  9. package/dist/terminal/src/telemetry.mjs +8 -60
  10. package/docs/agent-evaluations.md +8 -148
  11. package/docs/commands.md +3 -9
  12. package/docs/getting-started.md +2 -16
  13. package/docs/index.md +4 -18
  14. package/docs/inference-terminal.md +13 -594
  15. package/docs/roadmap.md +2 -2
  16. package/docs/scout.md +6 -2
  17. package/docs/security.md +5 -18
  18. package/docs/testing.md +4 -16
  19. package/docs/troubleshooting.md +2 -2
  20. package/package.json +10 -23
  21. package/dist/eval/src/adapters.mjs +0 -112
  22. package/dist/eval/src/cli.mjs +0 -151
  23. package/dist/eval/src/config.mjs +0 -108
  24. package/dist/eval/src/process.mjs +0 -44
  25. package/dist/eval/src/report.mjs +0 -59
  26. package/dist/eval/src/runner.mjs +0 -123
  27. package/dist/eval/src/scout.mjs +0 -48
  28. package/dist/terminal/public/app.js +0 -298
  29. package/dist/terminal/public/brands/anthropic.png +0 -0
  30. package/dist/terminal/public/brands/baai.png +0 -0
  31. package/dist/terminal/public/brands/baseten.png +0 -0
  32. package/dist/terminal/public/brands/cerebras.png +0 -0
  33. package/dist/terminal/public/brands/cohere.png +0 -0
  34. package/dist/terminal/public/brands/deepseek.ico +0 -0
  35. package/dist/terminal/public/brands/google.png +0 -0
  36. package/dist/terminal/public/brands/groq.ico +0 -0
  37. package/dist/terminal/public/brands/huggingface.svg +0 -8
  38. package/dist/terminal/public/brands/lm-studio.png +0 -0
  39. package/dist/terminal/public/brands/meta.ico +0 -0
  40. package/dist/terminal/public/brands/mistral.png +0 -0
  41. package/dist/terminal/public/brands/nomic.png +0 -0
  42. package/dist/terminal/public/brands/ollama.png +0 -0
  43. package/dist/terminal/public/brands/openai.png +0 -0
  44. package/dist/terminal/public/brands/openrouter.png +0 -0
  45. package/dist/terminal/public/brands/qwen.png +0 -0
  46. package/dist/terminal/public/brands/vllm.png +0 -0
  47. package/dist/terminal/public/favicon.svg +0 -1
  48. package/dist/terminal/public/index.html +0 -22
  49. package/dist/terminal/public/styles.css +0 -921
  50. package/dist/terminal/src/activity-merge.mjs +0 -66
  51. package/dist/terminal/src/collect.mjs +0 -370
  52. package/dist/terminal/src/coverage.mjs +0 -33
  53. package/dist/terminal/src/discovery.mjs +0 -99
  54. package/dist/terminal/src/hardware.mjs +0 -57
  55. package/dist/terminal/src/history.mjs +0 -32
  56. package/dist/terminal/src/log-continuity.mjs +0 -26
  57. package/dist/terminal/src/project-activity.mjs +0 -157
  58. package/dist/terminal/src/repository-background.mjs +0 -37
  59. package/dist/terminal/src/repository-index.mjs +0 -267
  60. package/dist/terminal/src/repository-policy.mjs +0 -3
  61. package/dist/terminal/src/repository-syntax.mjs +0 -236
  62. package/dist/terminal/src/repository-worker.mjs +0 -15
  63. package/dist/terminal/src/sample.mjs +0 -171
  64. package/dist/terminal/src/scout-credentials.d.ts +0 -12
  65. package/dist/terminal/src/scout-credentials.ts +0 -44
  66. package/dist/terminal/src/scout-instructions.d.ts +0 -5
  67. package/dist/terminal/src/scout-instructions.ts +0 -119
  68. package/dist/terminal/src/scout-pi.d.ts +0 -44
  69. package/dist/terminal/src/scout-pi.ts +0 -76
  70. package/dist/terminal/src/scout.d.ts +0 -77
  71. package/dist/terminal/src/scout.ts +0 -164
  72. package/dist/terminal/src/server.mjs +0 -95
  73. package/dist/terminal/src/service-registry.mjs +0 -78
  74. package/dist/terminal/src/services.mjs +0 -44
  75. package/dist/terminal/src/topology.mjs +0 -30
  76. package/dist/terminal/src/updates.d.ts +0 -62
  77. package/dist/terminal/src/updates.ts +0 -313
@@ -1,119 +0,0 @@
1
- import type { ScoutClient } from './scout-credentials.ts';
2
-
3
- export const rule = `# Application model decisions · OpenMerit Scout
4
-
5
- Before choosing, recommending, adding, downloading, replacing or configuring
6
- ANY application inference model, load the \`openmerit-scout\` skill and consult
7
- \`scout_models\` on the \`openmerit\` MCP server before committing to a model.
8
- This applies even when the user only asks you to build a feature, never mentions
9
- models or Scout, or names a model themselves (assess it; do not silently replace it).
10
-
11
- A pretrained model, local weights, browser inference, an SDK's default model,
12
- embeddings, OCR, speech, image classification, detection and segmentation all
13
- count as model choices, not implementation details. Identify this decision before
14
- writing model-specific code, installing a model package or downloading weights.
15
-
16
- Follow the skill's project access policy. If it is missing, ask and save it first.
17
- Scout may answer with clarifying questions: relay them to the user and continue
18
- the same Scout conversation with their answers before choosing.
19
- If Scout is unavailable, fails or has no suitable evidence/candidate, disclose the
20
- gap and pause the model decision; do not fall back to a remembered or SDK-default
21
- model. You may continue work that does not commit to a model.
22
-
23
- Do not consult Scout for unrelated edits, normal inference calls using an unchanged
24
- approved model, or this coding agent's own model. Do not reselect approved models
25
- unless the task introduces or changes a model decision.
26
-
27
- When the user says "update openmerit", run
28
- \`npx -y openmerit@latest update --scope user\` (use \`--scope project\` if this
29
- rule is in the project's .claude/rules directory). Report the result, including
30
- any conflict or manual-upgrade requirement. Do not delete edited files, bypass
31
- permissions or rerun login automatically. A new session loads updated instructions.
32
- `;
33
-
34
- export const skill = `---
35
- name: openmerit-scout
36
- description: Consults OpenMerit Scout before choosing, adding, downloading, replacing or configuring any application inference model, including implicit choices while building features, pretrained browser/local models, SDK defaults, embeddings, OCR, speech, classification, detection, segmentation, LLMs and agents. Not for unrelated edits, ordinary inference calls or this coding agent's own model.
37
- ---
38
-
39
- # OpenMerit · Scout
40
-
41
- Before introducing, recommending, replacing or configuring an application
42
- inference model, call the \`scout_models\` tool on the \`openmerit\` MCP server.
43
- This applies even when the user never mentions models or Scout. Downloading local
44
- weights, using a pretrained browser model or accepting an SDK default is a model
45
- decision: consult Scout before model-specific code, installs or downloads.
46
- Preserve the user's explicit model choice; assess it rather than silently replacing
47
- it. Leave unchanged, approved application selections in place.
48
-
49
- ## Project policy
50
-
51
- Read \`.openmerit/scout.json\` before searching. If it is absent, ask once whether this
52
- project uses **existing-only** (only endpoints it can already reach) or
53
- **existing-first** (prefer them, ask before searching the whole catalogue), and
54
- which public catalogue endpoint IDs it can access. Save the answer:
55
-
56
- \`\`\`sh
57
- npx -y openmerit@latest project --policy existing-only --endpoint-id <id> ...
58
- \`\`\`
59
-
60
- Never infer access from a model name, SDK or environment-variable name. Never
61
- record credentials, private URLs, prompts or customer data. An empty inventory
62
- is not permission to search all models.
63
-
64
- ## Searching
65
-
66
- Call \`scout_models\` with a non-confidential task description as \`prompt\` and
67
- \`endpointIds\` set to the saved inventory, including \`[]\` when empty. Send
68
- only task requirements, never source files, secrets or real customer records.
69
- If the MCP tool is unavailable, run
70
- \`npx -y openmerit@latest scout "<task>" --json\`; it applies the saved inventory.
71
-
72
- Scout is a conversation with the agent behind openmerit.ai/scout. When
73
- \`status\` is \`needs_answers\`, fill in what the user or project already states,
74
- show the user Scout's brief and remaining questions, and wait for their answers.
75
- Then call again with the returned \`conversationId\` and \`brief\` set to the
76
- rewritten components, or \`skip: true\` if the user wants Scout to assume. Never
77
- answer Scout's questions with guesses. Send later refinements to the same
78
- \`conversationId\`. A call with only a \`conversationId\` reads an interrupted turn.
79
-
80
- When \`status\` is \`recommended\`, Scout has split the work into \`tasks\`, one
81
- per model, each with up to five ranked \`models\`; cover every task. Each
82
- model's \`cost\` names the \`endpointId\` and \`providerModelId\` to call.
83
- Existing-only never broadens. For existing-first, if no existing candidate fits a
84
- task, explain the gap and ask before a new conversation without \`endpointIds\`.
85
- Compare quality evidence, reference cost/time and watch-outs. Prefer the cheapest
86
- usable option the evidence supports, without claiming benchmarks prove
87
- application accuracy. Use the project's actual SDK and protocol.
88
-
89
- ## Boundaries
90
-
91
- Ask before adding providers or credentials, spending, deploying or external
92
- writes. Report recommendation, endpoint connectivity and application checks as
93
- separate results, each only after it succeeds. Do not switch this coding agent's
94
- own model.
95
-
96
- If Scout fails or has no suitable candidate/evidence for the required deployment,
97
- say so and pause the model decision; do not substitute a remembered recommendation
98
- or SDK default. Continue only model-independent work. Local/browser models are not
99
- exempt when the catalogue lacks coverage. If it reports that the device key is missing,
100
- expired or revoked, run \`npx -y openmerit@latest login\` and then
101
- \`npx -y openmerit@latest install\` again. Never ask the user to paste a key.
102
- `;
103
-
104
- export function scoutRule(client: ScoutClient): string {
105
- if (client === 'claude-code') return rule;
106
- return rule.replace('load the `openmerit-scout` skill and consult', 'follow the Scout project policy below and consult')
107
- .replaceAll('`scout_models`', '`mcp__openmerit__scout_models`')
108
- .replace("Follow the skill's project access policy.", 'Follow the project access policy below.')
109
- .replace('update --scope user', 'install --client pi --scope user')
110
- .replace("rule is in the project's .claude/rules directory", "Pi installation is project-scoped");
111
- }
112
-
113
- export function scoutSkill(client: ScoutClient): string {
114
- if (client === 'claude-code') return skill;
115
- return skill.replaceAll('`scout_models`', '`mcp__openmerit__scout_models`')
116
- .replace('openmerit@latest login', 'openmerit@latest login --client pi')
117
- .replace('openmerit@latest install`', 'openmerit@latest install --client pi`')
118
- .replace('again. Never ask the user to paste a key.', 'again, using `--scope project` on both commands for a project-scoped installation. Never ask the user to paste a key.');
119
- }
@@ -1,44 +0,0 @@
1
- import type { ScoutScope } from './scout-credentials.ts';
2
- export declare const piVersion = "1.5.0";
3
- type RecordValue = Record<string, unknown>;
4
- export declare function piPaths(root: string, scope: ScoutScope): {
5
- skill: string;
6
- rule: string;
7
- connection: string;
8
- record: string;
9
- legacyRule: string;
10
- mcpFile: string;
11
- mcpScope: ScoutScope;
12
- agentDir: string;
13
- };
14
- export declare function inspectPi(root: string, scope: ScoutScope): Promise<{
15
- paths: {
16
- skill: string;
17
- rule: string;
18
- connection: string;
19
- record: string;
20
- legacyRule: string;
21
- mcpFile: string;
22
- mcpScope: ScoutScope;
23
- agentDir: string;
24
- };
25
- record: RecordValue;
26
- owned: boolean;
27
- current: string | null;
28
- installedSkill: string | null;
29
- installedRule: string | null;
30
- serverOwned: boolean;
31
- skillOwned: boolean;
32
- ruleOwned: boolean;
33
- updates: {
34
- saved: null;
35
- active: boolean;
36
- conflicts: string[];
37
- };
38
- legacyRule: string;
39
- shadowed: boolean;
40
- }>;
41
- export declare function piConflicts(state: Awaited<ReturnType<typeof inspectPi>>): string[];
42
- export declare function configurePi(root: string, scope: ScoutScope, credentialsPath: string, ids: RecordValue): Promise<void>;
43
- export declare function removePi(state: Awaited<ReturnType<typeof inspectPi>>): Promise<void>;
44
- export {};
@@ -1,76 +0,0 @@
1
- import { createHash } from 'node:crypto';
2
- import { readFile, mkdir, writeFile, rm } from 'node:fs/promises';
3
- import { homedir } from 'node:os';
4
- import { dirname, join, resolve } from 'node:path';
5
- import type { ScoutScope } from './scout-credentials.ts';
6
- import { scoutSkill } from './scout-instructions.ts';
7
-
8
- export const piVersion = '1.5.0';
9
- type RecordValue = Record<string, unknown>;
10
- const hash = (value: string) => createHash('sha256').update(value).digest('hex');
11
- const optional = async (path: string): Promise<string | null> => {
12
- try { return await readFile(path, 'utf8'); }
13
- catch (error) { if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null; throw error; }
14
- };
15
- async function object(path: string): Promise<RecordValue> {
16
- const value: unknown = JSON.parse(await optional(path) ?? '{}');
17
- if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`Invalid Scout installation metadata at ${path}; nothing changed.`);
18
- return value as RecordValue;
19
- }
20
- export function piPaths(root: string, scope: ScoutScope) {
21
- const agentDir = resolve(process.env.PI_CODING_AGENT_DIR ?? join(homedir(), '.pi/agent'));
22
- const base = scope === 'user' ? agentDir : join(root, '.pi');
23
- return { skill: join(base, 'skills/openmerit-scout/SKILL.md'), rule: join(base, 'extensions/openmerit-scout.ts'),
24
- connection: join(base, 'extensions/openmerit-scout.json'), record: join(base, 'openmerit-install.json'),
25
- legacyRule: join(base, 'rules/openmerit.md'), mcpFile: join(base, 'mcp.json'), mcpScope: scope, agentDir };
26
- }
27
- export async function inspectPi(root: string, scope: ScoutScope) {
28
- const paths = piPaths(root, scope);
29
- const otherPaths = piPaths(root, scope === 'user' ? 'project' : 'user');
30
- const [record, connection, installedSkill, installedRule, mcp, otherMcp, other] = await Promise.all([
31
- object(paths.record), optional(paths.connection), optional(paths.skill), optional(paths.rule), object(paths.mcpFile),
32
- object(otherPaths.mcpFile), optional(otherPaths.rule),
33
- ]);
34
- const owned = record.version === 1 && record.client === 'pi' && record.scope === scope;
35
- const matches = (value: string | null, field: string) => value === null || owned && [record[field], record[`pending${field[0]!.toUpperCase()}${field.slice(1)}`]].includes(hash(value));
36
- return { paths, record, owned, current: connection, installedSkill, installedRule,
37
- serverOwned: matches(connection, 'serverHash'), skillOwned: matches(installedSkill, 'skillHash'), ruleOwned: matches(installedRule, 'ruleHash'),
38
- updates: { saved: null, active: false, conflicts: [] as string[] }, legacyRule: 'none',
39
- shadowed: other !== null || [mcp, otherMcp].some(config => config.mcpServers && typeof config.mcpServers === 'object' && 'openmerit' in config.mcpServers),
40
- };
41
- }
42
- export function piConflicts(state: Awaited<ReturnType<typeof inspectPi>>): string[] {
43
- return [
44
- ...(!state.serverOwned ? [`${state.paths.connection} was edited or is unowned.`] : []),
45
- ...(!state.skillOwned ? [`${state.paths.skill} was edited or is unowned.`] : []),
46
- ...(!state.ruleOwned ? [`${state.paths.rule} was edited or is unowned.`] : []),
47
- ...(state.shadowed ? ['Another Pi Scout installation or an openmerit MCP entry shadows this installation. Resolve the duplicate before installing.'] : []),
48
- ];
49
- }
50
- export async function configurePi(root: string, scope: ScoutScope, credentialsPath: string, ids: RecordValue) {
51
- const state = await inspectPi(root, scope);
52
- const conflicts = piConflicts(state);
53
- if (conflicts.length) throw new Error(conflicts.join('\n'));
54
- const extension = await readFile(new URL('../../../dist/pi/scout-extension.mjs', import.meta.url), 'utf8');
55
- const connection = `${JSON.stringify({ credentialsPath }, null, 2)}\n`;
56
- const skill = scoutSkill('pi');
57
- const next = { version: 1, client: 'pi', scope, ...ids, serverHash: hash(connection), skillHash: hash(skill), ruleHash: hash(extension), installedAt: new Date().toISOString() };
58
- const save = async (path: string, value: string | RecordValue) => {
59
- await mkdir(dirname(path), { recursive: true, mode: 0o700 });
60
- await writeFile(path, typeof value === 'string' ? value : `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
61
- };
62
- // Record intent before writing files, so interrupted installs are recoverable.
63
- await save(state.paths.record, { ...state.record, version: 1, client: 'pi', scope,
64
- pendingServerHash: next.serverHash, pendingSkillHash: next.skillHash, pendingRuleHash: next.ruleHash });
65
- await save(state.paths.connection, connection);
66
- await save(state.paths.skill, skill);
67
- await save(state.paths.rule, extension);
68
- await save(state.paths.record, next);
69
- }
70
- export async function removePi(state: Awaited<ReturnType<typeof inspectPi>>) {
71
- if (state.current !== null && state.serverOwned) await rm(state.paths.connection, { force: true });
72
- if (state.installedRule !== null && state.ruleOwned) await rm(state.paths.rule, { force: true });
73
- if (state.installedSkill !== null && state.skillOwned) await rm(state.paths.skill, { force: true });
74
- // Keep ownership of edited files, so a future install does not adopt them.
75
- if (state.serverOwned && state.ruleOwned && state.skillOwned) await rm(state.paths.record, { force: true });
76
- }
@@ -1,77 +0,0 @@
1
- export declare const policyFile = ".openmerit/scout.json";
2
- export declare const components: readonly ["input", "work", "output", "criteria", "constraints"];
3
- export declare const help = "OpenMerit \u00B7 Scout\n\n openmerit scout \"<the work models need to do>\" [--all-endpoints] [--json]\n openmerit scout --conversation <id> [--brief <component>=<text> ...] [--skip] [\"<follow-up>\"] [--json]\n\nAsks Scout, the agent behind openmerit.ai/scout. Scout may first return a task\nbrief with clarifying questions: answer them with --brief (components: input,\nwork, output, criteria, constraints) in the same conversation, or --skip to let\nScout assume. It then splits the work into tasks and recommends up to five\nmodels for each. A turn usually takes one to three minutes.\n\nA new conversation is limited to the endpoint IDs in .openmerit/scout.json\n(set with openmerit project). --all-endpoints searches the whole catalogue and is\nrefused under the existing-only policy. Uses the device key that\n\"openmerit install\" saved for Claude Code or Pi, this project's first. Never include secrets, source files or customer records.\n";
4
- export interface ScoutRequest {
5
- prompt?: string;
6
- conversationId?: string;
7
- brief?: Partial<Record<typeof components[number], string>>;
8
- skip?: boolean;
9
- endpointIds?: string[];
10
- }
11
- export interface ScoutModel {
12
- rank: number;
13
- id: string;
14
- name: string;
15
- provider: string;
16
- rationale: string;
17
- watchOut: string | null;
18
- quality: {
19
- score: number;
20
- evidence: Array<{
21
- name: string;
22
- rank: number;
23
- of: number;
24
- }>;
25
- };
26
- cost: {
27
- usdPer1kTasks: number;
28
- inputPerMillion: number;
29
- outputPerMillion: number;
30
- route: string;
31
- endpointId: string;
32
- providerModelId: string;
33
- } | null;
34
- latency?: {
35
- seconds: number;
36
- } | null;
37
- recency?: {
38
- released: string;
39
- } | null;
40
- }
41
- type Priority = 'cost' | 'quality' | 'latency' | 'recency';
42
- export interface ScoutResult {
43
- conversationId: string;
44
- status: 'needs_answers' | 'recommended' | 'working' | 'message' | 'error';
45
- next?: string;
46
- brief?: Record<string, {
47
- text: string;
48
- source: string;
49
- }>;
50
- questions?: Array<{
51
- component: string;
52
- question: string;
53
- }>;
54
- assumptions?: string[];
55
- tasks?: Array<{
56
- title: string;
57
- work: string;
58
- summary: string;
59
- priority?: Priority;
60
- models: ScoutModel[];
61
- }>;
62
- text?: string;
63
- error?: string;
64
- }
65
- export declare function askScout(root: string, request: ScoutRequest, signal?: AbortSignal): Promise<ScoutResult>;
66
- export declare function scoutTurn(root: string, request: Omit<ScoutRequest, 'endpointIds'> & {
67
- allEndpoints?: boolean;
68
- }, signal?: AbortSignal): Promise<ScoutResult>;
69
- export declare function renderScout(result: ScoutResult): string;
70
- export declare function main(args: string[], { root, print }?: {
71
- root?: string | undefined;
72
- print?: {
73
- (...data: any[]): void;
74
- (message?: any, ...optionalParams: any[]): void;
75
- } | undefined;
76
- }): Promise<number>;
77
- export {};
@@ -1,164 +0,0 @@
1
- // `openmerit scout`: a conversation with Scout, the agent behind openmerit.ai/scout,
2
- // for coding agents without the MCP server and for people at a terminal. It calls
3
- // the same scout_models MCP tool with this device's key, so every client gets the
4
- // same clarifying questions and per-task shortlists.
5
- import { readFile } from 'node:fs/promises';
6
- import { join } from 'node:path';
7
- import { credentialsPath, readCredentials, type DeviceCredentials } from './scout-credentials.ts';
8
-
9
- export const policyFile = '.openmerit/scout.json';
10
- export const components = ['input', 'work', 'output', 'criteria', 'constraints'] as const;
11
-
12
- export const help = `OpenMerit · Scout
13
-
14
- openmerit scout "<the work models need to do>" [--all-endpoints] [--json]
15
- openmerit scout --conversation <id> [--brief <component>=<text> ...] [--skip] ["<follow-up>"] [--json]
16
-
17
- Asks Scout, the agent behind openmerit.ai/scout. Scout may first return a task
18
- brief with clarifying questions: answer them with --brief (components: input,
19
- work, output, criteria, constraints) in the same conversation, or --skip to let
20
- Scout assume. It then splits the work into tasks and recommends up to five
21
- models for each. A turn usually takes one to three minutes.
22
-
23
- A new conversation is limited to the endpoint IDs in ${policyFile}
24
- (set with openmerit project). --all-endpoints searches the whole catalogue and is
25
- refused under the existing-only policy. Uses the device key that
26
- "openmerit install" saved for Claude Code or Pi, this project's first. Never include secrets, source files or customer records.
27
- `;
28
-
29
- type Json = Record<string, unknown>;
30
- export interface ScoutRequest { prompt?: string; conversationId?: string; brief?: Partial<Record<typeof components[number], string>>; skip?: boolean; endpointIds?: string[] }
31
- export interface ScoutModel {
32
- rank: number; id: string; name: string; provider: string; rationale: string; watchOut: string | null;
33
- quality: { score: number; evidence: Array<{ name: string; rank: number; of: number }> };
34
- cost: { usdPer1kTasks: number; inputPerMillion: number; outputPerMillion: number; route: string; endpointId: string; providerModelId: string } | null;
35
- latency?: { seconds: number } | null; recency?: { released: string } | null;
36
- }
37
- // How a task's models are ordered. Cost (cheapest first among models that reach the quality bar) is the default.
38
- type Priority = 'cost' | 'quality' | 'latency' | 'recency';
39
- export interface ScoutResult {
40
- conversationId: string; status: 'needs_answers' | 'recommended' | 'working' | 'message' | 'error'; next?: string;
41
- brief?: Record<string, { text: string; source: string }>; questions?: Array<{ component: string; question: string }>;
42
- assumptions?: string[]; tasks?: Array<{ title: string; work: string; summary: string; priority?: Priority; models: ScoutModel[] }>;
43
- text?: string; error?: string;
44
- }
45
-
46
- const isObject = (v: unknown): v is Json => !!v && typeof v === 'object' && !Array.isArray(v);
47
- async function readJSON(path: string): Promise<Json | null> {
48
- try { const value: unknown = JSON.parse(await readFile(path, 'utf8')); return isObject(value) ? value : null; }
49
- catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; throw new Error(`Could not read ${path}.`); }
50
- }
51
-
52
- // The device key an installation saved: this project's installation first, then the user's.
53
- function deviceKey(root: string): DeviceCredentials | null {
54
- for (const scope of ['project', 'user'] as const) for (const client of ['claude-code', 'pi'] as const) {
55
- const credentials = readCredentials(credentialsPath(client, scope, root));
56
- if (credentials) return credentials;
57
- }
58
- return null;
59
- }
60
-
61
- // One turn through the scout_models MCP tool. Errors are written for the person or agent running it.
62
- export async function askScout(root: string, request: ScoutRequest, signal?: AbortSignal): Promise<ScoutResult> {
63
- const credentials = deviceKey(root);
64
- if (!credentials) throw new Error('This device is not signed in to OpenMerit. Run: npx -y openmerit@latest install');
65
- const server = credentials.server;
66
- let response: Response;
67
- try {
68
- response = await fetch(`${server}/scout/mcp`, {
69
- method: 'POST', redirect: 'error',
70
- // A turn can take minutes; Scout keeps working if the connection drops.
71
- signal: AbortSignal.any([AbortSignal.timeout(10 * 60_000), ...(signal ? [signal] : [])]),
72
- headers: { ...credentials.headers, 'Content-Type': 'application/json', Authorization: `Bearer ${credentials.key}`, 'X-Scout-Client': 'cli' },
73
- body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'scout_models', arguments: request } }),
74
- });
75
- } catch (error) {
76
- const id = request.conversationId;
77
- throw new Error((error as Error).name === 'TimeoutError' && id ? `Scout is still working. Read the outcome with: openmerit scout --conversation ${id}` : `Could not reach ${server}. Check your connection and retry.`);
78
- }
79
- if (response.status === 401) throw new Error('The device key was rejected. Run: npx -y openmerit@latest login, then install again.');
80
- const body: unknown = await response.json().catch(() => null);
81
- const result = isObject(body) && isObject(body.result) ? body.result : null;
82
- if (!result) throw new Error(`Scout returned HTTP ${response.status}${isObject(body) && isObject(body.error) ? `: ${String(body.error.message)}` : ''}.`);
83
- const text = Array.isArray(result.content) && isObject(result.content[0]) ? String(result.content[0].text) : '';
84
- if (result.isError && !isObject(result.structuredContent)) throw new Error(text || 'Scout could not finish this turn.');
85
- return result.structuredContent as unknown as ScoutResult;
86
- }
87
-
88
- // A turn under the project's access policy. A new conversation is limited to the saved
89
- // inventory; the whole catalogue needs an explicit choice the policy permits.
90
- export async function scoutTurn(root: string, request: Omit<ScoutRequest, 'endpointIds'> & { allEndpoints?: boolean }, signal?: AbortSignal): Promise<ScoutResult> {
91
- const { allEndpoints, ...turn } = request;
92
- if (turn.conversationId) {
93
- if (allEndpoints) throw new Error('--all-endpoints applies when a conversation starts. Start a new conversation instead.');
94
- return askScout(root, turn, signal);
95
- }
96
- const policy = await readJSON(join(root, policyFile));
97
- if (!policy) {
98
- if (!allEndpoints) throw new Error(`${policyFile} is missing. Ask which endpoints this project can use and save them with "npx -y openmerit@latest project --policy existing-only|existing-first --endpoint-id <id> ...", or pass --all-endpoints to search the whole catalogue.`);
99
- return askScout(root, turn, signal);
100
- }
101
- const ids = Array.isArray(policy.endpointIds) && policy.endpointIds.every(id => typeof id === 'string') ? policy.endpointIds as string[] : null;
102
- if (!ids || !['existing-only', 'existing-first'].includes(String(policy.policy))) throw new Error(`${policyFile} is not a valid Scout policy. Save it again with "openmerit project".`);
103
- if (allEndpoints && policy.policy === 'existing-only') throw new Error('This project uses the existing-only policy, so Scout cannot search the whole catalogue.');
104
- return askScout(root, allEndpoints ? turn : { ...turn, endpointIds: ids }, signal);
105
- }
106
-
107
- const money = (usd: number) => `$${usd < 0.1 ? usd.toFixed(3) : usd.toFixed(2)}`;
108
- const rankings: Record<Exclude<Priority, 'cost'>, string> = { quality: 'Highest quality estimate first.', latency: 'Fastest first.', recency: 'Newest first.' };
109
- export function renderScout(result: ScoutResult): string {
110
- const lines: string[] = [];
111
- if (result.status === 'needs_answers') {
112
- lines.push('Scout needs more detail. Its task brief:', '');
113
- for (const c of components) { const part = result.brief?.[c]; lines.push(` ${c.padEnd(12)}${part?.text || '(missing)'}${part ? ` [${part.source}]` : ''}`); }
114
- lines.push('', 'Questions:', ...(result.questions ?? []).map(q => ` ${q.component}: ${q.question}`), '',
115
- `Answer: openmerit scout --conversation ${result.conversationId} --brief <component>="<answer>" ...`,
116
- `Or skip: openmerit scout --conversation ${result.conversationId} --skip`);
117
- } else if (result.status === 'recommended') {
118
- const tasks = result.tasks ?? [];
119
- tasks.forEach((task, i) => {
120
- if (tasks.length > 1) lines.push(`${i + 1}. ${task.title}`, ` ${task.work}`);
121
- // As on the page, the default cheapest-first order is unlabelled; another priority is named with the value it sorts by.
122
- const priority = task.priority ?? 'cost';
123
- if (priority !== 'cost') lines.push(` ${rankings[priority]}`);
124
- for (const m of task.models) {
125
- const price = m.cost ? `${money(m.cost.inputPerMillion)} in / ${money(m.cost.outputPerMillion)} out per 1M via ${m.cost.route} (${m.cost.endpointId})` : 'no priced route';
126
- const metric = priority === 'latency' ? ` · ${m.latency ? `${m.latency.seconds.toFixed(1)}s per task` : 'time unknown'}`
127
- : priority === 'recency' ? ` · ${m.recency ? `released ${m.recency.released}` : 'release date unknown'}` : '';
128
- lines.push(` ${m.rank}. ${m.name}${metric} · quality ${m.quality.score}/100 · ${price}`, ` ${m.rationale}`);
129
- }
130
- lines.push(` ${task.summary}`, '');
131
- });
132
- if (result.assumptions?.length) lines.push('Assumptions:', ...result.assumptions.map(a => ` - ${a}`), '');
133
- lines.push(`Refine: openmerit scout --conversation ${result.conversationId} "<follow-up>"`);
134
- lines.push('Quality is Scout\'s estimate from public benchmarks, not measured accuracy for your application.');
135
- } else if (result.status === 'working') lines.push(`Scout is still working. Read the outcome with: openmerit scout --conversation ${result.conversationId}`);
136
- else lines.push(result.text ?? result.error ?? 'Scout replied without an answer.');
137
- return lines.join('\n');
138
- }
139
-
140
- export async function main(args: string[], { root = process.cwd(), print = console.log } = {}): Promise<number> {
141
- if (args.includes('--help') || args.includes('-h')) { print(help); return 0; }
142
- const request: Parameters<typeof scoutTurn>[1] = {};
143
- const words: string[] = [];
144
- let json = false;
145
- for (let i = 0; i < args.length; i++) {
146
- const arg = args[i] ?? '';
147
- const value = () => { const v = args[++i]; if (v === undefined || v.startsWith('--')) throw new Error(`${arg} needs a value.`); return v; };
148
- if (arg === '--conversation') request.conversationId = value();
149
- else if (arg === '--brief') {
150
- const [component, ...text] = value().split('=');
151
- if (!(components as readonly string[]).includes(component ?? '') || !text.length) throw new Error(`--brief takes <component>=<text>; components: ${components.join(', ')}.`);
152
- request.brief = { ...request.brief, [component as typeof components[number]]: text.join('=') };
153
- } else if (arg === '--skip') request.skip = true;
154
- else if (arg === '--all-endpoints') request.allEndpoints = true;
155
- else if (arg === '--json') json = true;
156
- else if (arg.startsWith('-')) throw new Error(`Unknown option for scout: ${arg}`);
157
- else words.push(arg);
158
- }
159
- if (words.length) request.prompt = words.join(' ');
160
- if (!request.prompt && !request.conversationId) { print(help); return 1; }
161
- const result = await scoutTurn(root, request);
162
- print(json ? JSON.stringify(result, null, 2) : renderScout(result));
163
- return result.status === 'error' ? 1 : 0;
164
- }
@@ -1,95 +0,0 @@
1
- import { createServer } from 'node:http';
2
- import { readFile } from 'node:fs/promises';
3
- import { fileURLToPath } from 'node:url';
4
- import { basename } from 'node:path';
5
- import { createCollector, loadConfig } from './collect.mjs';
6
- import { sampleSnapshot } from './sample.mjs';
7
- import { telemetryEnabled, terminalProperties, sendTerminalEvent, errorProperties } from './telemetry.mjs';
8
-
9
- const assets = new Map([['/', ['index.html', 'text/html']], ['/app.js', ['app.js', 'text/javascript']], ['/styles.css', ['styles.css', 'text/css']], ['/favicon.svg', ['favicon.svg', 'image/svg+xml']]]);
10
- for (const file of ['baseten.png', 'groq.ico', 'cerebras.png', 'qwen.png', 'meta.ico', 'nomic.png', 'baai.png', 'ollama.png', 'lm-studio.png', 'vllm.png', 'anthropic.png', 'google.png', 'deepseek.ico', 'cohere.png', 'openai.png', 'mistral.png', 'openrouter.png', 'huggingface.svg']) assets.set(`/brands/${file}`, [`brands/${file}`, file.endsWith('.svg') ? 'image/svg+xml' : file.endsWith('.ico') ? 'image/x-icon' : 'image/png']);
11
-
12
- export async function startTerminal({ root = process.cwd(), port = 4310, interval = 15, demo = false, collect: customCollect, config, fallbackPort = false } = {}) {
13
- const collect = customCollect ?? (demo ? async () => sampleSnapshot() : createCollector(root, config ?? await loadConfig(root), interval, {backgroundIndex:true}));
14
- let snapshot={schemaVersion:1,mode:demo?'sample':'live',loading:true,observedAt:null,interval,project:{name:basename(root),scope:'Current project'},models:[],records:[],sources:[],inventory:[],hosts:[],deployments:[],capacityPools:[],taskSummaries:[]};
15
- let busy=false,closed=false,timer;
16
- const update=async()=>{
17
- if(busy||closed)return;
18
- busy=true;
19
- try {const next=await collect();if(!closed)snapshot=next;}
20
- catch (error) {if(snapshot.loading)snapshot={...snapshot,loadError:true};sendTerminalEvent('terminal_error',{code:'collection_error',...errorProperties(error)});}
21
- finally {busy=false;}
22
- };
23
- const server = createServer(async (request, response) => {
24
- const address = server.address();
25
- const validHosts = [`127.0.0.1:${address.port}`, `localhost:${address.port}`];
26
- const host = request.headers.host;
27
- const headers = { 'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff', 'Referrer-Policy': 'no-referrer', 'Content-Security-Policy': "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; font-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'" };
28
- const send = (code, body, type = 'application/json') => { response.writeHead(code, { ...headers, 'Content-Type': `${type}; charset=utf-8` }); response.end(body); };
29
- if (!validHosts.includes(host) || (request.headers.origin && request.headers.origin !== `http://${host}`) || request.headers['sec-fetch-site'] === 'cross-site') return send(403, '{"error":"Local access only"}');
30
- let path;
31
- try { path = new URL(request.url, `http://${host}`).pathname; }
32
- catch { return send(400, '{"error":"Invalid request URL"}'); }
33
- if (path === '/api/analytics' && request.method === 'POST') {
34
- if (!telemetryEnabled() || request.headers.dnt === '1' || request.headers['sec-gpc'] === '1' || request.headers['x-scout-analytics'] === 'off') return send(204, '');
35
- if (request.headers['content-type'] !== 'application/json') return send(415, '{}');
36
- if (Number(request.headers['content-length']) > 2048) return send(413, '{}');
37
- let size = 0, body = '', done = false;
38
- const timeout = setTimeout(() => { if (!done) { done = true; send(408, '{}'); request.destroy(); } }, 1000);
39
- request.on('data', chunk => {
40
- if (done) return;
41
- size += chunk.length;
42
- if (size > 2048) { done = true; clearTimeout(timeout); send(413, '{}'); request.resume(); return; }
43
- body += chunk.toString('utf8');
44
- });
45
- request.on('error', () => { done = true; clearTimeout(timeout); });
46
- request.on('end', () => {
47
- clearTimeout(timeout); if (done) return; done = true;
48
- try {
49
- const value = JSON.parse(body), properties = terminalProperties(value.event, value.properties);
50
- if (!properties) return send(400, '{}');
51
- sendTerminalEvent(value.event, properties);
52
- return send(204, '');
53
- } catch { return send(400, '{}'); }
54
- });
55
- return;
56
- }
57
- if (request.method !== 'GET') return send(405, '{"error":"Read-only terminal"}');
58
- if (path === '/api/snapshot') {
59
- const data=collect.current?.(snapshot)??snapshot;
60
- const revision=new URL(request.url,`http://${host}`).searchParams.get('repositoryRevision');
61
- // Browser refreshes can reuse the unchanged reference catalog. The plain
62
- // API still returns a self-contained snapshot for agents and scripts.
63
- if(data.repository&&revision!==null&&String(data.repository.revision)===revision) {
64
- const {references,...repository}=data.repository;
65
- return send(200,JSON.stringify({...data,repository}));
66
- }
67
- return send(200,JSON.stringify(data));
68
- }
69
- if (path === '/api/sample') return send(200, JSON.stringify(sampleSnapshot()));
70
- if (path === '/api/info') return send(200, JSON.stringify({ version: 1, defaultMode: demo ? 'sample' : 'live', interval, telemetry: telemetryEnabled() }));
71
- const asset = assets.get(path);
72
- if (!asset) return send(404, '{"error":"Not found"}');
73
- try { return send(200, await readFile(fileURLToPath(new URL(`../public/${asset[0]}`, import.meta.url))), asset[1]); }
74
- catch { return send(500, '{"error":"Terminal assets unavailable. Run npm run build."}'); }
75
- });
76
- server.on('close', () => { closed=true;clearInterval(timer);void collect.close?.(); });
77
- const listen = selectedPort => new Promise((resolve, reject) => {
78
- const onError = error => { server.off('listening', onListening); reject(error); };
79
- const onListening = () => { server.off('error', onError); resolve(); };
80
- server.once('error', onError); server.once('listening', onListening);
81
- server.listen(selectedPort, '127.0.0.1');
82
- });
83
- try {
84
- try { await listen(port); }
85
- catch (error) {
86
- if (!fallbackPort || error.code !== 'EADDRINUSE') throw error;
87
- await listen(0);
88
- }
89
- }
90
- catch (error) { closed=true;clearInterval(timer);await collect.close?.();throw error; }
91
- // Listen first. The browser can paint while collection and indexing proceed.
92
- void update();
93
- timer=setInterval(update,interval*1000);timer.unref();
94
- return { server, url: `http://127.0.0.1:${server.address().port}`, close: async () => { closed=true;clearInterval(timer);await collect.close?.();await new Promise(resolve=>{server.close(resolve);server.closeIdleConnections();}); } };
95
- }