orcacompass 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +51 -0
- package/README.md +171 -3
- package/data/modelindex-snapshot.json +1537 -0
- package/data/models-snapshot.json +1356 -0
- package/dist/cli.js +906 -0
- package/dist/config.js +272 -0
- package/dist/install.js +56 -0
- package/dist/paths.js +30 -0
- package/dist/placements/agents/antigravity.js +87 -0
- package/dist/placements/agents/claude.js +72 -0
- package/dist/placements/agents/codex.js +66 -0
- package/dist/placements/agents/cursor.js +102 -0
- package/dist/placements/agents/deps.js +19 -0
- package/dist/placements/agents/grok.js +92 -0
- package/dist/placements/agents/list-cache.js +85 -0
- package/dist/placements/agents/muse.js +7 -0
- package/dist/placements/agents/opencode.js +77 -0
- package/dist/placements/agents/orca-catalog.js +34 -0
- package/dist/placements/benchmark.js +70 -0
- package/dist/placements/build.js +94 -0
- package/dist/placements/score.js +70 -0
- package/dist/placements/sources/artificialanalysis.js +133 -0
- package/dist/placements/sources/modelindex.js +84 -0
- package/dist/placements/sources/modelsdev.js +113 -0
- package/dist/placements/table.js +94 -0
- package/dist/placements/tiers.js +136 -0
- package/dist/routing/effort.js +31 -0
- package/dist/routing/environment.js +51 -0
- package/dist/routing/orca-settings.js +111 -0
- package/dist/routing/orca-usage.js +85 -0
- package/dist/routing/pace.js +52 -0
- package/dist/routing/rejection.js +58 -0
- package/dist/routing/rejections.js +18 -0
- package/dist/routing/router.js +552 -0
- package/dist/setup.js +358 -0
- package/dist/system/run.js +128 -0
- package/dist/types.js +8 -0
- package/dist/verify.js +106 -0
- package/dist/version.js +3 -0
- package/package.json +60 -6
- package/skills/orcacompass/SKILL.md +278 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { EFFORT_LADDER, parseEffort } from '../../routing/effort.js';
|
|
3
|
+
import { listModels, savedNote } from './list-cache.js';
|
|
4
|
+
const EFFORT_SUFFIX = /-(none|minimal|low|medium|high|xhigh|extra-high|max)$/;
|
|
5
|
+
const THINKING = /-thinking(?=-|$)/g;
|
|
6
|
+
// Older Cursor Claude ids put the version first: claude-4.6-sonnet is Anthropic's claude-sonnet-4-6.
|
|
7
|
+
const OLD_CLAUDE = /^claude-(\d+(?:\.\d+)?)-(sonnet|opus|haiku)$/;
|
|
8
|
+
/** Cursor bakes effort, speed and thinking into the id: claude-opus-5-thinking-high-fast. */
|
|
9
|
+
export function normalizeCursorId(id) {
|
|
10
|
+
let base = id;
|
|
11
|
+
const fast = base.endsWith('-fast');
|
|
12
|
+
if (fast)
|
|
13
|
+
base = base.slice(0, -'-fast'.length);
|
|
14
|
+
const thinking = base.search(THINKING) >= 0;
|
|
15
|
+
base = base.replace(THINKING, '');
|
|
16
|
+
const m = EFFORT_SUFFIX.exec(base);
|
|
17
|
+
const effort = m ? parseEffort(m[1]) : undefined;
|
|
18
|
+
if (m)
|
|
19
|
+
base = base.slice(0, m.index);
|
|
20
|
+
if (base.startsWith('cursor-'))
|
|
21
|
+
base = base.slice('cursor-'.length);
|
|
22
|
+
const old = OLD_CLAUDE.exec(base);
|
|
23
|
+
if (old)
|
|
24
|
+
base = `claude-${old[2]}-${old[1].replace('.', '-')}`;
|
|
25
|
+
return { base, ...(effort ? { effort } : {}), fast, thinking };
|
|
26
|
+
}
|
|
27
|
+
function parseList(text) {
|
|
28
|
+
const variants = [];
|
|
29
|
+
for (const line of text.split(/\r?\n/)) {
|
|
30
|
+
const m = /^(\S+) - (.+)$/.exec(line.trim());
|
|
31
|
+
if (m)
|
|
32
|
+
variants.push({ id: m[1], label: m[2] });
|
|
33
|
+
}
|
|
34
|
+
return variants;
|
|
35
|
+
}
|
|
36
|
+
// Fast variants burn more usage; thinking matches what --effort means on the claude/codex routes.
|
|
37
|
+
const better = (a, b) => a.fast !== b.fast ? !a.fast : a.thinking !== b.thinking ? a.thinking : false;
|
|
38
|
+
function toLaunchModels(variants, deps) {
|
|
39
|
+
const groups = new Map();
|
|
40
|
+
for (const v of variants) {
|
|
41
|
+
const g = groups.get(v.base) ?? { byEffort: new Map(), cursorTrained: false };
|
|
42
|
+
groups.set(v.base, g);
|
|
43
|
+
if (v.id.startsWith('cursor-'))
|
|
44
|
+
g.cursorTrained = true;
|
|
45
|
+
if (v.effort) {
|
|
46
|
+
const cur = g.byEffort.get(v.effort);
|
|
47
|
+
if (!cur || better(v, cur))
|
|
48
|
+
g.byEffort.set(v.effort, v);
|
|
49
|
+
}
|
|
50
|
+
else if (!g.fallback || better(v, g.fallback)) {
|
|
51
|
+
g.fallback = v;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return [...groups].map(([base, g]) => {
|
|
55
|
+
// The unsuffixed id is Cursor's default, i.e. medium effort.
|
|
56
|
+
if (g.fallback && g.byEffort.size > 0 && !g.byEffort.has('medium'))
|
|
57
|
+
g.byEffort.set('medium', g.fallback);
|
|
58
|
+
// Drop fast-only levels so a request snaps to a non-fast id instead of silently picking one.
|
|
59
|
+
if ([...g.byEffort.values()].some((v) => !v.fast)) {
|
|
60
|
+
for (const [e, v] of g.byEffort)
|
|
61
|
+
if (v.fast)
|
|
62
|
+
g.byEffort.delete(e);
|
|
63
|
+
}
|
|
64
|
+
const efforts = EFFORT_LADDER.filter((e) => g.byEffort.has(e));
|
|
65
|
+
const chosen = [...g.byEffort.values(), ...(g.fallback ? [g.fallback] : [])];
|
|
66
|
+
const meta = deps.index[base];
|
|
67
|
+
const own = g.cursorTrained || base.startsWith('composer') || base === 'auto';
|
|
68
|
+
const bucket = own ? 'bucket:Cursor Models' : 'bucket:Other Models';
|
|
69
|
+
return {
|
|
70
|
+
agent: 'cursor',
|
|
71
|
+
modelId: base,
|
|
72
|
+
...(meta ? { meta } : {}),
|
|
73
|
+
efforts,
|
|
74
|
+
effortSource: 'provider',
|
|
75
|
+
launch: {
|
|
76
|
+
kind: 'variant',
|
|
77
|
+
ids: Object.fromEntries(efforts.map((e) => [e, g.byEffort.get(e).id])),
|
|
78
|
+
...(g.fallback ? { defaultId: g.fallback.id } : {}),
|
|
79
|
+
},
|
|
80
|
+
// Heuristic: Orca only reports the two Cursor buckets, not which models draw on which.
|
|
81
|
+
poolNames: ['monthly', bucket],
|
|
82
|
+
...(chosen.every((v) => v.fast) ? { fast: true } : {}),
|
|
83
|
+
...(chosen.some((v) => v.label.includes('NO ZDR')) ? { noZdr: true } : {}),
|
|
84
|
+
};
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
export function locateCursor(deps) {
|
|
88
|
+
const extra = deps.env.LOCALAPPDATA ? [join(deps.env.LOCALAPPDATA, 'cursor-agent')] : [];
|
|
89
|
+
return deps.find('cursor-agent', extra);
|
|
90
|
+
}
|
|
91
|
+
export async function discoverCursor(deps) {
|
|
92
|
+
const exe = locateCursor(deps);
|
|
93
|
+
if (!exe)
|
|
94
|
+
return { agent: 'cursor', installed: false, models: [], note: 'cursor-agent not found' };
|
|
95
|
+
const listing = await listModels(deps, exe, ['models'], 'cursor', parseList);
|
|
96
|
+
if (!listing.items) {
|
|
97
|
+
const note = listing.signedOut ? `not signed in: ${listing.error}` : `cursor-agent models failed: ${listing.error}`;
|
|
98
|
+
return { agent: 'cursor', installed: true, models: [], note, ...(listing.signedOut ? { signedOut: true } : {}) };
|
|
99
|
+
}
|
|
100
|
+
const note = savedNote('cursor-agent models', listing, deps.now);
|
|
101
|
+
return { agent: 'cursor', installed: true, models: toLaunchModels(listing.items.map((m) => ({ id: m.id, label: m.label ?? '', ...normalizeCursorId(m.id) })), deps), ...(note ? { note } : {}) };
|
|
102
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
2
|
+
/** Reads a UTF-8 file, or undefined when it is missing or unreadable. */
|
|
3
|
+
export function readTextFile(p) {
|
|
4
|
+
try {
|
|
5
|
+
return readFileSync(p, 'utf8');
|
|
6
|
+
}
|
|
7
|
+
catch {
|
|
8
|
+
return undefined;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
/** Names in a directory, or none when it is missing or unreadable. */
|
|
12
|
+
export function listDirectory(p) {
|
|
13
|
+
try {
|
|
14
|
+
return readdirSync(p);
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return [];
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { homeDir } from '../../paths.js';
|
|
3
|
+
import { EFFORT_LADDER, parseEffort } from '../../routing/effort.js';
|
|
4
|
+
import { listModels, savedNote } from './list-cache.js';
|
|
5
|
+
// Orca's effort choices for grok (it passes them on as --reasoning-effort).
|
|
6
|
+
const EFFORTS = ['low', 'medium', 'high', 'xhigh'];
|
|
7
|
+
// Orca's grok catalog seed caps these models; other ids get the full menu, as in Orca.
|
|
8
|
+
const CEILINGS = { 'grok-4.7': 'xhigh', 'grok-4.6': 'xhigh', 'grok-4.5': 'high' };
|
|
9
|
+
/**
|
|
10
|
+
* grok's own model cache (~/.grok/models_cache.json, written by grok from its account's model endpoint):
|
|
11
|
+
* `models` keyed by id, each with `info.reasoning_efforts` and the default `info.reasoning_effort`.
|
|
12
|
+
*/
|
|
13
|
+
export function parseGrokCache(text) {
|
|
14
|
+
let models;
|
|
15
|
+
try {
|
|
16
|
+
models = text === undefined ? undefined : JSON.parse(text)?.models;
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return [];
|
|
20
|
+
}
|
|
21
|
+
if (typeof models !== 'object' || models === null)
|
|
22
|
+
return [];
|
|
23
|
+
const out = [];
|
|
24
|
+
for (const [key, entry] of Object.entries(models)) {
|
|
25
|
+
const info = entry?.info ?? {};
|
|
26
|
+
if (info.hidden === true)
|
|
27
|
+
continue;
|
|
28
|
+
const id = typeof info.id === 'string' ? info.id : key;
|
|
29
|
+
// On xAI's multi-agent models effort sets how many agents run, not how long one thinks: never pass it.
|
|
30
|
+
const multiAgent = /multi-?agent/i.test(id);
|
|
31
|
+
const levels = multiAgent || info.supports_reasoning_effort === false || !Array.isArray(info.reasoning_efforts) ? [] : info.reasoning_efforts;
|
|
32
|
+
const supported = new Set(levels.map((l) => parseEffort(String(l?.value ?? l?.id ?? ''))).filter((e) => e !== undefined));
|
|
33
|
+
const native = typeof info.reasoning_effort === 'string' ? parseEffort(info.reasoning_effort) : undefined;
|
|
34
|
+
out.push({ id, efforts: EFFORT_LADDER.filter((e) => supported.has(e)), ...(native ? { defaultEffort: native } : {}) });
|
|
35
|
+
}
|
|
36
|
+
return out;
|
|
37
|
+
}
|
|
38
|
+
/** `grok models` is text only: bulleted ids (`*` marks the default) under "Available models:", up to a blank line. */
|
|
39
|
+
export function parseGrokModels(text) {
|
|
40
|
+
const lines = text.split(/\r?\n/);
|
|
41
|
+
const header = lines.findIndex((l) => l.trim() === 'Available models:');
|
|
42
|
+
if (header < 0)
|
|
43
|
+
return [];
|
|
44
|
+
const ids = [];
|
|
45
|
+
for (const line of lines.slice(header + 1)) {
|
|
46
|
+
if (line.trim() === '' && ids.length > 0)
|
|
47
|
+
break;
|
|
48
|
+
const m = /^\s*[*-]\s+([^\s(]+)/.exec(line);
|
|
49
|
+
if (m && !ids.includes(m[1]))
|
|
50
|
+
ids.push(m[1]);
|
|
51
|
+
}
|
|
52
|
+
return ids;
|
|
53
|
+
}
|
|
54
|
+
const effortsOf = (id) => (CEILINGS[id] ? EFFORTS.slice(0, EFFORTS.indexOf(CEILINGS[id]) + 1) : EFFORTS);
|
|
55
|
+
function toLaunchModel(m, source, deps) {
|
|
56
|
+
return {
|
|
57
|
+
agent: 'grok',
|
|
58
|
+
modelId: m.id,
|
|
59
|
+
...(deps.index[m.id] ? { meta: deps.index[m.id] } : {}),
|
|
60
|
+
efforts: m.efforts,
|
|
61
|
+
effortSource: source,
|
|
62
|
+
...(m.defaultEffort ? { defaultEffort: m.defaultEffort } : {}),
|
|
63
|
+
launch: { kind: 'flag', model: m.id },
|
|
64
|
+
// xAI billing: a weekly credit window or a monthly budget, never a session window.
|
|
65
|
+
poolNames: ['weekly', 'monthly'],
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
// Orca looks in $GROK_HOME/bin and adds ~/.grok/bin to its PATH; the installer leaves ~/.grok/bin off the
|
|
69
|
+
// PATH of processes started before it ran.
|
|
70
|
+
export function locateGrok(deps) {
|
|
71
|
+
const dirs = [...(deps.env.GROK_HOME ? [join(deps.env.GROK_HOME, 'bin')] : []), join(homeDir(deps.env), '.grok', 'bin')];
|
|
72
|
+
return deps.find('grok', dirs);
|
|
73
|
+
}
|
|
74
|
+
export async function discoverGrok(deps) {
|
|
75
|
+
const grokHome = deps.env.GROK_HOME || join(homeDir(deps.env), '.grok');
|
|
76
|
+
const exe = locateGrok(deps);
|
|
77
|
+
if (!exe)
|
|
78
|
+
return { agent: 'grok', installed: false, models: [], note: 'grok not found' };
|
|
79
|
+
const cachePath = join(grokHome, 'models_cache.json');
|
|
80
|
+
const cached = parseGrokCache(deps.readFile(cachePath));
|
|
81
|
+
if (cached.length > 0) {
|
|
82
|
+
return { agent: 'grok', installed: true, models: cached.map((m) => toLaunchModel(m, 'provider', deps)), note: `models from ${cachePath}` };
|
|
83
|
+
}
|
|
84
|
+
const listing = await listModels(deps, exe, ['models'], 'grok', (text) => parseGrokModels(text).map((id) => ({ id })));
|
|
85
|
+
if (!listing.items) {
|
|
86
|
+
const note = listing.signedOut ? `not signed in: ${listing.error}` : `grok models failed: ${listing.error}`;
|
|
87
|
+
return { agent: 'grok', installed: true, models: [], note, ...(listing.signedOut ? { signedOut: true } : {}) };
|
|
88
|
+
}
|
|
89
|
+
const note = savedNote('grok models', listing, deps.now);
|
|
90
|
+
const models = listing.items.map(({ id }) => toLaunchModel({ id, efforts: effortsOf(id) }, 'orca', deps));
|
|
91
|
+
return { agent: 'grok', installed: true, models, ...(note ? { note } : {}) };
|
|
92
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
/** Output of a listing command that asks the user to sign in, e.g. "Please run grok login". */
|
|
5
|
+
const SIGNED_OUT = /not (logged|signed) in|(log|sign) ?in (is )?required|please (log|sign) ?in|run [\w-]+ login|\blogin\b.*\bfirst\b|unauthori[sz]ed|unauthenticated/i;
|
|
6
|
+
function readSaved(deps, agent) {
|
|
7
|
+
try {
|
|
8
|
+
const saved = JSON.parse(deps.readFile(join(deps.cacheDir, `${agent}-models.json`)) ?? 'null');
|
|
9
|
+
if (!saved || saved.schemaVersion !== 1 || saved.agent !== agent ||
|
|
10
|
+
!Number.isSafeInteger(saved.fetchedAt) || saved.fetchedAt < 0 ||
|
|
11
|
+
!Array.isArray(saved.models) || saved.models.length === 0 ||
|
|
12
|
+
!saved.models.every((m) => {
|
|
13
|
+
if (!m || typeof m !== 'object')
|
|
14
|
+
return false;
|
|
15
|
+
const entry = m;
|
|
16
|
+
return typeof entry.id === 'string' && /^\S+$/.test(entry.id) &&
|
|
17
|
+
(entry.label === undefined || typeof entry.label === 'string');
|
|
18
|
+
}))
|
|
19
|
+
return undefined;
|
|
20
|
+
return { fetchedAt: saved.fetchedAt, models: saved.models.map((m) => ({
|
|
21
|
+
id: m.id, ...(m.label !== undefined ? { label: m.label } : {}),
|
|
22
|
+
})) };
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function save(deps, agent, models, fetchedAt = deps.now) {
|
|
29
|
+
const file = join(deps.cacheDir, `${agent}-models.json`);
|
|
30
|
+
const temporary = `${file}.${randomUUID()}.tmp`;
|
|
31
|
+
try {
|
|
32
|
+
mkdirSync(deps.cacheDir, { recursive: true });
|
|
33
|
+
writeFileSync(temporary, JSON.stringify({ schemaVersion: 1, agent, fetchedAt, models }, null, 2) + '\n');
|
|
34
|
+
renameSync(temporary, file);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
// The saved copy is only a fallback for a failed listing.
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
try {
|
|
41
|
+
unlinkSync(temporary);
|
|
42
|
+
}
|
|
43
|
+
catch { /* renamed or unavailable */ }
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
const firstLine = (s) => s?.split(/\r?\n/).map((l) => l.trim()).find(Boolean);
|
|
47
|
+
/**
|
|
48
|
+
* Runs an agent's model-listing command on every call, so the route sees what the provider offers now.
|
|
49
|
+
* A good listing is saved; when the command fails for any reason but a sign-in prompt, the saved listing
|
|
50
|
+
* is used instead, with its age. A sign-in prompt is reported as `signedOut` and never falls back.
|
|
51
|
+
*/
|
|
52
|
+
export async function listModels(deps, exe, args, agent, parse, env) {
|
|
53
|
+
const r = await deps.run(exe, args, { timeoutMs: 30_000, env });
|
|
54
|
+
const items = r.code === 0 ? parse(r.stdout) : [];
|
|
55
|
+
if (items.length > 0) {
|
|
56
|
+
save(deps, agent, items);
|
|
57
|
+
return { items };
|
|
58
|
+
}
|
|
59
|
+
const error = firstLine(r.stderr) ?? r.error ?? firstLine(r.stdout) ?? `exit code ${r.code}`;
|
|
60
|
+
if (SIGNED_OUT.test(`${r.stdout}\n${r.stderr}`))
|
|
61
|
+
return { error, signedOut: true };
|
|
62
|
+
const saved = readSaved(deps, agent);
|
|
63
|
+
if (saved)
|
|
64
|
+
return { items: saved.models, savedAt: saved.fetchedAt, error };
|
|
65
|
+
// Migrate old stdout caches only when needed, retaining the original observation time.
|
|
66
|
+
const legacy = deps.readFile(join(deps.cacheDir, `${agent}-models.txt`));
|
|
67
|
+
const header = legacy && /^fetchedAt (\d+)\r?\n/.exec(legacy);
|
|
68
|
+
if (legacy && header && Number.isSafeInteger(Number(header[1]))) {
|
|
69
|
+
const models = parse(legacy.slice(header[0].length));
|
|
70
|
+
if (models.length > 0) {
|
|
71
|
+
const fetchedAt = Number(header[1]);
|
|
72
|
+
save(deps, agent, models, fetchedAt);
|
|
73
|
+
return { items: models, savedAt: fetchedAt, error };
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return { error, signedOut: false };
|
|
77
|
+
}
|
|
78
|
+
/** The discovery note for a listing that fell back to its saved copy, e.g. "agy models failed (timed out); using the list from 3h ago". */
|
|
79
|
+
export function savedNote(command, listing, now) {
|
|
80
|
+
if (listing.savedAt === undefined)
|
|
81
|
+
return undefined;
|
|
82
|
+
const hours = Math.floor((now - listing.savedAt) / 3_600_000);
|
|
83
|
+
const age = hours >= 48 ? `${Math.floor(hours / 24)}d` : hours >= 1 ? `${hours}h` : `${Math.max(0, Math.floor((now - listing.savedAt) / 60_000))}m`;
|
|
84
|
+
return `${command} failed (${listing.error}); using the list from ${age} ago`;
|
|
85
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** Orca launches Muse with any --model but has no Muse model list (account-scoped ids), so Muse is detected, never routed. */
|
|
2
|
+
export const locateMuse = (deps) => deps.find('muse');
|
|
3
|
+
export async function discoverMuse(deps) {
|
|
4
|
+
if (!locateMuse(deps))
|
|
5
|
+
return { agent: 'muse', installed: false, models: [], note: 'muse not found on PATH' };
|
|
6
|
+
return { agent: 'muse', installed: true, models: [], note: 'no model list available from Orca; not routed' };
|
|
7
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { homeDir } from '../../paths.js';
|
|
3
|
+
import { normalizeModelId } from '../score.js';
|
|
4
|
+
import { listModels, savedNote } from './list-cache.js';
|
|
5
|
+
// Providers that serve models from this machine.
|
|
6
|
+
const LOCAL = new Set(['ollama', 'lmstudio', 'llamacpp', 'llama.cpp', 'localai', 'vllm']);
|
|
7
|
+
/**
|
|
8
|
+
* `opencode models`: one `provider/model` id per line, for every provider OpenCode can reach (an API key, a
|
|
9
|
+
* subscription or a local server). OpenRouter's `~vendor/x-latest` ids are moving aliases of models it also
|
|
10
|
+
* lists by their own id, so they are skipped.
|
|
11
|
+
*/
|
|
12
|
+
export function parseOpencodeModels(text) {
|
|
13
|
+
const ids = [];
|
|
14
|
+
for (const line of text.split(/\r?\n/)) {
|
|
15
|
+
const id = line.trim();
|
|
16
|
+
if (/^[\w.-]+\/\S+$/.test(id) && !id.includes('/~') && !ids.includes(id))
|
|
17
|
+
ids.push(id);
|
|
18
|
+
}
|
|
19
|
+
return ids;
|
|
20
|
+
}
|
|
21
|
+
/** How a model is paid for: on this machine, free, or per token. */
|
|
22
|
+
export function billingOf(provider, model) {
|
|
23
|
+
if (LOCAL.has(provider))
|
|
24
|
+
return { text: `runs on this machine (${provider})`, rank: 0 };
|
|
25
|
+
if (/[-:]free$/.test(model))
|
|
26
|
+
return { text: `free model through ${provider}`, rank: 1 };
|
|
27
|
+
return { text: `pay per use through ${provider}`, rank: 2 };
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* OpenCode reaches many providers, so one model can be listed more than once (Zen and OpenRouter): each model
|
|
31
|
+
* is kept once, from the cheapest way to run it. Effort is not passed: Orca does not support it for OpenCode.
|
|
32
|
+
*/
|
|
33
|
+
export function toOpencodeModels(ids, deps) {
|
|
34
|
+
const byNorm = new Map();
|
|
35
|
+
for (const m of Object.values(deps.index))
|
|
36
|
+
if (!byNorm.has(normalizeModelId(m.id)))
|
|
37
|
+
byNorm.set(normalizeModelId(m.id), m);
|
|
38
|
+
const best = new Map();
|
|
39
|
+
for (const id of ids) {
|
|
40
|
+
const provider = id.slice(0, id.indexOf('/'));
|
|
41
|
+
const sub = id.slice(provider.length + 1);
|
|
42
|
+
const billing = billingOf(provider, sub);
|
|
43
|
+
// A free copy (glm-5.3-flash-free, cohere/north-mini-code:free) is the same model: rate and keep it as one.
|
|
44
|
+
const base = sub.replace(/[-:]free$/, '');
|
|
45
|
+
const meta = deps.index[base] ?? byNorm.get(normalizeModelId(base));
|
|
46
|
+
const model = {
|
|
47
|
+
agent: 'opencode',
|
|
48
|
+
modelId: meta?.id ?? base,
|
|
49
|
+
...(meta ? { meta } : {}),
|
|
50
|
+
efforts: [],
|
|
51
|
+
effortSource: 'orca',
|
|
52
|
+
launch: { kind: 'flag', model: id },
|
|
53
|
+
// No usage window: Orca tracks no OpenCode limits, and these models cost per token or local resources.
|
|
54
|
+
poolNames: [],
|
|
55
|
+
billing: billing.text,
|
|
56
|
+
};
|
|
57
|
+
const cur = best.get(model.modelId);
|
|
58
|
+
if (!cur || billing.rank < cur.rank)
|
|
59
|
+
best.set(model.modelId, { model, rank: billing.rank });
|
|
60
|
+
}
|
|
61
|
+
return [...best.values()].map((b) => b.model);
|
|
62
|
+
}
|
|
63
|
+
export function locateOpencode(deps) {
|
|
64
|
+
return deps.find('opencode', [join(homeDir(deps.env), '.opencode', 'bin')]);
|
|
65
|
+
}
|
|
66
|
+
export async function discoverOpencode(deps) {
|
|
67
|
+
const exe = locateOpencode(deps);
|
|
68
|
+
if (!exe)
|
|
69
|
+
return { agent: 'opencode', installed: false, models: [], note: 'opencode not found' };
|
|
70
|
+
const listing = await listModels(deps, exe, ['models'], 'opencode', (text) => parseOpencodeModels(text).map((id) => ({ id })));
|
|
71
|
+
if (!listing.items) {
|
|
72
|
+
const note = listing.signedOut ? `not signed in: ${listing.error}` : `opencode models failed: ${listing.error}`;
|
|
73
|
+
return { agent: 'opencode', installed: true, models: [], note, ...(listing.signedOut ? { signedOut: true } : {}) };
|
|
74
|
+
}
|
|
75
|
+
const note = savedNote('opencode models', listing, deps.now);
|
|
76
|
+
return { agent: 'opencode', installed: true, models: toOpencodeModels(listing.items.map((m) => m.id), deps), ...(note ? { note } : {}) };
|
|
77
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { homeDir } from '../../paths.js';
|
|
3
|
+
import { orcaUserDataPath } from '../../routing/orca-settings.js';
|
|
4
|
+
export const description = (v) => typeof v === 'string' && v.trim() !== '' ? { description: v.trim() } : {};
|
|
5
|
+
export const parseJson = (text) => {
|
|
6
|
+
if (text === undefined)
|
|
7
|
+
return undefined;
|
|
8
|
+
try {
|
|
9
|
+
return JSON.parse(text);
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
return undefined;
|
|
13
|
+
}
|
|
14
|
+
};
|
|
15
|
+
/** Orca's internal (undocumented) catalog; the agent's last entry is the newest probe. */
|
|
16
|
+
export function fromOrcaCatalog(text, agent) {
|
|
17
|
+
const entries = parseJson(text)?.entries;
|
|
18
|
+
if (!Array.isArray(entries))
|
|
19
|
+
return [];
|
|
20
|
+
const models = entries.filter((e) => e?.agent === agent).at(-1)?.discovered?.models;
|
|
21
|
+
if (!Array.isArray(models))
|
|
22
|
+
return [];
|
|
23
|
+
return models
|
|
24
|
+
.filter((m) => typeof m?.id === 'string')
|
|
25
|
+
.map((m) => ({
|
|
26
|
+
id: typeof m.resolvedModel === 'string' ? m.resolvedModel : m.id,
|
|
27
|
+
efforts: (m.efforts ?? []).map((e) => String(e?.value)),
|
|
28
|
+
}));
|
|
29
|
+
}
|
|
30
|
+
/** The catalog sits in Orca's userData folder; a Windows machine without APPDATA gets the usual location. */
|
|
31
|
+
export function orcaCatalogPath(env) {
|
|
32
|
+
const home = homeDir(env);
|
|
33
|
+
return join(orcaUserDataPath(env, process.platform, home) ?? join(home, 'AppData', 'Roaming', 'orca'), 'agent-model-catalog.json');
|
|
34
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { compact, normalizeModelId, stripVersion } from './score.js';
|
|
2
|
+
import { TIERS } from '../types.js';
|
|
3
|
+
// Benchmark placement: a model's Artificial Analysis capability relative to the best model measured,
|
|
4
|
+
// nudged by its intelligence per token cost, and cut into tiers at fixed fractions of the top.
|
|
5
|
+
/** BENCHMARK_CUTS[i] separates TIERS[i] from TIERS[i + 1], as a fraction of the top capability. */
|
|
6
|
+
export const BENCHMARK_CUTS = [0.55, 0.78, 0.93];
|
|
7
|
+
/** Placement shift per tenfold difference from the median points per dollar: dear models up, cheap ones down. */
|
|
8
|
+
export const VALUE_WEIGHT = 0.05;
|
|
9
|
+
/** Intelligence and engineering index averaged, or intelligence alone. */
|
|
10
|
+
export const capability = (b) => (b.engineering === undefined ? b.intelligence : (b.intelligence + b.engineering) / 2);
|
|
11
|
+
/** USD per million tokens blended 3:1 input to output, Artificial Analysis's own convention. */
|
|
12
|
+
export function blendedPrice(b) {
|
|
13
|
+
return b.in !== undefined && b.out !== undefined ? (3 * b.in + b.out) / 4 : undefined;
|
|
14
|
+
}
|
|
15
|
+
/** Capability points per blended dollar: intelligence per token cost. */
|
|
16
|
+
export function pointsPerDollar(b) {
|
|
17
|
+
const price = blendedPrice(b);
|
|
18
|
+
return price === undefined ? undefined : capability(b) / price;
|
|
19
|
+
}
|
|
20
|
+
function median(values) {
|
|
21
|
+
if (values.length === 0)
|
|
22
|
+
return undefined;
|
|
23
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
24
|
+
const mid = sorted.length / 2;
|
|
25
|
+
return sorted.length % 2 ? sorted[Math.floor(mid)] : (sorted[mid - 1] + sorted[mid]) / 2;
|
|
26
|
+
}
|
|
27
|
+
export function benchmarkScale(benchmarks) {
|
|
28
|
+
if (benchmarks.length === 0)
|
|
29
|
+
return undefined;
|
|
30
|
+
const values = benchmarks.map(pointsPerDollar).filter((v) => v !== undefined);
|
|
31
|
+
return { top: Math.max(...benchmarks.map(capability)), medianValue: median(values), benchmarks };
|
|
32
|
+
}
|
|
33
|
+
const EFFORT_SUFFIX = /-(none|minimal|low|medium|high|xhigh|max|non-reasoning|thinking)$/;
|
|
34
|
+
const DATE_SUFFIX = /-\d{8}$/;
|
|
35
|
+
function lookup(id, benchmarks) {
|
|
36
|
+
return (benchmarks.find((b) => b.slugs.some((s) => normalizeModelId(s) === id)) ??
|
|
37
|
+
benchmarks.find((b) => b.slugs.some((s) => compact(normalizeModelId(s)) === compact(id))));
|
|
38
|
+
}
|
|
39
|
+
/** The benchmark of a routed model id: the effort variant's slug counts as the model's. */
|
|
40
|
+
export function matchBenchmark(modelId, benchmarks) {
|
|
41
|
+
const id = normalizeModelId(modelId);
|
|
42
|
+
const candidates = [id, id.replace(DATE_SUFFIX, ''), id.replace(EFFORT_SUFFIX, ''), stripVersion(id)];
|
|
43
|
+
for (const c of new Set(candidates)) {
|
|
44
|
+
const hit = lookup(c, benchmarks);
|
|
45
|
+
if (hit)
|
|
46
|
+
return hit;
|
|
47
|
+
}
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
/** capability / top + VALUE_WEIGHT × log10(median value / value); no value term without a price. */
|
|
51
|
+
export function placement(scale, b) {
|
|
52
|
+
const relative = capability(b) / scale.top;
|
|
53
|
+
const value = pointsPerDollar(b);
|
|
54
|
+
if (value === undefined || scale.medianValue === undefined)
|
|
55
|
+
return relative;
|
|
56
|
+
return relative + VALUE_WEIGHT * Math.log10(scale.medianValue / value);
|
|
57
|
+
}
|
|
58
|
+
const fixed = (n, digits = 2) => n.toFixed(digits);
|
|
59
|
+
/** The tier a placement lands in, with the numbers behind it in one line. */
|
|
60
|
+
export function benchmarkTier(scale, b) {
|
|
61
|
+
const p = placement(scale, b);
|
|
62
|
+
const tier = TIERS[BENCHMARK_CUTS.filter((cut) => cut <= p).length];
|
|
63
|
+
const indices = b.engineering === undefined ? `${b.intelligence}` : `${b.intelligence}/${b.engineering}`;
|
|
64
|
+
const value = pointsPerDollar(b);
|
|
65
|
+
const priced = value === undefined || scale.medianValue === undefined
|
|
66
|
+
? 'no price'
|
|
67
|
+
: `${fixed(value, 1)} points per dollar vs median ${fixed(scale.medianValue, 1)}`;
|
|
68
|
+
const detail = `benchmarks (Artificial Analysis ${indices} = ${fixed(capability(b) / scale.top)} of top, ${priced} → ${fixed(p)}; cuts ${BENCHMARK_CUTS.map((c) => fixed(c)).join('/')})`;
|
|
69
|
+
return { tier, placement: p, detail };
|
|
70
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { cacheDir } from '../paths.js';
|
|
2
|
+
import { findExecutable, runCommand } from '../system/run.js';
|
|
3
|
+
import { AGENT_IDS } from '../types.js';
|
|
4
|
+
import { VERSION } from '../version.js';
|
|
5
|
+
import { discoverAntigravity, locateAntigravity } from './agents/antigravity.js';
|
|
6
|
+
import { discoverClaude, locateClaude } from './agents/claude.js';
|
|
7
|
+
import { discoverCodex, locateCodex } from './agents/codex.js';
|
|
8
|
+
import { discoverCursor, locateCursor } from './agents/cursor.js';
|
|
9
|
+
import { listDirectory, readTextFile } from './agents/deps.js';
|
|
10
|
+
import { discoverGrok, locateGrok } from './agents/grok.js';
|
|
11
|
+
import { discoverMuse, locateMuse } from './agents/muse.js';
|
|
12
|
+
import { discoverOpencode, locateOpencode } from './agents/opencode.js';
|
|
13
|
+
import { benchmarkScale } from './benchmark.js';
|
|
14
|
+
import { AA_KEY_ENV, loadBenchmarks } from './sources/artificialanalysis.js';
|
|
15
|
+
import { loadRatings } from './sources/modelindex.js';
|
|
16
|
+
import { loadCatalog } from './sources/modelsdev.js';
|
|
17
|
+
import { builtWith, writeTable } from './table.js';
|
|
18
|
+
import { classifyLaunch, scoreScale } from './tiers.js';
|
|
19
|
+
// Part 1 of OrcaCompass: every source at once, every model placed, one table written.
|
|
20
|
+
/** Each agent's executable lookup and discovery; the freshness check and the build use the same lookup. */
|
|
21
|
+
export const AGENTS = {
|
|
22
|
+
claude: { locate: locateClaude, discover: discoverClaude },
|
|
23
|
+
codex: { locate: locateCodex, discover: discoverCodex },
|
|
24
|
+
cursor: { locate: locateCursor, discover: discoverCursor },
|
|
25
|
+
grok: { locate: locateGrok, discover: discoverGrok },
|
|
26
|
+
antigravity: { locate: locateAntigravity, discover: discoverAntigravity },
|
|
27
|
+
opencode: { locate: locateOpencode, discover: discoverOpencode },
|
|
28
|
+
muse: { locate: locateMuse, discover: discoverMuse },
|
|
29
|
+
};
|
|
30
|
+
/** The agents whose executable is found right now, looked up exactly as discovery does. */
|
|
31
|
+
export const installedAgents = (deps) => AGENT_IDS.filter((a) => AGENTS[a].locate(deps) !== undefined);
|
|
32
|
+
/** The scales the placements draw on, computed once over the whole datasets. */
|
|
33
|
+
export function placementScales(ratings, benchmarks, config) {
|
|
34
|
+
const score = scoreScale(ratings, config.scoreWeight);
|
|
35
|
+
const benchmark = benchmarks && benchmarkScale(benchmarks);
|
|
36
|
+
return { ...(score ? { score } : {}), ...(benchmark ? { benchmark } : {}) };
|
|
37
|
+
}
|
|
38
|
+
/** Pure: every discovered model with its placements, in the input order. */
|
|
39
|
+
export function placeModels(discoveries, config, scales) {
|
|
40
|
+
return discoveries.map((d) => ({
|
|
41
|
+
agent: d.agent,
|
|
42
|
+
installed: d.installed,
|
|
43
|
+
models: d.models.map((model) => {
|
|
44
|
+
const placements = classifyLaunch(model, config, scales);
|
|
45
|
+
return { ...model, placements, tiers: placements.map((p) => p.tier) };
|
|
46
|
+
}),
|
|
47
|
+
...(d.note !== undefined ? { note: d.note } : {}),
|
|
48
|
+
...(d.signedOut !== undefined ? { signedOut: d.signedOut } : {}),
|
|
49
|
+
}));
|
|
50
|
+
}
|
|
51
|
+
/** Loads the three sources, discovers every agent, places every model and writes the table. */
|
|
52
|
+
export async function buildPlacements(config, deps = {}) {
|
|
53
|
+
const now = deps.now ?? Date.now();
|
|
54
|
+
const env = deps.env ?? process.env;
|
|
55
|
+
const opts = { ttlHours: config.modelsDevTtlHours, now, fetchImpl: deps.fetchImpl, refresh: deps.refresh };
|
|
56
|
+
const catalogP = loadCatalog(opts);
|
|
57
|
+
// Scores only add tiers for models outside the family table, so a failure must not stop the build.
|
|
58
|
+
const ratingsP = loadRatings(opts).catch(() => undefined);
|
|
59
|
+
// Benchmarks need the user's own Artificial Analysis key; without one, the cache from an earlier refresh still serves.
|
|
60
|
+
const benchmarksP = loadBenchmarks({ ...opts, key: env[AA_KEY_ENV] }).catch(() => undefined);
|
|
61
|
+
const catalog = await catalogP;
|
|
62
|
+
const agentDeps = {
|
|
63
|
+
index: catalog.index,
|
|
64
|
+
run: deps.run ?? runCommand,
|
|
65
|
+
find: deps.find ?? findExecutable,
|
|
66
|
+
readFile: deps.readFile ?? readTextFile,
|
|
67
|
+
listDir: deps.listDir ?? listDirectory,
|
|
68
|
+
env,
|
|
69
|
+
cacheDir: deps.cacheDir ?? cacheDir(),
|
|
70
|
+
now,
|
|
71
|
+
};
|
|
72
|
+
const [discoveries, ratings, benchmarks] = await Promise.all([
|
|
73
|
+
Promise.all(AGENT_IDS.map((a) => AGENTS[a].discover(agentDeps))),
|
|
74
|
+
ratingsP,
|
|
75
|
+
benchmarksP,
|
|
76
|
+
]);
|
|
77
|
+
const scales = placementScales(ratings?.ratings, benchmarks?.benchmarks, config);
|
|
78
|
+
const table = {
|
|
79
|
+
schemaVersion: 1,
|
|
80
|
+
version: deps.version ?? VERSION,
|
|
81
|
+
builtAt: now,
|
|
82
|
+
builtWith: builtWith(config),
|
|
83
|
+
sources: {
|
|
84
|
+
catalog: { source: catalog.source, fetchedAt: catalog.fetchedAt, models: Object.keys(catalog.index).length },
|
|
85
|
+
scores: ratings ? { source: ratings.source, fetchedAt: ratings.fetchedAt, models: ratings.ratings.length } : null,
|
|
86
|
+
benchmarks: benchmarks ? { source: benchmarks.source, fetchedAt: benchmarks.fetchedAt, models: benchmarks.benchmarks.length } : null,
|
|
87
|
+
benchmarkScale: scales.benchmark ? { top: scales.benchmark.top, medianValue: scales.benchmark.medianValue ?? null } : null,
|
|
88
|
+
scoreScale: scales.score ? { cuts: scales.score.cuts } : null,
|
|
89
|
+
},
|
|
90
|
+
agents: placeModels(discoveries, config, scales),
|
|
91
|
+
};
|
|
92
|
+
const saveError = writeTable(table, agentDeps.cacheDir);
|
|
93
|
+
return { table, ...(saveError !== undefined ? { saveError } : {}) };
|
|
94
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { TIERS } from '../types.js';
|
|
2
|
+
/** Lowercase, no provider prefix, dashes only: anthropic/claude-opus-5.5 → claude-opus-5-5. */
|
|
3
|
+
export function normalizeModelId(id) {
|
|
4
|
+
return id
|
|
5
|
+
.toLowerCase()
|
|
6
|
+
.replace(/^[^/]*\//, '')
|
|
7
|
+
.replace(/[._\s]+/g, '-')
|
|
8
|
+
.replace(/-{2,}/g, '-')
|
|
9
|
+
.replace(/^-|-$/g, '');
|
|
10
|
+
}
|
|
11
|
+
export const compact = (id) => id.replace(/-/g, '');
|
|
12
|
+
/** Drops a trailing ".0" version (and a "1.0" after a name): muse-wonder-1-0 → muse-wonder, grok-4-6-0 → grok-4-6. */
|
|
13
|
+
export function stripVersion(id) {
|
|
14
|
+
const parts = id.split('-');
|
|
15
|
+
const n = parts.length;
|
|
16
|
+
while (parts.length > 1 && parts.at(-1) === '0')
|
|
17
|
+
parts.pop();
|
|
18
|
+
if (parts.length < n && parts.length > 2 && parts.at(-1) === '1' && !/^\d+$/.test(parts.at(-2)))
|
|
19
|
+
parts.pop();
|
|
20
|
+
return parts.join('-');
|
|
21
|
+
}
|
|
22
|
+
function keysOf(r) {
|
|
23
|
+
const keys = [normalizeModelId(r.id)];
|
|
24
|
+
if (r.orId) {
|
|
25
|
+
const or = normalizeModelId(r.orId);
|
|
26
|
+
keys.push(or, or.replace(/-preview$/, ''));
|
|
27
|
+
}
|
|
28
|
+
return keys;
|
|
29
|
+
}
|
|
30
|
+
function lookup(id, ratings) {
|
|
31
|
+
// Exact keys first (entry id before OpenRouter id), then Cursor's dashless spelling (deepseek-v41-flash).
|
|
32
|
+
return (ratings.find((r) => normalizeModelId(r.id) === id) ??
|
|
33
|
+
ratings.find((r) => keysOf(r).includes(id)) ??
|
|
34
|
+
ratings.find((r) => keysOf(r).some((k) => compact(k) === compact(id))));
|
|
35
|
+
}
|
|
36
|
+
/** The Model Index entry for a routed model id, or undefined. */
|
|
37
|
+
export function matchRating(modelId, ratings) {
|
|
38
|
+
const id = normalizeModelId(modelId);
|
|
39
|
+
const hit = lookup(id, ratings);
|
|
40
|
+
if (hit)
|
|
41
|
+
return hit;
|
|
42
|
+
const stripped = stripVersion(id);
|
|
43
|
+
return stripped !== id ? lookup(stripped, ratings) : undefined;
|
|
44
|
+
}
|
|
45
|
+
const unit = (v, range) => (v - range.min) / (range.max - range.min);
|
|
46
|
+
/** c = w·score + (1 − w)·ln(price), both min-max normalized over the anchors. */
|
|
47
|
+
export function combinedScore(scale, r) {
|
|
48
|
+
return scale.weight * unit(r.score, scale.score) + (1 - scale.weight) * unit(Math.log(r.out), scale.lnPrice);
|
|
49
|
+
}
|
|
50
|
+
const span = (values) => ({ min: Math.min(...values), max: Math.max(...values) });
|
|
51
|
+
/** Undefined unless every tier has an anchor and anchors differ in score and price. */
|
|
52
|
+
export function buildScale(anchors, weight, ratings) {
|
|
53
|
+
if (!TIERS.every((t) => anchors.some((a) => a.tier === t)))
|
|
54
|
+
return undefined;
|
|
55
|
+
const score = span(anchors.map((a) => a.rating.score));
|
|
56
|
+
const lnPrice = span(anchors.map((a) => Math.log(a.rating.out)));
|
|
57
|
+
if (score.max === score.min || lnPrice.max === lnPrice.min)
|
|
58
|
+
return undefined;
|
|
59
|
+
const scale = { weight, score, lnPrice, cuts: [], ratings };
|
|
60
|
+
const cs = (t) => anchors.filter((a) => a.tier === t).map((a) => combinedScore(scale, a.rating));
|
|
61
|
+
for (let i = 1; i < TIERS.length; i++) {
|
|
62
|
+
scale.cuts.push((Math.max(...cs(TIERS[i - 1])) + Math.min(...cs(TIERS[i]))) / 2);
|
|
63
|
+
}
|
|
64
|
+
return scale;
|
|
65
|
+
}
|
|
66
|
+
/** Tier = number of cuts at or below c, capped at Advanced: a score never makes a model Frontier. */
|
|
67
|
+
export function scoreTier(scale, c) {
|
|
68
|
+
const above = scale.cuts.filter((cut) => cut <= c).length;
|
|
69
|
+
return TIERS[Math.min(above, TIERS.indexOf('advanced'))];
|
|
70
|
+
}
|