orcacompass 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +51 -0
- package/README.md +171 -3
- package/data/modelindex-snapshot.json +1537 -0
- package/data/models-snapshot.json +1356 -0
- package/dist/cli.js +906 -0
- package/dist/config.js +272 -0
- package/dist/install.js +56 -0
- package/dist/paths.js +30 -0
- package/dist/placements/agents/antigravity.js +87 -0
- package/dist/placements/agents/claude.js +72 -0
- package/dist/placements/agents/codex.js +66 -0
- package/dist/placements/agents/cursor.js +102 -0
- package/dist/placements/agents/deps.js +19 -0
- package/dist/placements/agents/grok.js +92 -0
- package/dist/placements/agents/list-cache.js +85 -0
- package/dist/placements/agents/muse.js +7 -0
- package/dist/placements/agents/opencode.js +77 -0
- package/dist/placements/agents/orca-catalog.js +34 -0
- package/dist/placements/benchmark.js +70 -0
- package/dist/placements/build.js +94 -0
- package/dist/placements/score.js +70 -0
- package/dist/placements/sources/artificialanalysis.js +133 -0
- package/dist/placements/sources/modelindex.js +84 -0
- package/dist/placements/sources/modelsdev.js +113 -0
- package/dist/placements/table.js +94 -0
- package/dist/placements/tiers.js +136 -0
- package/dist/routing/effort.js +31 -0
- package/dist/routing/environment.js +51 -0
- package/dist/routing/orca-settings.js +111 -0
- package/dist/routing/orca-usage.js +85 -0
- package/dist/routing/pace.js +52 -0
- package/dist/routing/rejection.js +58 -0
- package/dist/routing/rejections.js +18 -0
- package/dist/routing/router.js +552 -0
- package/dist/setup.js +358 -0
- package/dist/system/run.js +128 -0
- package/dist/types.js +8 -0
- package/dist/verify.js +106 -0
- package/dist/version.js +3 -0
- package/package.json +60 -6
- package/skills/orcacompass/SKILL.md +278 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { cacheDir } from '../../paths.js';
|
|
4
|
+
// Artificial Analysis Data API, free tier (artificialanalysis.ai). The data is licensed to the key's
|
|
5
|
+
// owner for internal use only, so it lives in the user's cache and is never bundled with the package.
|
|
6
|
+
export const AA_URL = 'https://artificialanalysis.ai/api/v2/language/models/free';
|
|
7
|
+
export const AA_KEY_ENV = 'ARTIFICIAL_ANALYSIS_API_KEY';
|
|
8
|
+
const FETCH_TIMEOUT_MS = 15_000;
|
|
9
|
+
// The free tier lists about 700 models in 4 pages; the guard only stops a runaway has_more.
|
|
10
|
+
const MAX_PAGES = 10;
|
|
11
|
+
const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
12
|
+
const str = (v) => (typeof v === 'string' && v !== '' ? v : undefined);
|
|
13
|
+
const finite = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : undefined);
|
|
14
|
+
const positive = (v) => {
|
|
15
|
+
const n = finite(v);
|
|
16
|
+
return n !== undefined && n > 0 ? n : undefined;
|
|
17
|
+
};
|
|
18
|
+
/** "Claude Opus 5.5 (Max, Default Fallback)" → "Claude Opus 5.5": the effort lives in the parenthesis. */
|
|
19
|
+
const baseName = (name) => name.replace(/\s*\(.*$/, '').trim();
|
|
20
|
+
function toRow(raw) {
|
|
21
|
+
const slug = str(raw.slug);
|
|
22
|
+
const name = str(raw.name);
|
|
23
|
+
if (!slug || !name)
|
|
24
|
+
return undefined;
|
|
25
|
+
const evals = isObject(raw.evaluations) ? raw.evaluations : {};
|
|
26
|
+
const cost = isObject(raw.artificial_analysis_intelligence_index_cost) ? raw.artificial_analysis_intelligence_index_cost : {};
|
|
27
|
+
const perTask = isObject(cost.cost_per_task) ? cost.cost_per_task : {};
|
|
28
|
+
const pricing = isObject(raw.pricing) ? raw.pricing : {};
|
|
29
|
+
const perf = isObject(raw.performance) ? raw.performance : {};
|
|
30
|
+
const creator = isObject(raw.model_creator) ? str(raw.model_creator.name) : undefined;
|
|
31
|
+
return {
|
|
32
|
+
slug, name, base: baseName(name), creator: creator ?? '', date: str(raw.release_date) ?? '',
|
|
33
|
+
intelligence: finite(evals.artificial_analysis_intelligence_index),
|
|
34
|
+
engineering: finite(evals.artificial_analysis_engineering_index),
|
|
35
|
+
costPerTask: positive(perTask.total_cost),
|
|
36
|
+
in: positive(pricing.price_1m_input_tokens),
|
|
37
|
+
out: positive(pricing.price_1m_output_tokens),
|
|
38
|
+
tokensPerSecond: positive(perf.median_output_tokens_per_second),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
const rowCapability = (r) => (r.engineering === undefined ? r.intelligence : (r.intelligence + r.engineering) / 2);
|
|
42
|
+
/**
|
|
43
|
+
* Trims API pages (or their `data` arrays) to one Benchmark per model: the effort variant with the highest
|
|
44
|
+
* capability represents the model, and every variant's slug is kept for matching. Unscored models are dropped.
|
|
45
|
+
*/
|
|
46
|
+
export function toBenchmarks(pages) {
|
|
47
|
+
const groups = new Map();
|
|
48
|
+
for (const page of pages) {
|
|
49
|
+
const data = Array.isArray(page) ? page : isObject(page) && Array.isArray(page.data) ? page.data : [];
|
|
50
|
+
for (const raw of data) {
|
|
51
|
+
const row = isObject(raw) ? toRow(raw) : undefined;
|
|
52
|
+
if (!row)
|
|
53
|
+
continue;
|
|
54
|
+
const key = `${row.creator}\u0000${row.base}`;
|
|
55
|
+
groups.set(key, [...(groups.get(key) ?? []), row]);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
const benchmarks = [];
|
|
59
|
+
for (const rows of groups.values()) {
|
|
60
|
+
const scored = rows.filter((r) => r.intelligence !== undefined);
|
|
61
|
+
if (scored.length === 0)
|
|
62
|
+
continue;
|
|
63
|
+
const best = scored.sort((a, b) => rowCapability(b) - rowCapability(a))[0];
|
|
64
|
+
const b = { slug: best.slug, slugs: rows.map((r) => r.slug), name: best.base, creator: best.creator, date: best.date, intelligence: best.intelligence };
|
|
65
|
+
if (best.engineering !== undefined)
|
|
66
|
+
b.engineering = best.engineering;
|
|
67
|
+
if (best.costPerTask !== undefined)
|
|
68
|
+
b.costPerTask = best.costPerTask;
|
|
69
|
+
if (best.in !== undefined)
|
|
70
|
+
b.in = best.in;
|
|
71
|
+
if (best.out !== undefined)
|
|
72
|
+
b.out = best.out;
|
|
73
|
+
if (best.tokensPerSecond !== undefined)
|
|
74
|
+
b.tokensPerSecond = best.tokensPerSecond;
|
|
75
|
+
benchmarks.push(b);
|
|
76
|
+
}
|
|
77
|
+
return benchmarks;
|
|
78
|
+
}
|
|
79
|
+
/** Fetches every page with the user's key and trims it; throws on HTTP errors or an empty result. */
|
|
80
|
+
export async function fetchBenchmarks(key, fetchImpl = fetch) {
|
|
81
|
+
const pages = [];
|
|
82
|
+
for (let page = 1; page <= MAX_PAGES; page++) {
|
|
83
|
+
const res = await fetchImpl(`${AA_URL}?page=${page}`, { headers: { 'x-api-key': key }, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) });
|
|
84
|
+
if (!res.ok)
|
|
85
|
+
throw new Error(`artificialanalysis.ai: HTTP ${res.status}`);
|
|
86
|
+
const body = await res.json();
|
|
87
|
+
pages.push(body);
|
|
88
|
+
if (!(isObject(body) && isObject(body.pagination) && body.pagination.has_more === true))
|
|
89
|
+
break;
|
|
90
|
+
}
|
|
91
|
+
const benchmarks = toBenchmarks(pages);
|
|
92
|
+
if (benchmarks.length === 0)
|
|
93
|
+
throw new Error('artificialanalysis.ai: no benchmarked models');
|
|
94
|
+
return benchmarks;
|
|
95
|
+
}
|
|
96
|
+
async function readStored(path) {
|
|
97
|
+
try {
|
|
98
|
+
const data = JSON.parse(await readFile(path, 'utf8'));
|
|
99
|
+
if (isObject(data) && typeof data.fetchedAt === 'number' && Array.isArray(data.benchmarks)) {
|
|
100
|
+
return { fetchedAt: data.fetchedAt, benchmarks: data.benchmarks };
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
// missing or unreadable: treated as absent
|
|
105
|
+
}
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
/** Fresh cache → network with the key (then cache it) → stale cache; undefined without a key and a cache. */
|
|
109
|
+
export async function loadBenchmarks(opts) {
|
|
110
|
+
const now = opts.now ?? Date.now();
|
|
111
|
+
const cacheFile = join(cacheDir(), 'artificialanalysis.json');
|
|
112
|
+
const cached = await readStored(cacheFile);
|
|
113
|
+
if (cached && !opts.refresh && now - cached.fetchedAt < opts.ttlHours * 3_600_000) {
|
|
114
|
+
return { ...cached, source: 'cache' };
|
|
115
|
+
}
|
|
116
|
+
if (opts.key) {
|
|
117
|
+
try {
|
|
118
|
+
const benchmarks = await fetchBenchmarks(opts.key, opts.fetchImpl);
|
|
119
|
+
try {
|
|
120
|
+
await mkdir(cacheDir(), { recursive: true });
|
|
121
|
+
await writeFile(cacheFile, JSON.stringify({ fetchedAt: now, benchmarks }));
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
// an unwritable cache only costs a refetch next time
|
|
125
|
+
}
|
|
126
|
+
return { benchmarks, source: 'network', fetchedAt: now };
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
// fall through to the stale cache
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return cached && { ...cached, source: 'cache' };
|
|
133
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { cacheDir } from '../../paths.js';
|
|
4
|
+
// The Model Index (themodelindex.org), CC BY 4.0: capability scores and OpenRouter prices.
|
|
5
|
+
export const MODEL_INDEX_URL = 'https://themodelindex.org/data/models.json';
|
|
6
|
+
const FETCH_TIMEOUT_MS = 10_000;
|
|
7
|
+
// Same relative path from src/placements/sources/ (Vitest) and dist/placements/sources/ (built).
|
|
8
|
+
const SNAPSHOT_URL = new URL('../../../data/modelindex-snapshot.json', import.meta.url);
|
|
9
|
+
const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
10
|
+
const str = (v) => (typeof v === 'string' && v !== '' ? v : undefined);
|
|
11
|
+
const finite = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : undefined);
|
|
12
|
+
function toRating(raw) {
|
|
13
|
+
const id = str(raw.id);
|
|
14
|
+
const score = isObject(raw.index) ? finite(raw.index.score) : undefined;
|
|
15
|
+
const api = isObject(raw.api) ? raw.api : {};
|
|
16
|
+
const out = finite(api.out);
|
|
17
|
+
if (!id || score === undefined || out === undefined || out <= 0)
|
|
18
|
+
return undefined;
|
|
19
|
+
const r = { id, name: str(raw.name) ?? id, date: str(raw.date) ?? '', score, out };
|
|
20
|
+
const orId = str(api.or_id);
|
|
21
|
+
if (orId)
|
|
22
|
+
r.orId = orId;
|
|
23
|
+
const hf = str(api.hf);
|
|
24
|
+
if (hf)
|
|
25
|
+
r.hf = hf;
|
|
26
|
+
if (typeof raw.open === 'boolean')
|
|
27
|
+
r.open = raw.open;
|
|
28
|
+
return r;
|
|
29
|
+
}
|
|
30
|
+
/** Trims a Model Index models.json payload to scored entries with a positive output price. */
|
|
31
|
+
export function toRatings(raw) {
|
|
32
|
+
if (!isObject(raw) || !Array.isArray(raw.models))
|
|
33
|
+
return [];
|
|
34
|
+
return raw.models.flatMap((m) => (isObject(m) ? (toRating(m) ?? []) : []));
|
|
35
|
+
}
|
|
36
|
+
/** Fetches and trims the Model Index; throws on network/HTTP errors or an empty result. */
|
|
37
|
+
export async function fetchRatings(fetchImpl = fetch) {
|
|
38
|
+
const res = await fetchImpl(MODEL_INDEX_URL, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) });
|
|
39
|
+
if (!res.ok)
|
|
40
|
+
throw new Error(`themodelindex.org: HTTP ${res.status}`);
|
|
41
|
+
const ratings = toRatings(await res.json());
|
|
42
|
+
if (ratings.length === 0)
|
|
43
|
+
throw new Error('themodelindex.org: no rated models');
|
|
44
|
+
return ratings;
|
|
45
|
+
}
|
|
46
|
+
async function readStored(path) {
|
|
47
|
+
try {
|
|
48
|
+
const data = JSON.parse(await readFile(path, 'utf8'));
|
|
49
|
+
if (isObject(data) && typeof data.fetchedAt === 'number' && Array.isArray(data.ratings)) {
|
|
50
|
+
return { fetchedAt: data.fetchedAt, ratings: data.ratings };
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
// missing or unreadable: treated as absent
|
|
55
|
+
}
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
/** Fresh cache → network (then cache it) → stale cache → bundled snapshot; undefined when all fail. */
|
|
59
|
+
export async function loadRatings(opts) {
|
|
60
|
+
const now = opts.now ?? Date.now();
|
|
61
|
+
const cacheFile = join(cacheDir(), 'modelindex.json');
|
|
62
|
+
const cached = await readStored(cacheFile);
|
|
63
|
+
if (cached && !opts.refresh && now - cached.fetchedAt < opts.ttlHours * 3_600_000) {
|
|
64
|
+
return { ...cached, source: 'cache' };
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
const ratings = await fetchRatings(opts.fetchImpl);
|
|
68
|
+
try {
|
|
69
|
+
await mkdir(cacheDir(), { recursive: true });
|
|
70
|
+
await writeFile(cacheFile, JSON.stringify({ fetchedAt: now, ratings }));
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
// an unwritable cache only costs a refetch next time
|
|
74
|
+
}
|
|
75
|
+
return { ratings, source: 'network', fetchedAt: now };
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
if (cached)
|
|
79
|
+
return { ...cached, source: 'cache' };
|
|
80
|
+
}
|
|
81
|
+
// No scores only disables score placement; routing by the family table still works.
|
|
82
|
+
const snapshot = await readStored(SNAPSHOT_URL);
|
|
83
|
+
return snapshot && { ...snapshot, source: 'snapshot' };
|
|
84
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { parseEffort } from '../../routing/effort.js';
|
|
5
|
+
import { cacheDir } from '../../paths.js';
|
|
6
|
+
export const MODELS_DEV_URL = 'https://models.dev/api.json';
|
|
7
|
+
/** Lookup priority: the first provider wins when two share a model id. */
|
|
8
|
+
export const PROVIDERS = ['anthropic', 'openai', 'google', 'xai'];
|
|
9
|
+
const FETCH_TIMEOUT_MS = 10_000;
|
|
10
|
+
// Same relative path from src/placements/sources/ (Vitest) and dist/placements/sources/ (built).
|
|
11
|
+
const SNAPSHOT_URL = new URL('../../../data/models-snapshot.json', import.meta.url);
|
|
12
|
+
const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
13
|
+
const str = (v) => (typeof v === 'string' && v !== '' ? v : undefined);
|
|
14
|
+
function toEfforts(options) {
|
|
15
|
+
if (!Array.isArray(options))
|
|
16
|
+
return undefined;
|
|
17
|
+
const opt = options.find((o) => isObject(o) && o.type === 'effort');
|
|
18
|
+
if (!isObject(opt) || !Array.isArray(opt.values))
|
|
19
|
+
return undefined;
|
|
20
|
+
const efforts = [];
|
|
21
|
+
for (const v of opt.values) {
|
|
22
|
+
const e = typeof v === 'string' ? parseEffort(v) : undefined;
|
|
23
|
+
if (e && !efforts.includes(e))
|
|
24
|
+
efforts.push(e);
|
|
25
|
+
}
|
|
26
|
+
return efforts.length > 0 ? efforts : undefined;
|
|
27
|
+
}
|
|
28
|
+
function toMeta(key, raw, provider) {
|
|
29
|
+
const id = str(raw.id) ?? key;
|
|
30
|
+
const meta = { id, provider, name: str(raw.name) ?? id };
|
|
31
|
+
const family = str(raw.family);
|
|
32
|
+
if (family)
|
|
33
|
+
meta.family = family;
|
|
34
|
+
const releaseDate = str(raw.release_date);
|
|
35
|
+
if (releaseDate)
|
|
36
|
+
meta.releaseDate = releaseDate;
|
|
37
|
+
const output = isObject(raw.cost) ? raw.cost.output : undefined;
|
|
38
|
+
if (typeof output === 'number' && Number.isFinite(output))
|
|
39
|
+
meta.outputCost = output;
|
|
40
|
+
const efforts = toEfforts(raw.reasoning_options);
|
|
41
|
+
if (efforts)
|
|
42
|
+
meta.efforts = efforts;
|
|
43
|
+
return meta;
|
|
44
|
+
}
|
|
45
|
+
/** Trims a Models.dev api.json payload down to the providers and fields we route on. */
|
|
46
|
+
export function toModelIndex(raw) {
|
|
47
|
+
const index = {};
|
|
48
|
+
if (!isObject(raw))
|
|
49
|
+
return index;
|
|
50
|
+
for (const provider of PROVIDERS) {
|
|
51
|
+
const p = raw[provider];
|
|
52
|
+
if (!isObject(p) || !isObject(p.models))
|
|
53
|
+
continue;
|
|
54
|
+
for (const [key, m] of Object.entries(p.models)) {
|
|
55
|
+
if (!isObject(m))
|
|
56
|
+
continue;
|
|
57
|
+
const meta = toMeta(key, m, provider);
|
|
58
|
+
if (!(meta.id in index))
|
|
59
|
+
index[meta.id] = meta;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return index;
|
|
63
|
+
}
|
|
64
|
+
/** Fetches and trims Models.dev; throws on network/HTTP errors or an empty result. */
|
|
65
|
+
export async function fetchModelIndex(fetchImpl = fetch) {
|
|
66
|
+
const res = await fetchImpl(MODELS_DEV_URL, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) });
|
|
67
|
+
if (!res.ok)
|
|
68
|
+
throw new Error(`models.dev: HTTP ${res.status}`);
|
|
69
|
+
const index = toModelIndex(await res.json());
|
|
70
|
+
if (Object.keys(index).length === 0)
|
|
71
|
+
throw new Error('models.dev: no models for known providers');
|
|
72
|
+
return index;
|
|
73
|
+
}
|
|
74
|
+
async function readStored(path) {
|
|
75
|
+
try {
|
|
76
|
+
const data = JSON.parse(await readFile(path, 'utf8'));
|
|
77
|
+
if (isObject(data) && typeof data.fetchedAt === 'number' && isObject(data.index)) {
|
|
78
|
+
return { fetchedAt: data.fetchedAt, index: data.index };
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// missing or unreadable: treated as absent
|
|
83
|
+
}
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
86
|
+
/** Fresh cache → network (then cache it) → stale cache → bundled snapshot. */
|
|
87
|
+
export async function loadCatalog(opts) {
|
|
88
|
+
const now = opts.now ?? Date.now();
|
|
89
|
+
const cacheFile = join(cacheDir(), 'modelsdev.json');
|
|
90
|
+
const cached = await readStored(cacheFile);
|
|
91
|
+
if (cached && !opts.refresh && now - cached.fetchedAt < opts.ttlHours * 3_600_000) {
|
|
92
|
+
return { ...cached, source: 'cache' };
|
|
93
|
+
}
|
|
94
|
+
try {
|
|
95
|
+
const index = await fetchModelIndex(opts.fetchImpl);
|
|
96
|
+
try {
|
|
97
|
+
await mkdir(cacheDir(), { recursive: true });
|
|
98
|
+
await writeFile(cacheFile, JSON.stringify({ fetchedAt: now, index }));
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
// an unwritable cache only costs a refetch next time
|
|
102
|
+
}
|
|
103
|
+
return { index, source: 'network', fetchedAt: now };
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
if (cached)
|
|
107
|
+
return { ...cached, source: 'cache' };
|
|
108
|
+
}
|
|
109
|
+
const snapshot = await readStored(SNAPSHOT_URL);
|
|
110
|
+
if (!snapshot)
|
|
111
|
+
throw new Error(`bundled Models.dev snapshot missing or invalid: ${fileURLToPath(SNAPSHOT_URL)}`);
|
|
112
|
+
return { ...snapshot, source: 'snapshot' };
|
|
113
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { AGENT_IDS, CLASSIFICATION_SOURCES, TIERS } from '../types.js';
|
|
5
|
+
// The placement table: every model's tiers, computed once from every source and reused until stale.
|
|
6
|
+
export const PLACEMENTS_FILE = 'placements.json';
|
|
7
|
+
export const PLACEMENTS_SCHEMA = 1;
|
|
8
|
+
export const builtWith = (config) => structuredClone({ tierOverrides: config.tierOverrides, cursorAuto: config.cursorAuto, scoreWeight: config.scoreWeight });
|
|
9
|
+
const isObj = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
10
|
+
const isInt = (v) => Number.isSafeInteger(v) && v >= 0;
|
|
11
|
+
function validSource(v) {
|
|
12
|
+
return isObj(v) && ['network', 'cache', 'snapshot'].includes(v.source) && isInt(v.fetchedAt) && isInt(v.models);
|
|
13
|
+
}
|
|
14
|
+
/** A launch the router can act on: a flag with a model id, or a variant map (ids may be empty, as for Cursor Auto). */
|
|
15
|
+
function validLaunch(v) {
|
|
16
|
+
return isObj(v) && ((v.kind === 'flag' && typeof v.model === 'string') || (v.kind === 'variant' && isObj(v.ids)));
|
|
17
|
+
}
|
|
18
|
+
function validModel(v) {
|
|
19
|
+
if (!isObj(v))
|
|
20
|
+
return false;
|
|
21
|
+
return typeof v.agent === 'string' && typeof v.modelId === 'string' && Array.isArray(v.efforts) && validLaunch(v.launch) &&
|
|
22
|
+
Array.isArray(v.poolNames) && Array.isArray(v.tiers) &&
|
|
23
|
+
Array.isArray(v.placements) && v.placements.every((p) => isObj(p) && TIERS.includes(p.tier) && CLASSIFICATION_SOURCES.includes(p.source));
|
|
24
|
+
}
|
|
25
|
+
function validAgent(v) {
|
|
26
|
+
return isObj(v) && typeof v.agent === 'string' && typeof v.installed === 'boolean' && Array.isArray(v.models) && v.models.every(validModel);
|
|
27
|
+
}
|
|
28
|
+
/** The table a file holds, or undefined when it is missing, not JSON, or not the shape this version writes. */
|
|
29
|
+
export function parseTable(text) {
|
|
30
|
+
if (text === undefined)
|
|
31
|
+
return undefined;
|
|
32
|
+
let raw;
|
|
33
|
+
try {
|
|
34
|
+
raw = JSON.parse(text);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
if (!isObj(raw) || raw.schemaVersion !== PLACEMENTS_SCHEMA || typeof raw.version !== 'string' || !isInt(raw.builtAt))
|
|
40
|
+
return undefined;
|
|
41
|
+
const b = raw.builtWith;
|
|
42
|
+
if (!isObj(b) || !isObj(b.tierOverrides) || typeof b.cursorAuto !== 'boolean' || typeof b.scoreWeight !== 'number')
|
|
43
|
+
return undefined;
|
|
44
|
+
const s = raw.sources;
|
|
45
|
+
if (!isObj(s) || !validSource(s.catalog) || (s.scores !== null && !validSource(s.scores)) || (s.benchmarks !== null && !validSource(s.benchmarks)))
|
|
46
|
+
return undefined;
|
|
47
|
+
if (!Array.isArray(raw.agents) || !raw.agents.every(validAgent))
|
|
48
|
+
return undefined;
|
|
49
|
+
return raw;
|
|
50
|
+
}
|
|
51
|
+
export function readTable(readFile, dir) {
|
|
52
|
+
const text = readFile(join(dir, PLACEMENTS_FILE));
|
|
53
|
+
return { table: parseTable(text), present: text !== undefined };
|
|
54
|
+
}
|
|
55
|
+
const canonical = (b) => JSON.stringify({ cursorAuto: b.cursorAuto, scoreWeight: b.scoreWeight, tierOverrides: Object.fromEntries(Object.entries(b.tierOverrides).sort()) });
|
|
56
|
+
/** Why the table must be rebuilt, or undefined when it can be used as is. */
|
|
57
|
+
export function staleness(read, opts) {
|
|
58
|
+
if (!read.present)
|
|
59
|
+
return 'missing';
|
|
60
|
+
if (!read.table)
|
|
61
|
+
return 'invalid';
|
|
62
|
+
const t = read.table;
|
|
63
|
+
if (t.version !== opts.version)
|
|
64
|
+
return 'version';
|
|
65
|
+
// A build time in the future (clock set back, hand edit) is as untrustworthy as an old one.
|
|
66
|
+
if (t.builtAt > opts.now || opts.now - t.builtAt >= opts.ttlHours * 3_600_000)
|
|
67
|
+
return 'expired';
|
|
68
|
+
if (canonical(t.builtWith) !== canonical(builtWith(opts.config)))
|
|
69
|
+
return 'config';
|
|
70
|
+
if (t.agents.some((a) => !a.installed && AGENT_IDS.includes(a.agent) && opts.onPath.includes(a.agent)))
|
|
71
|
+
return 'agent-appeared';
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
/** Writes the file atomically; returns the error message when it could not be written. */
|
|
75
|
+
export function writeTable(table, dir) {
|
|
76
|
+
const file = join(dir, PLACEMENTS_FILE);
|
|
77
|
+
const temporary = `${file}.${randomUUID()}.tmp`;
|
|
78
|
+
try {
|
|
79
|
+
mkdirSync(dir, { recursive: true });
|
|
80
|
+
writeFileSync(temporary, JSON.stringify(table, null, 2) + '\n');
|
|
81
|
+
renameSync(temporary, file);
|
|
82
|
+
return undefined;
|
|
83
|
+
}
|
|
84
|
+
catch (e) {
|
|
85
|
+
return e.message;
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
try {
|
|
89
|
+
unlinkSync(temporary);
|
|
90
|
+
}
|
|
91
|
+
catch { /* renamed or never written */ }
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
export const tableAge = (table, now) => now - table.builtAt;
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { benchmarkTier, matchBenchmark } from './benchmark.js';
|
|
2
|
+
import { buildScale, combinedScore, matchRating, normalizeModelId, scoreTier } from './score.js';
|
|
3
|
+
import { TIERS } from '../types.js';
|
|
4
|
+
// Families are stable across releases, so new versions get a tier automatically.
|
|
5
|
+
export const FAMILY_TIERS = {
|
|
6
|
+
'claude-haiku': 'light',
|
|
7
|
+
'claude-sonnet': 'daily',
|
|
8
|
+
'claude-opus': 'advanced',
|
|
9
|
+
'claude-fable': 'frontier',
|
|
10
|
+
'gpt-luna': 'light',
|
|
11
|
+
'gpt-terra': 'daily',
|
|
12
|
+
'gpt-sol': 'advanced',
|
|
13
|
+
'gpt-astra': 'frontier',
|
|
14
|
+
'gemini-flash-lite': 'light',
|
|
15
|
+
};
|
|
16
|
+
/** Model lines matched by id, where the family is too broad: Models.dev files Gemini Argon under `gemini` and every Grok under `grok`. */
|
|
17
|
+
export const MODEL_LINE_TIERS = [
|
|
18
|
+
{ pattern: /^gemini(-\d+)+-argon(-|$)/, tier: 'frontier' },
|
|
19
|
+
// Grok 4.7 to 4.9 and 5+; older releases (4.20 is from early 2026) and Grok Build stay on the score.
|
|
20
|
+
{ pattern: /^grok-(4-[7-9]|[5-9]|\d{2,})(-\d+)*$/, tier: 'advanced' },
|
|
21
|
+
];
|
|
22
|
+
/** Model Index ids of a table family: claude-opus-5-5 for claude-opus, gpt-6-1-sol for gpt-sol. */
|
|
23
|
+
function familyPattern(family) {
|
|
24
|
+
const m = /^(claude|gpt)-([a-z]+)$/.exec(family);
|
|
25
|
+
if (!m)
|
|
26
|
+
return undefined;
|
|
27
|
+
return m[1] === 'claude' ? new RegExp(`^claude-${m[2]}(-\\d+)+$`) : new RegExp(`^gpt(-\\d+)+-${m[2]}$`);
|
|
28
|
+
}
|
|
29
|
+
/** Claude and GPT families: the ladder's own vendors, preferred over other vendors when usage ties. */
|
|
30
|
+
export function isLadderVendor(family) {
|
|
31
|
+
return family !== undefined && familyPattern(family) !== undefined;
|
|
32
|
+
}
|
|
33
|
+
/** The newest Model Index entry of each Claude/GPT table family, with the family's tier. */
|
|
34
|
+
export function findAnchors(ratings) {
|
|
35
|
+
const anchors = [];
|
|
36
|
+
for (const [family, tier] of Object.entries(FAMILY_TIERS)) {
|
|
37
|
+
const pattern = familyPattern(family);
|
|
38
|
+
const newest = pattern && ratings.filter((r) => pattern.test(r.id)).sort((a, b) => (a.date < b.date ? 1 : -1))[0];
|
|
39
|
+
if (newest)
|
|
40
|
+
anchors.push({ tier, rating: newest });
|
|
41
|
+
}
|
|
42
|
+
return anchors;
|
|
43
|
+
}
|
|
44
|
+
/** The score scale calibrated on the family table, or undefined when the data cannot support one. */
|
|
45
|
+
export function scoreScale(ratings, weight) {
|
|
46
|
+
return ratings ? buildScale(findAnchors(ratings), weight, ratings) : undefined;
|
|
47
|
+
}
|
|
48
|
+
const fixed = (n) => n.toFixed(2);
|
|
49
|
+
function byScore(modelId, scale) {
|
|
50
|
+
const r = matchRating(modelId, scale.ratings);
|
|
51
|
+
if (!r)
|
|
52
|
+
return undefined;
|
|
53
|
+
const c = combinedScore(scale, r);
|
|
54
|
+
const detail = `capability score (Model Index ${r.score}, $${r.out}/M → ${fixed(c)}; cuts ${scale.cuts.map(fixed).join('/')})`;
|
|
55
|
+
return { tier: scoreTier(scale, c), source: 'score', detail };
|
|
56
|
+
}
|
|
57
|
+
function byOverride(modelId, meta, config) {
|
|
58
|
+
const { tierOverrides } = config;
|
|
59
|
+
if (Object.hasOwn(tierOverrides, modelId))
|
|
60
|
+
return { tier: tierOverrides[modelId], source: 'override' };
|
|
61
|
+
const family = meta?.family;
|
|
62
|
+
if (family !== undefined && Object.hasOwn(tierOverrides, family))
|
|
63
|
+
return { tier: tierOverrides[family], source: 'override' };
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
/** The built-in model lines, then the family table. */
|
|
67
|
+
function byTable(modelId, meta) {
|
|
68
|
+
const line = MODEL_LINE_TIERS.find((l) => l.pattern.test(normalizeModelId(modelId)));
|
|
69
|
+
if (line)
|
|
70
|
+
return { tier: line.tier, source: 'family' };
|
|
71
|
+
const family = meta?.family;
|
|
72
|
+
if (family !== undefined && Object.hasOwn(FAMILY_TIERS, family))
|
|
73
|
+
return { tier: FAMILY_TIERS[family], source: 'family' };
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
/** Override by id, override by family, built-in model line, built-in family map, then capability score. Undefined when nothing applies. */
|
|
77
|
+
export function classifyModel(modelId, meta, config, scale) {
|
|
78
|
+
return byOverride(modelId, meta, config) ?? byTable(modelId, meta) ?? (scale ? byScore(modelId, scale) : undefined);
|
|
79
|
+
}
|
|
80
|
+
// Providers describe each model's place in their lineup in a few words; the strongest claim wins.
|
|
81
|
+
const LINEUP_WORDS = [
|
|
82
|
+
['frontier', /frontier|toughest|most demanding|most capable|most intelligent|hardest/],
|
|
83
|
+
['light', /\bfast(est)?\b|quick|affordable|easier|cheapest|lightweight|\bsmall/],
|
|
84
|
+
['advanced', /complex|workhorse|demanding/],
|
|
85
|
+
['daily', /simpler|everyday|efficient|balanced|straightforward|general/],
|
|
86
|
+
];
|
|
87
|
+
/** The tier a provider's own description places a model in, or undefined when it says nothing usable. */
|
|
88
|
+
export function lineupTier(description) {
|
|
89
|
+
const text = description?.toLowerCase() ?? '';
|
|
90
|
+
return LINEUP_WORDS.find(([, words]) => words.test(text))?.[0];
|
|
91
|
+
}
|
|
92
|
+
/** Tiers of Cursor's `auto` (Cost mode): Cursor picks a cheap model, so never Advanced or Frontier. */
|
|
93
|
+
export const CURSOR_AUTO_TIERS = ['light', 'daily'];
|
|
94
|
+
/** Cursor's `auto`, routed by the cursorAuto rule rather than a user override. */
|
|
95
|
+
export function isCursorAuto(model, config) {
|
|
96
|
+
return config.cursorAuto && model.agent === 'cursor' && model.modelId === 'auto' && !Object.hasOwn(config.tierOverrides, 'auto');
|
|
97
|
+
}
|
|
98
|
+
const quoted = (s) => `the provider's lineup ("${s}")`;
|
|
99
|
+
/**
|
|
100
|
+
* Every tier a discovered model is routed in. Overrides and Cursor Auto win; then the model's home tier (the
|
|
101
|
+
* provider's lineup words, else the family table) and its benchmark tier span a range, and it is routed in
|
|
102
|
+
* every tier between them. One signal alone places it in one tier; with neither, the capability score does.
|
|
103
|
+
*/
|
|
104
|
+
export function classifyLaunch(model, config, scales = {}) {
|
|
105
|
+
if (isCursorAuto(model, config))
|
|
106
|
+
return CURSOR_AUTO_TIERS.map((tier) => ({ tier, source: 'auto' }));
|
|
107
|
+
const override = byOverride(model.modelId, model.meta, config);
|
|
108
|
+
if (override)
|
|
109
|
+
return [override];
|
|
110
|
+
const lineup = lineupTier(model.lineup);
|
|
111
|
+
const home = lineup
|
|
112
|
+
? { tier: lineup, source: 'lineup', detail: quoted(model.lineup) }
|
|
113
|
+
: byTable(model.modelId, model.meta);
|
|
114
|
+
const matched = scales.benchmark && matchBenchmark(model.modelId, scales.benchmark.benchmarks);
|
|
115
|
+
const bench = matched && scales.benchmark ? benchmarkTier(scales.benchmark, matched) : undefined;
|
|
116
|
+
if (!bench) {
|
|
117
|
+
if (home)
|
|
118
|
+
return [home];
|
|
119
|
+
const score = scales.score && byScore(model.modelId, scales.score);
|
|
120
|
+
return score ? [score] : [];
|
|
121
|
+
}
|
|
122
|
+
if (!home)
|
|
123
|
+
return [{ tier: bench.tier, source: 'benchmark', detail: bench.detail }];
|
|
124
|
+
const homeWords = home.source === 'lineup' ? quoted(model.lineup) : 'the family table';
|
|
125
|
+
if (home.tier === bench.tier)
|
|
126
|
+
return [{ ...home, detail: `${homeWords}, confirmed by ${bench.detail}` }];
|
|
127
|
+
const lo = Math.min(TIERS.indexOf(home.tier), TIERS.indexOf(bench.tier));
|
|
128
|
+
const hi = Math.max(TIERS.indexOf(home.tier), TIERS.indexOf(bench.tier));
|
|
129
|
+
return TIERS.slice(lo, hi + 1).map((tier) => {
|
|
130
|
+
if (tier === home.tier)
|
|
131
|
+
return { tier, source: home.source, detail: `${homeWords}; benchmarks place it ${bench.tier} (${bench.detail.slice('benchmarks ('.length)}` };
|
|
132
|
+
if (tier === bench.tier)
|
|
133
|
+
return { tier, source: 'benchmark', detail: `${bench.detail}; ${homeWords} places it ${home.tier}` };
|
|
134
|
+
return { tier, source: 'benchmark', detail: `between ${homeWords.replace(/ \(".*"\)$/, '')} (${home.tier}) and benchmarks (${bench.tier})` };
|
|
135
|
+
});
|
|
136
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export const EFFORT_LADDER = ['none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'];
|
|
2
|
+
const rank = (e) => EFFORT_LADDER.indexOf(e);
|
|
3
|
+
/** Target if supported, else the nearest lower supported effort, else the lowest higher one. */
|
|
4
|
+
export function snapEffort(target, supported) {
|
|
5
|
+
if (supported.length === 0)
|
|
6
|
+
return null;
|
|
7
|
+
if (supported.includes(target))
|
|
8
|
+
return target;
|
|
9
|
+
const sorted = [...supported].sort((a, b) => rank(a) - rank(b));
|
|
10
|
+
const lower = sorted.filter((e) => rank(e) < rank(target));
|
|
11
|
+
if (lower.length > 0)
|
|
12
|
+
return lower[lower.length - 1];
|
|
13
|
+
return sorted[0];
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Effort per reasoning demand. A starting heuristic, not a measured optimum: `balanced` keeps exceptional
|
|
17
|
+
* work at high, `quality` allows xhigh for it only, `efficient` saves effort on moderate and high work.
|
|
18
|
+
* Nothing maps to max; that takes an explicit --effort.
|
|
19
|
+
*/
|
|
20
|
+
export const DEMAND_EFFORT = {
|
|
21
|
+
efficient: { low: 'low', moderate: 'low', high: 'medium', exceptional: 'high' },
|
|
22
|
+
balanced: { low: 'low', moderate: 'medium', high: 'high', exceptional: 'high' },
|
|
23
|
+
quality: { low: 'low', moderate: 'medium', high: 'high', exceptional: 'xhigh' },
|
|
24
|
+
};
|
|
25
|
+
/** Accepts provider spellings such as "extra-high". "ultra" (Codex auto-delegation) is not an effort. */
|
|
26
|
+
export function parseEffort(s) {
|
|
27
|
+
const v = s.trim().toLowerCase();
|
|
28
|
+
if (v === 'extra-high' || v === 'extra_high')
|
|
29
|
+
return 'xhigh';
|
|
30
|
+
return EFFORT_LADDER.includes(v) ? v : undefined;
|
|
31
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { readTextFile } from '../placements/agents/deps.js';
|
|
2
|
+
import { buildPlacements, installedAgents } from '../placements/build.js';
|
|
3
|
+
import { readTable, staleness } from '../placements/table.js';
|
|
4
|
+
import { cacheDir } from '../paths.js';
|
|
5
|
+
import { findExecutable, runCommand } from '../system/run.js';
|
|
6
|
+
import { VERSION } from '../version.js';
|
|
7
|
+
import { readOrcaAgents } from './orca-settings.js';
|
|
8
|
+
import { readOrcaUsage } from './orca-usage.js';
|
|
9
|
+
import { activeRejections } from './rejection.js';
|
|
10
|
+
import { readRejections } from './rejections.js';
|
|
11
|
+
/** The cached table when it is fresh, else a rebuilt one. */
|
|
12
|
+
export async function loadPlacements(config, deps = {}) {
|
|
13
|
+
const now = deps.now ?? Date.now();
|
|
14
|
+
const find = deps.find ?? findExecutable;
|
|
15
|
+
const dir = deps.cacheDir ?? cacheDir();
|
|
16
|
+
const read = readTable(deps.readFile ?? readTextFile, dir);
|
|
17
|
+
const onPath = installedAgents({ find, env: deps.env ?? process.env });
|
|
18
|
+
const why = deps.refresh
|
|
19
|
+
? 'refresh'
|
|
20
|
+
: staleness(read, { now, ttlHours: config.placementsTtlHours, version: deps.version ?? VERSION, config, onPath });
|
|
21
|
+
if (why === undefined)
|
|
22
|
+
return { table: read.table, rebuilt: undefined };
|
|
23
|
+
const built = await buildPlacements(config, { ...deps, now, cacheDir: dir });
|
|
24
|
+
return { table: built.table, rebuilt: why, ...(built.saveError !== undefined ? { saveError: built.saveError } : {}) };
|
|
25
|
+
}
|
|
26
|
+
export async function gather(config, deps = {}) {
|
|
27
|
+
const now = deps.now ?? Date.now();
|
|
28
|
+
const run = deps.run ?? runCommand;
|
|
29
|
+
const [placements, usage, orcaAgents] = await Promise.all([
|
|
30
|
+
loadPlacements(config, { ...deps, now }),
|
|
31
|
+
readOrcaUsage(run),
|
|
32
|
+
(deps.readOrcaAgents ?? readOrcaAgents)(),
|
|
33
|
+
]);
|
|
34
|
+
const { table } = placements;
|
|
35
|
+
const rejections = activeRejections(readRejections(deps.readFile ?? readTextFile, deps.cacheDir ?? cacheDir()), now, config.rejectionTtlHours);
|
|
36
|
+
const env = {
|
|
37
|
+
now,
|
|
38
|
+
discoveries: table.agents,
|
|
39
|
+
usage,
|
|
40
|
+
orcaAgents,
|
|
41
|
+
placementsBuiltAt: table.builtAt,
|
|
42
|
+
catalogSource: table.sources.catalog.source,
|
|
43
|
+
...(table.sources.scores ? { scoreSource: table.sources.scores.source } : {}),
|
|
44
|
+
...(table.sources.benchmarks ? { benchmarkSource: table.sources.benchmarks.source } : {}),
|
|
45
|
+
...(rejections.length > 0 ? { rejections } : {}),
|
|
46
|
+
};
|
|
47
|
+
return { env, ...placements };
|
|
48
|
+
}
|
|
49
|
+
export async function gatherEnvironment(config, deps = {}) {
|
|
50
|
+
return (await gather(config, deps)).env;
|
|
51
|
+
}
|