orcacompass 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +51 -0
- package/README.md +171 -3
- package/data/modelindex-snapshot.json +1537 -0
- package/data/models-snapshot.json +1356 -0
- package/dist/cli.js +906 -0
- package/dist/config.js +272 -0
- package/dist/install.js +56 -0
- package/dist/paths.js +30 -0
- package/dist/placements/agents/antigravity.js +87 -0
- package/dist/placements/agents/claude.js +72 -0
- package/dist/placements/agents/codex.js +66 -0
- package/dist/placements/agents/cursor.js +102 -0
- package/dist/placements/agents/deps.js +19 -0
- package/dist/placements/agents/grok.js +92 -0
- package/dist/placements/agents/list-cache.js +85 -0
- package/dist/placements/agents/muse.js +7 -0
- package/dist/placements/agents/opencode.js +77 -0
- package/dist/placements/agents/orca-catalog.js +34 -0
- package/dist/placements/benchmark.js +70 -0
- package/dist/placements/build.js +94 -0
- package/dist/placements/score.js +70 -0
- package/dist/placements/sources/artificialanalysis.js +133 -0
- package/dist/placements/sources/modelindex.js +84 -0
- package/dist/placements/sources/modelsdev.js +113 -0
- package/dist/placements/table.js +94 -0
- package/dist/placements/tiers.js +136 -0
- package/dist/routing/effort.js +31 -0
- package/dist/routing/environment.js +51 -0
- package/dist/routing/orca-settings.js +111 -0
- package/dist/routing/orca-usage.js +85 -0
- package/dist/routing/pace.js +52 -0
- package/dist/routing/rejection.js +58 -0
- package/dist/routing/rejections.js +18 -0
- package/dist/routing/router.js +552 -0
- package/dist/setup.js +358 -0
- package/dist/system/run.js +128 -0
- package/dist/types.js +8 -0
- package/dist/verify.js +106 -0
- package/dist/version.js +3 -0
- package/package.json +60 -6
- package/skills/orcacompass/SKILL.md +278 -0
|
@@ -0,0 +1,552 @@
|
|
|
1
|
+
import { DEMAND_EFFORT, snapEffort } from './effort.js';
|
|
2
|
+
import { poolState, projectPool, rankHeadroom } from './pace.js';
|
|
3
|
+
import { FAMILY_TIERS, isLadderVendor } from '../placements/tiers.js';
|
|
4
|
+
import { AGENT_IDS, TIERS, } from '../types.js';
|
|
5
|
+
/** The effort to ask for: an explicit --effort, else the demand through the policy, else the requested tier's default. */
|
|
6
|
+
function effortPlan(req, config) {
|
|
7
|
+
if (req.effort)
|
|
8
|
+
return { source: 'explicit', target: req.effort };
|
|
9
|
+
if (req.demand) {
|
|
10
|
+
return { source: 'demand', demand: req.demand, policy: config.effortPolicy, target: DEMAND_EFFORT[config.effortPolicy][req.demand] };
|
|
11
|
+
}
|
|
12
|
+
return { source: 'tier-default', target: config.defaultEffort[req.tier] };
|
|
13
|
+
}
|
|
14
|
+
/** Whether a launch can apply exactly this effort (a flag the model supports, or an effort variant). */
|
|
15
|
+
export function canApplyEffort(model, effort) {
|
|
16
|
+
return model.launch.kind === 'flag' ? model.efforts.includes(effort) : model.launch.ids[effort] !== undefined;
|
|
17
|
+
}
|
|
18
|
+
const CONFIDENCE_ORDER = ['high', 'medium', 'low'];
|
|
19
|
+
/**
|
|
20
|
+
* The route for a job. Last-resort agents (`lastResortAgents`, OpenCode by default) are left out first; they
|
|
21
|
+
* are tried only when that finds no route in any allowed tier, and then with a warning. Naming one with
|
|
22
|
+
* `--agent`, or preferring it for the requested tier, makes it an ordinary agent for that route.
|
|
23
|
+
*/
|
|
24
|
+
export function route(req, env, config) {
|
|
25
|
+
const decision = selectRoute(req, env, config);
|
|
26
|
+
if (req.tierReason) {
|
|
27
|
+
decision.tierReason = req.tierReason;
|
|
28
|
+
decision.reasons.push(`task tier ${req.tier}: ${req.tierReason}`);
|
|
29
|
+
}
|
|
30
|
+
return decision;
|
|
31
|
+
}
|
|
32
|
+
function selectRoute(req, env, config) {
|
|
33
|
+
const pinned = config.pins[req.tier]?.agent;
|
|
34
|
+
const deferred = config.lastResortAgents.filter((a) => !req.agents?.includes(a) && a !== pinned);
|
|
35
|
+
const usable = (a) => env.discoveries.some((d) => d.agent === a && !agentSkipReason(d, env, config, req));
|
|
36
|
+
if (!deferred.some(usable))
|
|
37
|
+
return routeOnce(req, env, config, []);
|
|
38
|
+
const first = routeOnce(req, env, config, deferred);
|
|
39
|
+
if (first.status === 'ok')
|
|
40
|
+
return first;
|
|
41
|
+
const second = routeOnce(req, env, config, []);
|
|
42
|
+
if (second.status !== 'ok' || !second.route || !deferred.includes(second.route.agent))
|
|
43
|
+
return first;
|
|
44
|
+
const instead = first.status === 'wait' && first.waitUntil !== undefined ? `instead of waiting ${duration(first.waitUntil - env.now)}` : 'as nothing else can take it';
|
|
45
|
+
second.warnings.push(`${second.route.agent} is a last-resort agent (lastResortAgents), used ${instead}: ${billingOf(second, env)}`);
|
|
46
|
+
return second;
|
|
47
|
+
}
|
|
48
|
+
function billingOf(d, env) {
|
|
49
|
+
const m = env.discoveries.find((x) => x.agent === d.route?.agent)?.models.find((x) => launchIds(x).includes(d.route.model));
|
|
50
|
+
return m?.billing ?? 'no usage window';
|
|
51
|
+
}
|
|
52
|
+
function routeOnce(req, env, config, deferred) {
|
|
53
|
+
const skipped = [];
|
|
54
|
+
const agents = eligibleAgents(req, env, config, skipped, deferred);
|
|
55
|
+
const all = collectCandidates(agents, env, config, req.tier, skipped);
|
|
56
|
+
// An explicit effort is a constraint, like --agent: a route that cannot apply it exactly is not used.
|
|
57
|
+
const honors = (c) => !req.effort || canApplyEffort(c.model, req.effort);
|
|
58
|
+
const byTier = (tier) => autoFirst(all.filter((c) => c.tier === tier && honors(c)).sort(compareRanked));
|
|
59
|
+
const requested = req.tier;
|
|
60
|
+
const plan = effortPlan(req, config);
|
|
61
|
+
const target = plan.target;
|
|
62
|
+
const warnings = [];
|
|
63
|
+
const overRisk = new Set();
|
|
64
|
+
const choose = (tier) => {
|
|
65
|
+
const ranked = byTier(tier);
|
|
66
|
+
const pin = config.pins[tier];
|
|
67
|
+
if (pin) {
|
|
68
|
+
// Bans and agent filters already removed their candidates, so a preference only chooses among the rest.
|
|
69
|
+
const isPin = (c) => c.model.agent === pin.agent && (!pin.model || launchIds(c.model).includes(pin.model));
|
|
70
|
+
const mine = ranked.filter(isPin);
|
|
71
|
+
const match = mine.find((c) => !c.blocked) ?? mine[0];
|
|
72
|
+
const label = `preferred ${pin.agent}${pin.model ? ` ${pin.model}` : ''} skipped`;
|
|
73
|
+
if (!match && req.effort && all.some((c) => c.tier === tier && isPin(c))) {
|
|
74
|
+
warnings.push(`${label}: does not support effort ${req.effort}`);
|
|
75
|
+
}
|
|
76
|
+
else if (!match) {
|
|
77
|
+
const gone = skipped.find((s) => s.agent === pin.agent && (!s.model || !pin.model || s.model === pin.model));
|
|
78
|
+
warnings.push(`${label}: not available${gone ? ` (${gone.reason})` : ''}`);
|
|
79
|
+
}
|
|
80
|
+
else if (match.blocked) {
|
|
81
|
+
warnings.push(`${label}: blocked (${blockers(match, env.now)})`);
|
|
82
|
+
}
|
|
83
|
+
else if (match.risk && match.risk.projectedPercent > config.preferRiskLimit) {
|
|
84
|
+
const safer = ranked.find((c) => !c.blocked && c.rank !== null && c.risk && c.risk.projectedPercent <= config.preferRiskLimit);
|
|
85
|
+
if (!safer) {
|
|
86
|
+
warnings.push(`preferred ${pin.agent}${pin.model ? ` ${pin.model}` : ''} retained: no unblocked alternative with known usage within the ${config.preferRiskLimit}% risk limit`);
|
|
87
|
+
return { cand: match, pinned: true };
|
|
88
|
+
}
|
|
89
|
+
for (const candidate of mine) {
|
|
90
|
+
if (candidate.risk && candidate.risk.projectedPercent > config.preferRiskLimit)
|
|
91
|
+
overRisk.add(candidate);
|
|
92
|
+
}
|
|
93
|
+
const r = match.risk;
|
|
94
|
+
warnings.push(`${label}: ${poolLabel(r.pool)} on pace for ${percent(Math.round(r.projectedPercent))} (limit ${config.preferRiskLimit}%), resets in ${duration(r.resetsAt - env.now)}`);
|
|
95
|
+
return { cand: safer, pinned: isPin(safer) };
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
return { cand: match, pinned: true };
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const first = ranked.find((c) => !c.blocked && !overRisk.has(c));
|
|
102
|
+
return first ? { cand: first, pinned: false } : undefined;
|
|
103
|
+
};
|
|
104
|
+
const base = () => ({
|
|
105
|
+
status: 'none', requestedTier: requested, downgraded: false, upgraded: false,
|
|
106
|
+
reasons: [], warnings, alternatives: [], skipped,
|
|
107
|
+
});
|
|
108
|
+
const skippedLine = () => skipped.length ? [`skipped: ${skipped.map((s) => `${s.agent}${s.model ? ` ${s.model}` : ''} (${s.reason})`).join(', ')}`] : [];
|
|
109
|
+
const finish = (pick, tier, move, why) => {
|
|
110
|
+
const { cand } = pick;
|
|
111
|
+
// Effort follows the work, not the model: a fallback to another tier keeps the target.
|
|
112
|
+
const chosen = resolve(cand, target);
|
|
113
|
+
const effort = effortResolution(cand, plan, chosen);
|
|
114
|
+
const others = byTier(tier).filter((c) => c !== cand && !overRisk.has(c));
|
|
115
|
+
const reasons = [`${cand.model.agent} ${chosen.model}: ${describe(cand, env.now)}`];
|
|
116
|
+
if (cand.scoreDetail)
|
|
117
|
+
reasons.push(`${cand.model.modelId}: tier ${cand.tier} from ${cand.scoreDetail}`);
|
|
118
|
+
if (cand.source === 'auto')
|
|
119
|
+
reasons.push('cursor auto: Cursor Auto picks the model (Cost mode), billed to Cursor Models');
|
|
120
|
+
if (cand.model.agent === 'opencode')
|
|
121
|
+
reasons.push('opencode: Orca takes --model for OpenCode only in an existing worktree (--worktree current or a selector)');
|
|
122
|
+
if (pick.pinned)
|
|
123
|
+
reasons.push(`preferred route, ${cand.risk ? paceText(cand.risk) : 'usage unknown'}`);
|
|
124
|
+
for (const name of cand.expiredPools)
|
|
125
|
+
reasons.push(`${cand.model.agent} ${poolLabel(name)}: reset time already passed (expired), pool ignored`);
|
|
126
|
+
reasons.push(effortText(effort, cand));
|
|
127
|
+
if (move === 'down') {
|
|
128
|
+
reasons.push(`downgraded from ${requested}: ${why}`);
|
|
129
|
+
warnings.push(`a ${tier} model is doing ${requested} work; effort is not raised to make up for it: split the job or add stronger checks`);
|
|
130
|
+
}
|
|
131
|
+
if (move === 'up') {
|
|
132
|
+
reasons.push(`upgraded from ${requested}: no ${requested}-tier or lower model installed`);
|
|
133
|
+
warnings.push(`no ${requested}-tier model installed; using ${tier}`);
|
|
134
|
+
}
|
|
135
|
+
const next = others.find((c) => !c.blocked);
|
|
136
|
+
if (next)
|
|
137
|
+
reasons.push(`next best: ${next.model.agent} ${resolve(next, target).model} (${summary(next, env.now)})`);
|
|
138
|
+
reasons.push(...skippedLine());
|
|
139
|
+
return {
|
|
140
|
+
status: 'ok',
|
|
141
|
+
requestedTier: requested,
|
|
142
|
+
tier,
|
|
143
|
+
downgraded: move === 'down',
|
|
144
|
+
upgraded: move === 'up',
|
|
145
|
+
route: { agent: cand.model.agent, model: chosen.model, effort: chosen.effort },
|
|
146
|
+
effort,
|
|
147
|
+
launchArgs: chosen.args,
|
|
148
|
+
confidence: cand.confidence,
|
|
149
|
+
risk: cand.risk ?? null,
|
|
150
|
+
reasons,
|
|
151
|
+
warnings,
|
|
152
|
+
alternatives: others.slice(0, 5).map((c) => alternative(c, target, env.now)),
|
|
153
|
+
skipped,
|
|
154
|
+
};
|
|
155
|
+
};
|
|
156
|
+
if (req.effort) {
|
|
157
|
+
const inTier = all.filter((c) => c.tier === requested).sort(compareRanked);
|
|
158
|
+
if (inTier.length > 0 && !inTier.some(honors)) {
|
|
159
|
+
const offered = inTier.slice(0, 6).map((c) => `${c.model.agent} ${c.model.modelId} (${supportedEfforts(c.model)})`).join(', ');
|
|
160
|
+
return {
|
|
161
|
+
...base(),
|
|
162
|
+
alternatives: inTier.slice(0, 5).map((c) => alternative(c, target, env.now)),
|
|
163
|
+
reasons: [`no ${requested} route supports effort ${req.effort}: ${offered}`, 'choose one of these efforts, or route without --effort', ...skippedLine()],
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const same = choose(requested);
|
|
168
|
+
if (same)
|
|
169
|
+
return finish(same, requested, 'same');
|
|
170
|
+
const idx = TIERS.indexOf(requested);
|
|
171
|
+
const lower = TIERS.slice(0, idx).reverse();
|
|
172
|
+
const higher = TIERS.slice(idx + 1);
|
|
173
|
+
const requestedCands = byTier(requested);
|
|
174
|
+
const structural = requestedCands.length === 0 && lower.every((t) => byTier(t).length === 0);
|
|
175
|
+
const blockedSummary = requestedCands.map((c) => `${c.model.agent} ${c.model.modelId} (${summary(c, env.now)})`).join(', ');
|
|
176
|
+
const why = requestedCands.length > 0 ? `all ${requested} candidates blocked: ${blockedSummary}` : `no ${requested}-tier model installed`;
|
|
177
|
+
const waitUntil = earliestUnblock(requestedCands);
|
|
178
|
+
if (!req.noDowngrade) {
|
|
179
|
+
for (const t of lower) {
|
|
180
|
+
const pick = choose(t);
|
|
181
|
+
if (pick)
|
|
182
|
+
return finish(pick, t, 'down', why);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
if (structural) {
|
|
186
|
+
for (const t of higher) {
|
|
187
|
+
const pick = choose(t);
|
|
188
|
+
if (pick)
|
|
189
|
+
return finish(pick, t, 'up');
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
const alternatives = requestedCands.filter((c) => !overRisk.has(c)).slice(0, 5).map((c) => alternative(c, target, env.now));
|
|
193
|
+
if (req.noDowngrade && requestedCands.length > 0 && waitUntil !== undefined) {
|
|
194
|
+
return {
|
|
195
|
+
...base(), status: 'wait', waitUntil, alternatives,
|
|
196
|
+
reasons: [why, `waiting until ${new Date(waitUntil).toISOString()} (${duration(waitUntil - env.now)})`, ...skippedLine()],
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
const reasons = [agents.length === 0 ? 'no usable agent' : why];
|
|
200
|
+
if (!req.noDowngrade && !structural && agents.length > 0)
|
|
201
|
+
reasons.push('no eligible unblocked model in any lower tier');
|
|
202
|
+
reasons.push(...skippedLine());
|
|
203
|
+
return { ...base(), alternatives, reasons, ...(waitUntil !== undefined ? { waitUntil } : {}) };
|
|
204
|
+
}
|
|
205
|
+
/** The coordinator's launch command for a decision, or undefined when there is no route. */
|
|
206
|
+
export function formatLaunch(d) {
|
|
207
|
+
if (!d.launchArgs)
|
|
208
|
+
return undefined;
|
|
209
|
+
return `orca orchestration worker-start --spec "<task>" ${d.launchArgs.join(' ')}`;
|
|
210
|
+
}
|
|
211
|
+
// --- candidate collection -------------------------------------------------------------------
|
|
212
|
+
function eligibleAgents(req, env, config, skipped, deferred) {
|
|
213
|
+
const known = new Set(env.discoveries.map((d) => d.agent));
|
|
214
|
+
for (const agent of req.agents ?? []) {
|
|
215
|
+
if (!known.has(agent))
|
|
216
|
+
skipped.push({ agent, reason: 'unknown agent' });
|
|
217
|
+
}
|
|
218
|
+
return env.discoveries.filter((d) => {
|
|
219
|
+
const reason = agentSkipReason(d, env, config, req) ?? (deferred.includes(d.agent) ? 'last resort: used only when no other agent can take the job' : undefined);
|
|
220
|
+
if (reason)
|
|
221
|
+
skipped.push({ agent: d.agent, reason });
|
|
222
|
+
return reason === undefined;
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Why an agent is not used, strongest reason first: the user's choice (config, then request), Orca's
|
|
227
|
+
* "not enabled", Orca's "unavailable", not installed, signed out of its model list, no models. Undefined
|
|
228
|
+
* when the agent is used.
|
|
229
|
+
*/
|
|
230
|
+
export function agentSkipReason(d, env, config, req = {}) {
|
|
231
|
+
const agent = d.agent;
|
|
232
|
+
const usage = env.usage[agent];
|
|
233
|
+
if (config.denyAgents.includes(agent) || (config.allowAgents && !config.allowAgents.includes(agent)))
|
|
234
|
+
return 'excluded by config';
|
|
235
|
+
if ((req.agents && !req.agents.includes(agent)) || req.excludeAgents?.includes(agent))
|
|
236
|
+
return 'excluded by request';
|
|
237
|
+
if (env.orcaAgents?.known && env.orcaAgents.disabled.includes(agent))
|
|
238
|
+
return 'not enabled in Orca';
|
|
239
|
+
if (usage?.status === 'unavailable')
|
|
240
|
+
return `not signed in or unavailable${usage.error ? `: ${usage.error}` : ''}`;
|
|
241
|
+
if (!d.installed)
|
|
242
|
+
return 'not installed';
|
|
243
|
+
if (d.signedOut)
|
|
244
|
+
return d.note ?? 'not signed in';
|
|
245
|
+
if (d.models.length === 0)
|
|
246
|
+
return `no models${d.note ? `: ${d.note}` : ''}`;
|
|
247
|
+
return undefined;
|
|
248
|
+
}
|
|
249
|
+
/** Orca's own switch for an agent; unknown when the setting could not be read. */
|
|
250
|
+
export const enabledInOrca = (o, agent) => !o?.known ? 'unknown' : o.disabled.includes(agent) ? 'no' : 'yes';
|
|
251
|
+
/** The user's ban entry matching a model by its base id or Models.dev family, if any. */
|
|
252
|
+
export function banEntry(model, banModels) {
|
|
253
|
+
return banModels.find((b) => b === model.modelId || (model.meta?.family !== undefined && b === model.meta.family));
|
|
254
|
+
}
|
|
255
|
+
/** The first ban entry that names an agent and no model family: that takes `agents disable`, not a ban. */
|
|
256
|
+
export function agentNotFamily(entries, matches) {
|
|
257
|
+
return entries.find((e) => AGENT_IDS.includes(e) && !Object.hasOwn(FAMILY_TIERS, e) && (matches[e]?.length ?? 0) === 0);
|
|
258
|
+
}
|
|
259
|
+
/** Discovered models each ban entry matches, as agent + base model id. */
|
|
260
|
+
export function banMatches(entries, env) {
|
|
261
|
+
return Object.fromEntries(entries.map((entry) => [
|
|
262
|
+
entry,
|
|
263
|
+
env.discoveries.flatMap((d) => d.models.filter((m) => banEntry(m, [entry])).map((m) => ({ agent: d.agent, model: m.modelId }))),
|
|
264
|
+
]));
|
|
265
|
+
}
|
|
266
|
+
/** The agent's recent refusal of this model under any of its launch ids, if any. */
|
|
267
|
+
export function rejectionOf(model, rejections) {
|
|
268
|
+
if (!rejections?.length)
|
|
269
|
+
return undefined;
|
|
270
|
+
const ids = launchIds(model);
|
|
271
|
+
return rejections.find((r) => r.agent === model.agent && ids.includes(r.model));
|
|
272
|
+
}
|
|
273
|
+
/** e.g. 'refused by codex 2h ago: "The 'x' model is not supported when using Codex with a ChatGPT account."' */
|
|
274
|
+
export const rejectedText = (r, now) => `refused by ${r.agent} ${duration(now - r.at)} ago: "${r.message}"`;
|
|
275
|
+
/** The Model Index and Artificial Analysis scales the environment supports. */
|
|
276
|
+
/** Candidates of every tier; banned or refused models placed in `tier` are reported as skipped. */
|
|
277
|
+
function collectCandidates(agents, env, config, tier, skipped) {
|
|
278
|
+
const classified = [];
|
|
279
|
+
for (const d of agents) {
|
|
280
|
+
for (const model of d.models) {
|
|
281
|
+
if (model.noZdr && config.excludeNoZdr)
|
|
282
|
+
continue;
|
|
283
|
+
// The table already holds the placement (built from every source); the route only reads it.
|
|
284
|
+
const placed = model.placements;
|
|
285
|
+
const rejected = rejectionOf(model, env.rejections);
|
|
286
|
+
const leftOut = banEntry(model, config.banModels) ? 'banned by you' : rejected ? rejectedText(rejected, env.now) : undefined;
|
|
287
|
+
if (leftOut) {
|
|
288
|
+
const inTier = placed.some((c) => c.tier === tier);
|
|
289
|
+
if (inTier && !skipped.some((s) => s.agent === model.agent && s.model === model.modelId)) {
|
|
290
|
+
skipped.push({ agent: model.agent, model: model.modelId, reason: leftOut });
|
|
291
|
+
}
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
for (const cls of placed)
|
|
295
|
+
classified.push({ model, cls });
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
return newestPerFamily(classified).map(({ model, cls }) => evaluate(model, cls, env, config));
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Keep only the newest release per (agent, family, tier); models without a date count as oldest.
|
|
302
|
+
* Table families share one tier; score placement can split a family (grok-4.6 Daily, grok-4.5 Light).
|
|
303
|
+
*/
|
|
304
|
+
function newestPerFamily(items) {
|
|
305
|
+
const newest = new Map();
|
|
306
|
+
const key = ({ model, cls }) => `${model.agent}\u0000${model.meta?.family ?? model.modelId}\u0000${cls.tier}`;
|
|
307
|
+
const date = (m) => m.meta?.releaseDate ?? '';
|
|
308
|
+
for (const item of items) {
|
|
309
|
+
const k = key(item);
|
|
310
|
+
if (date(item.model) > (newest.get(k) ?? '') || !newest.has(k))
|
|
311
|
+
newest.set(k, date(item.model));
|
|
312
|
+
}
|
|
313
|
+
return items.filter((item) => date(item.model) === newest.get(key(item)));
|
|
314
|
+
}
|
|
315
|
+
function evaluate(model, cls, env, config) {
|
|
316
|
+
const base = {
|
|
317
|
+
model, tier: cls.tier, source: cls.source, headroom: null, rank: null, blocked: false, blockedPools: [], expiredPools: [], confidence: 'low', projections: {},
|
|
318
|
+
...(cls.detail ? { scoreDetail: cls.detail } : {}),
|
|
319
|
+
};
|
|
320
|
+
if (model.poolNames.length === 0)
|
|
321
|
+
return { ...base, usageNote: `no usage window: ${model.billing ?? 'limits not tracked'}` };
|
|
322
|
+
const usage = env.usage[model.agent];
|
|
323
|
+
if (!usage || usage.status !== 'ok')
|
|
324
|
+
return { ...base, usageNote: 'usage unknown (Orca not reachable)' };
|
|
325
|
+
// Blocking is evaluated even on stale data: within a window usedPercent only grows, so a stale
|
|
326
|
+
// "blocked" is a valid lower bound. Staleness only degrades the ranking signal (see below).
|
|
327
|
+
let lowered = false;
|
|
328
|
+
const lower = (a, b) => (!a || b.rank < a.rank ? b : a);
|
|
329
|
+
let long;
|
|
330
|
+
let session;
|
|
331
|
+
let bindingSession;
|
|
332
|
+
for (const name of model.poolNames) {
|
|
333
|
+
const pool = usage.pools[name];
|
|
334
|
+
if (!pool)
|
|
335
|
+
continue;
|
|
336
|
+
const state = poolState(pool, env.now, config.thresholds);
|
|
337
|
+
if (state.expired) {
|
|
338
|
+
base.expiredPools.push(name);
|
|
339
|
+
lowered = true;
|
|
340
|
+
continue;
|
|
341
|
+
}
|
|
342
|
+
if (state.blocked) {
|
|
343
|
+
base.blocked = true;
|
|
344
|
+
base.blockedPools.push(pool);
|
|
345
|
+
}
|
|
346
|
+
else if (pool.kind === 'session' && pool.usedPercent >= config.thresholds.sessionPercent - 20) {
|
|
347
|
+
base.hotSession = pool;
|
|
348
|
+
}
|
|
349
|
+
const projected = projectPool(pool, env.now);
|
|
350
|
+
base.projections[name] = projected;
|
|
351
|
+
if (!base.risk || projected.projectedPercent > base.risk.projectedPercent)
|
|
352
|
+
base.risk = projected;
|
|
353
|
+
const ranked = { pool, headroom: state.headroom, rank: rankHeadroom(state.headroom, pool.resetsAt, env.now), elapsed: state.elapsed };
|
|
354
|
+
if (pool.kind === 'long') {
|
|
355
|
+
long = lower(long, ranked);
|
|
356
|
+
}
|
|
357
|
+
else {
|
|
358
|
+
session = lower(session, ranked);
|
|
359
|
+
// A session counts once its pace would hit the block threshold before it resets.
|
|
360
|
+
if (!projected.tooEarly && projected.projectedPercent >= config.thresholds.sessionPercent)
|
|
361
|
+
bindingSession = lower(bindingSession, ranked);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
const ranked = long ? (bindingSession ? lower(long, bindingSession) : long) : session;
|
|
365
|
+
if (!ranked)
|
|
366
|
+
return { ...base, usageNote: 'usage unknown (no matching usage pool)' };
|
|
367
|
+
base.headroom = ranked.headroom;
|
|
368
|
+
base.rank = ranked.rank;
|
|
369
|
+
base.tightest = { pool: ranked.pool, elapsed: ranked.elapsed };
|
|
370
|
+
// The provider's own lineup and the family table are firm; a benchmark or score placement is an estimate.
|
|
371
|
+
base.confidence = lowered ? 'low' : cls.source === 'family' || cls.source === 'lineup' ? 'high' : 'medium';
|
|
372
|
+
const age = usage.updatedAt === undefined ? 0 : env.now - usage.updatedAt;
|
|
373
|
+
const staleMs = config.usageStaleMinutes * 60_000;
|
|
374
|
+
if (age > 4 * staleMs) {
|
|
375
|
+
return { ...base, headroom: null, rank: null, tightest: undefined, confidence: 'low', usageNote: `usage stale (updated ${duration(age)} ago)` };
|
|
376
|
+
}
|
|
377
|
+
if (age > staleMs) {
|
|
378
|
+
base.staleNote = `usage ${duration(age)} old`;
|
|
379
|
+
if (base.confidence === 'high')
|
|
380
|
+
base.confidence = 'medium';
|
|
381
|
+
}
|
|
382
|
+
return base;
|
|
383
|
+
}
|
|
384
|
+
// --- ranking --------------------------------------------------------------------------------
|
|
385
|
+
/** How much more headroom a benchmark-placed model needs to pass the provider's own pick for the tier, within one agent. */
|
|
386
|
+
export const HOME_TIER_MARGIN = 0.2;
|
|
387
|
+
const isHome = (c) => c.source === 'lineup' || c.source === 'family';
|
|
388
|
+
/**
|
|
389
|
+
* Within one agent, the model the provider (or the ladder) puts in this tier keeps it against a model that only
|
|
390
|
+
* benchmarks into it, unless the other has more than HOME_TIER_MARGIN more headroom: Fable before Opus for Frontier.
|
|
391
|
+
*/
|
|
392
|
+
function homeFirst(a, b) {
|
|
393
|
+
if (a.model.agent !== b.model.agent || isHome(a) === isHome(b) || a.rank === null || b.rank === null)
|
|
394
|
+
return 0;
|
|
395
|
+
const homeIsA = isHome(a);
|
|
396
|
+
const homeRank = homeIsA ? a.rank : b.rank;
|
|
397
|
+
const otherRank = homeIsA ? b.rank : a.rank;
|
|
398
|
+
if (otherRank - homeRank > HOME_TIER_MARGIN)
|
|
399
|
+
return 0;
|
|
400
|
+
return homeIsA ? -1 : 1;
|
|
401
|
+
}
|
|
402
|
+
/** Within a tier: unblocked first, then most headroom, then the plan's ordering (Claude/GPT before other vendors on a tie), ending in a total order for determinism. */
|
|
403
|
+
function compareRanked(a, b) {
|
|
404
|
+
if (a.blocked !== b.blocked)
|
|
405
|
+
return a.blocked ? 1 : -1;
|
|
406
|
+
if ((a.rank === null) !== (b.rank === null))
|
|
407
|
+
return a.rank === null ? 1 : -1;
|
|
408
|
+
const home = homeFirst(a, b);
|
|
409
|
+
if (home !== 0)
|
|
410
|
+
return home;
|
|
411
|
+
if (a.rank !== null && b.rank !== null && a.rank !== b.rank)
|
|
412
|
+
return b.rank - a.rank;
|
|
413
|
+
if (!!a.hotSession !== !!b.hotSession)
|
|
414
|
+
return a.hotSession ? 1 : -1;
|
|
415
|
+
const conf = CONFIDENCE_ORDER.indexOf(a.confidence) - CONFIDENCE_ORDER.indexOf(b.confidence);
|
|
416
|
+
if (conf !== 0)
|
|
417
|
+
return conf;
|
|
418
|
+
if (!!a.model.fast !== !!b.model.fast)
|
|
419
|
+
return a.model.fast ? 1 : -1;
|
|
420
|
+
const va = isLadderVendor(a.model.meta?.family);
|
|
421
|
+
if (va !== isLadderVendor(b.model.meta?.family))
|
|
422
|
+
return va ? -1 : 1;
|
|
423
|
+
const da = a.model.meta?.releaseDate ?? '';
|
|
424
|
+
const db = b.model.meta?.releaseDate ?? '';
|
|
425
|
+
if (da !== db)
|
|
426
|
+
return da < db ? 1 : -1;
|
|
427
|
+
if (a.model.agent !== b.model.agent)
|
|
428
|
+
return a.model.agent < b.model.agent ? -1 : 1;
|
|
429
|
+
return a.model.modelId < b.model.modelId ? -1 : a.model.modelId > b.model.modelId ? 1 : 0;
|
|
430
|
+
}
|
|
431
|
+
/**
|
|
432
|
+
* Cursor's representative in a tier is Auto when it is unblocked: named Cursor models that out-ranked
|
|
433
|
+
* it move to just behind it, so other agents are compared with Auto by the usual ranking.
|
|
434
|
+
*/
|
|
435
|
+
function autoFirst(ranked) {
|
|
436
|
+
const at = ranked.findIndex((c) => c.source === 'auto' && !c.blocked);
|
|
437
|
+
if (at <= 0)
|
|
438
|
+
return ranked;
|
|
439
|
+
const behind = ranked.slice(0, at).filter((c) => c.model.agent === 'cursor');
|
|
440
|
+
const ahead = ranked.slice(0, at).filter((c) => c.model.agent !== 'cursor');
|
|
441
|
+
return [...ahead, ranked[at], ...behind, ...ranked.slice(at + 1)];
|
|
442
|
+
}
|
|
443
|
+
/** Earliest moment at which some blocked candidate has all of its blocking pools reset. */
|
|
444
|
+
function earliestUnblock(cands) {
|
|
445
|
+
const times = cands.filter((c) => c.blocked).map((c) => Math.max(...c.blockedPools.map((p) => p.resetsAt)));
|
|
446
|
+
return times.length ? Math.min(...times) : undefined;
|
|
447
|
+
}
|
|
448
|
+
// --- launch resolution ----------------------------------------------------------------------
|
|
449
|
+
export function launchIds(m) {
|
|
450
|
+
const ids = m.launch.kind === 'flag' ? [m.launch.model] : [...Object.values(m.launch.ids), ...(m.launch.defaultId ? [m.launch.defaultId] : [])];
|
|
451
|
+
return [m.modelId, ...ids];
|
|
452
|
+
}
|
|
453
|
+
/** Launch for a target effort: the target if offered, else the nearest lower one, else the lowest higher one. */
|
|
454
|
+
function resolve(c, target) {
|
|
455
|
+
const { agent, launch, efforts, modelId } = c.model;
|
|
456
|
+
if (launch.kind === 'flag') {
|
|
457
|
+
const effort = snapEffort(target, efforts);
|
|
458
|
+
const args = ['--agent', agent, '--model', launch.model, ...(effort ? ['--effort', effort] : [])];
|
|
459
|
+
return { model: launch.model, effort, resolved: effort, applied: effort ? 'flag' : 'provider-default', args };
|
|
460
|
+
}
|
|
461
|
+
const variant = snapEffort(target, Object.keys(launch.ids));
|
|
462
|
+
const model = (variant && launch.ids[variant]) ?? launch.defaultId ?? modelId;
|
|
463
|
+
return { model, effort: null, resolved: variant, applied: variant ? 'model-id' : 'provider-default', args: ['--agent', agent, '--model', model] };
|
|
464
|
+
}
|
|
465
|
+
function effortResolution(c, plan, r) {
|
|
466
|
+
return {
|
|
467
|
+
...plan,
|
|
468
|
+
resolved: r.resolved,
|
|
469
|
+
applied: r.applied,
|
|
470
|
+
adjusted: r.resolved !== null && r.resolved !== plan.target,
|
|
471
|
+
supportSource: c.model.effortSource ?? 'unknown',
|
|
472
|
+
...(c.model.defaultEffort ? { providerDefault: c.model.defaultEffort } : {}),
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
/** "low/medium/high", or "provider-controlled" when the launch cannot set an effort. */
|
|
476
|
+
function supportedEfforts(m) {
|
|
477
|
+
const efforts = m.launch.kind === 'flag' ? m.efforts : Object.keys(m.launch.ids);
|
|
478
|
+
return efforts.length ? efforts.join('/') : 'provider-controlled';
|
|
479
|
+
}
|
|
480
|
+
/** e.g. "effort: high from high reasoning demand (balanced policy), passed as --effort". */
|
|
481
|
+
function effortText(e, c) {
|
|
482
|
+
const why = e.source === 'explicit' ? 'request' : e.source === 'demand' ? `${e.demand} reasoning demand (${e.policy} policy)` : 'tier default';
|
|
483
|
+
if (e.applied === 'provider-default') {
|
|
484
|
+
const own = e.providerDefault ? ` (${e.providerDefault})` : '';
|
|
485
|
+
const who = c.source === 'auto' ? 'Cursor Auto decides' : `${c.model.agent} uses its own default${own}`;
|
|
486
|
+
return `effort: not set, ${who}; ${e.target} wanted from ${why}`;
|
|
487
|
+
}
|
|
488
|
+
const how = e.applied === 'flag' ? 'passed as --effort' : 'chosen through the model id';
|
|
489
|
+
const snapped = e.adjusted ? ` (${e.target} not offered, nearest supported)` : '';
|
|
490
|
+
const assumed = e.supportSource === 'assumed' ? '; supported levels assumed, not reported by the provider' : '';
|
|
491
|
+
return `effort: ${e.resolved} from ${why}${snapped}, ${how}${assumed}`;
|
|
492
|
+
}
|
|
493
|
+
function alternative(c, target, now) {
|
|
494
|
+
const r = resolve(c, target);
|
|
495
|
+
return {
|
|
496
|
+
agent: c.model.agent, model: r.model, effort: r.effort, tier: c.tier,
|
|
497
|
+
headroom: c.headroom, confidence: c.confidence, blocked: c.blocked, risk: c.risk ?? null, reason: describe(c, now),
|
|
498
|
+
};
|
|
499
|
+
}
|
|
500
|
+
// --- explanations ---------------------------------------------------------------------------
|
|
501
|
+
/** Full one-line account of a candidate's usage, e.g. "weekly 20% used, 56% of window elapsed (headroom 60%) → on pace for 36% before reset in 3d". */
|
|
502
|
+
function describe(c, now) {
|
|
503
|
+
if (c.blocked)
|
|
504
|
+
return `blocked: ${blockers(c, now)}`;
|
|
505
|
+
if (c.usageNote || !c.tightest || c.headroom === null)
|
|
506
|
+
return c.usageNote ?? 'usage unknown';
|
|
507
|
+
const { pool, elapsed } = c.tightest;
|
|
508
|
+
const own = c.projections[pool.name];
|
|
509
|
+
const pace = (r, resetsAt) => r && !r.tooEarly ? ` → ${paceText(r)} before reset in ${duration(resetsAt - now)}` : `, resets in ${duration(resetsAt - now)}${r ? ' (too early to project)' : ''}`;
|
|
510
|
+
const parts = [`${poolLabel(pool.name)} ${percent(pool.usedPercent)} used, ${Math.round(elapsed * 100)}% of window elapsed (headroom ${headroomText(c.headroom)}${scaled(c)})${pace(own, pool.resetsAt)}`];
|
|
511
|
+
// The pool on pace to end most used, when it is not the ranked one (usually a hot session).
|
|
512
|
+
const worst = c.risk && c.risk.pool !== pool.name && !c.risk.tooEarly ? c.risk : undefined;
|
|
513
|
+
if (worst)
|
|
514
|
+
parts.push(`${poolLabel(worst.pool)} ${percent(worst.usedPercent)} used${pace(worst, worst.resetsAt)}`);
|
|
515
|
+
else if (c.hotSession && c.hotSession !== pool)
|
|
516
|
+
parts.push(`session ${percent(c.hotSession.usedPercent)} used, resets in ${duration(c.hotSession.resetsAt - now)}`);
|
|
517
|
+
if (c.staleNote)
|
|
518
|
+
parts.push(c.staleNote);
|
|
519
|
+
return parts.join('; ');
|
|
520
|
+
}
|
|
521
|
+
/** Short form used for runner-ups and blocked lists, e.g. "fableWeekly headroom 47%, resets in 14h". */
|
|
522
|
+
function summary(c, now) {
|
|
523
|
+
if (c.blocked)
|
|
524
|
+
return blockers(c, now);
|
|
525
|
+
if (c.model.poolNames.length === 0 && c.usageNote)
|
|
526
|
+
return c.usageNote;
|
|
527
|
+
if (c.usageNote || !c.tightest || c.headroom === null)
|
|
528
|
+
return 'usage unknown';
|
|
529
|
+
return `${poolLabel(c.tightest.pool.name)} headroom ${headroomText(c.headroom)}${scaled(c)}, resets in ${duration(c.tightest.pool.resetsAt - now)}`;
|
|
530
|
+
}
|
|
531
|
+
function blockers(c, now) {
|
|
532
|
+
return c.blockedPools.map((p) => `${poolLabel(p.name)} ${percent(p.usedPercent)} used, resets in ${duration(p.resetsAt - now)}`).join('; ');
|
|
533
|
+
}
|
|
534
|
+
const poolLabel = (name) => name.replace(/^bucket:/, '');
|
|
535
|
+
/** "on pace for 118%", or for an early window "too early to project (weekly 3% used)". */
|
|
536
|
+
const paceText = (r) => r.tooEarly ? `too early to project (${poolLabel(r.pool)} ${percent(r.usedPercent)} used)` : `on pace for ${percent(Math.round(r.projectedPercent))}`;
|
|
537
|
+
const percent = (n) => `${Number.isInteger(n) ? n : n.toFixed(1)}%`;
|
|
538
|
+
/** ", counted as 21% for a reset beyond 7d" when the ranked headroom was scaled for a distant reset. */
|
|
539
|
+
const scaled = (c) => c.rank !== null && c.headroom !== null && headroomText(c.rank) !== headroomText(c.headroom) ? `, counted as ${headroomText(c.rank)} for a reset beyond 7d` : '';
|
|
540
|
+
/** Headroom as a whole percent of the window, e.g. "60%" or "-19%". */
|
|
541
|
+
export const headroomText = (h) => `${Math.round(h * 100)}%`;
|
|
542
|
+
function duration(ms) {
|
|
543
|
+
if (ms <= 0)
|
|
544
|
+
return '0m';
|
|
545
|
+
const days = Math.floor(ms / 86_400_000);
|
|
546
|
+
if (days >= 1)
|
|
547
|
+
return `${days}d`;
|
|
548
|
+
const hours = Math.floor(ms / 3_600_000);
|
|
549
|
+
if (hours >= 1)
|
|
550
|
+
return `${hours}h`;
|
|
551
|
+
return `${Math.floor(ms / 60_000)}m`;
|
|
552
|
+
}
|