@kal-elsam/kairo-runtime 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +76 -0
- package/package.json +2 -1
- package/scripts/cockpit-smoke.mjs +1 -1
- package/scripts/ux-smoke-test.sh +3 -3
- package/src/cli.js +106 -11
- package/src/global/agent-capabilities/create-capability-adapter.js +2 -2
- package/src/global/architect/architect-cli.js +76 -0
- package/src/global/architect/architect-codex.js +146 -0
- package/src/global/architect/architect-manager.js +125 -0
- package/src/global/architect/architect-store.js +377 -0
- package/src/global/architect/architect-types.js +47 -0
- package/src/global/cli-help.js +12 -1
- package/src/global/cockpit/app.js +475 -0
- package/src/global/cockpit/card.js +111 -0
- package/src/global/cockpit/cli.js +33 -0
- package/src/global/cockpit/gauge.js +31 -0
- package/src/global/cockpit/project-overlay.js +683 -0
- package/src/global/cockpit/rows.js +148 -0
- package/src/global/cockpit/theme.js +118 -0
- package/src/global/cockpit/view.js +1263 -0
- package/src/global/control-plane/attention.js +141 -0
- package/src/global/control-plane/build-report.js +146 -0
- package/src/global/control-plane/cli.js +36 -0
- package/src/global/control-plane/constants.js +38 -0
- package/src/global/control-plane/gentle-adapters.js +183 -0
- package/src/global/control-plane/provider.js +69 -0
- package/src/global/control-plane/review-status.js +115 -0
- package/src/global/control-plane/sdd-status.js +49 -0
- package/src/global/control-plane/team.js +63 -0
- package/src/global/conversation/bootstrap-analyzer-adapters.js +251 -0
- package/src/global/conversation/cli.js +53 -0
- package/src/global/conversation/codex-sandbox.js +230 -0
- package/src/global/conversation/cursor-sandbox.js +215 -0
- package/src/global/conversation/project-analysis.js +204 -0
- package/src/global/conversation/project-profile.js +178 -0
- package/src/global/conversation/project-router.js +149 -0
- package/src/global/conversation/project-strategy-store.js +64 -0
- package/src/global/conversation/project-strategy.js +514 -0
- package/src/global/conversation/sanitized-snapshot.js +169 -0
- package/src/global/conversation/secret-scanner.js +71 -0
- package/src/global/conversation/service.js +1063 -0
- package/src/global/conversation/session-store.js +75 -0
- package/src/global/conversation/transcript-store.js +79 -0
- package/src/global/conversation/ui.js +195 -0
- package/src/global/intelligence/capability-scoring.js +480 -0
- package/src/global/intelligence/execution-router.js +444 -0
- package/src/global/intelligence/kairo-telemetry-source.js +59 -0
- package/src/global/intelligence/kairobench-runner.js +85 -0
- package/src/global/intelligence/kairobench-source.js +34 -0
- package/src/global/intelligence/kairobench-tasks.js +47 -0
- package/src/global/intelligence/model-candidate-catalog.js +456 -0
- package/src/global/intelligence/model-capability-registry-sources.js +145 -0
- package/src/global/intelligence/model-capability-registry.js +125 -0
- package/src/global/intelligence/model-intelligence.js +1646 -0
- package/src/global/intelligence/official-benchmark-snapshots.js +162 -0
- package/src/global/intelligence/quick-ask.js +149 -0
- package/src/global/intelligence/role-profiles.js +251 -0
- package/src/global/intelligence/skill-catalog.js +67 -0
- package/src/global/intelligence/subscription-pressure-source.js +41 -0
- package/src/global/mcp/kairo-mcp.js +51 -18
- package/src/global/mcp/work-snapshot-rule.js +4 -2
- package/src/global/mcp/workspace-binding.js +88 -0
- package/src/global/mcp/workspace-mcp-entry.js +74 -0
- package/src/global/mcp-install.js +8 -1
- package/src/global/observability/artificial-analysis-models.js +118 -0
- package/src/global/observability/claude-models.js +31 -0
- package/src/global/observability/claude-usage.js +112 -0
- package/src/global/observability/codex-models.js +96 -0
- package/src/global/observability/codex-usage.js +160 -0
- package/src/global/observability/cursor-auth.js +88 -0
- package/src/global/observability/cursor-models.js +101 -0
- package/src/global/observability/gentle-probe.js +30 -2
- package/src/global/observability/huggingface-leaderboard.js +97 -0
- package/src/global/observability/index.js +2 -1
- package/src/global/observability/opencode-models.js +101 -0
- package/src/global/observability/opencode-usage.js +162 -0
- package/src/global/paths.js +49 -2
- package/src/global/profile.js +23 -1
- package/src/global/runtime/execution-adapters/claude.js +63 -30
- package/src/global/runtime/execution-adapters/codex.js +9 -2
- package/src/global/runtime/execution-adapters/create-execution-adapter.js +6 -1
- package/src/global/runtime/execution-adapters/opencode.js +83 -18
- package/src/global/runtime/execution-worktree-manager.js +924 -0
- package/src/global/runtime/execution-worktree-orchestrator.js +194 -0
- package/src/global/runtime/execution-worktree-store.js +83 -0
- package/src/global/runtime/execution-worktree-types.js +45 -0
- package/src/global/runtime/run-events.js +38 -0
- package/src/global/runtime/run-manager.js +22 -6
- package/src/global/runtime/run-supervisor.js +41 -12
- package/src/global/runtime/usage-manager.js +96 -0
- package/src/global/runtime/usage-store.js +69 -0
- package/src/global/runtime/usage-types.js +62 -0
|
@@ -0,0 +1,1646 @@
|
|
|
1
|
+
// Cross-references the models Kairo can actually launch right now (each
|
|
2
|
+
// provider's real, discovered/documented catalog) with real Artificial
|
|
3
|
+
// Analysis benchmark scores — deliberately NOT "download every model AA
|
|
4
|
+
// tracks": only the ones we actually have access to matter for routing.
|
|
5
|
+
//
|
|
6
|
+
// A model with no confident match gets no score, never a guessed one —
|
|
7
|
+
// same fail-closed rule as everywhere else in Kairo's routing.
|
|
8
|
+
|
|
9
|
+
import { bestEvidence, createCapabilityRegistry } from "./model-capability-registry.js";
|
|
10
|
+
import { CONFIDENCE_RANK, computeRoleEvaluations, computeRoleGapValue, activeBenchmarkCountForCapability, isCapabilityComparable } from "./capability-scoring.js";
|
|
11
|
+
import { ROLE_CAPABILITIES, getRoleProfile } from "./role-profiles.js";
|
|
12
|
+
|
|
13
|
+
// Real per-benchmark metrics worth surfacing as corroborating evidence
|
|
14
|
+
// alongside a pick — never blended into the ranking itself, since
|
|
15
|
+
// Terminal-Bench/GPQA/HLE aren't the same measurement as AA's
|
|
16
|
+
// intelligenceIndex/codingIndex and averaging them would violate the
|
|
17
|
+
// registry's own no-blending contract.
|
|
18
|
+
const CORROBORATION_METRICS = [
|
|
19
|
+
"terminal-bench", "terminal-bench-science", "gpqa-diamond", "hle", "cursorbench", "kairo.success",
|
|
20
|
+
// AA's own real per-benchmark scores (0-1 scale, as AA reports them) —
|
|
21
|
+
// verified live to already be in the free API response alongside the
|
|
22
|
+
// composite indices, kept distinct from the manufacturer-reported
|
|
23
|
+
// 0-100 scale metrics above.
|
|
24
|
+
"gpqa", "sciCode", "mmluPro", "liveCodeBench", "ifBench", "terminalBenchHard", "terminalBenchV2", "tau2", "tauBanking"
|
|
25
|
+
];
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Attaches real registry evidence (Hugging Face, manufacturer snapshots,
|
|
29
|
+
* Kairo's own telemetry) to a model, purely for transparency — never used
|
|
30
|
+
* to change a ranking value. `registry` is optional; without one, models
|
|
31
|
+
* pass through unchanged (existing callers/tests keep working).
|
|
32
|
+
* @param {object} model - has adapterId/modelId
|
|
33
|
+
* @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>|null} registry
|
|
34
|
+
*/
|
|
35
|
+
function withCorroboration(model, registry) {
|
|
36
|
+
if (!registry) return model;
|
|
37
|
+
const id = registry.registerIdentity(model.adapterId, model.modelId);
|
|
38
|
+
const corroboration = [];
|
|
39
|
+
for (const metric of CORROBORATION_METRICS) {
|
|
40
|
+
const best = bestEvidence(registry, id, metric);
|
|
41
|
+
if (best) corroboration.push({ metric, value: best.value, source: best.source });
|
|
42
|
+
}
|
|
43
|
+
return corroboration.length ? { ...model, corroboration } : model;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* @param {Array<object>} models - scoreAvailableModels() output
|
|
48
|
+
* @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>|null} [registry]
|
|
49
|
+
*/
|
|
50
|
+
export function annotateWithRegistryEvidence(models, registry = null) {
|
|
51
|
+
if (!registry) return models;
|
|
52
|
+
return models.map((model) => withCorroboration(model, registry));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function normalizeId(id) {
|
|
56
|
+
return String(id ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function sortedTokens(normalizedId) {
|
|
60
|
+
return normalizedId.split("-").filter(Boolean).sort().join("-");
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Finds the Artificial Analysis entry for a real provider model id. Tries
|
|
65
|
+
* an exact normalized match first (e.g. "gpt-6-astra" == "gpt-6-astra"),
|
|
66
|
+
* then a same-tokens match for cases where the two sources order words
|
|
67
|
+
* differently (e.g. Kairo's "claude-haiku-4-5" vs AA's "claude-4-5-haiku").
|
|
68
|
+
* @param {string} modelId
|
|
69
|
+
* @param {Array<{slug: string}>} aaModels
|
|
70
|
+
*/
|
|
71
|
+
export function matchArtificialAnalysisScore(modelId, aaModels) {
|
|
72
|
+
const normalized = normalizeId(modelId);
|
|
73
|
+
if (!normalized) return null;
|
|
74
|
+
const exact = aaModels.find((model) => normalizeId(model.slug) === normalized);
|
|
75
|
+
if (exact) return exact;
|
|
76
|
+
const tokens = sortedTokens(normalized);
|
|
77
|
+
return aaModels.find((model) => sortedTokens(normalizeId(model.slug)) === tokens) ?? null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* @param {Array<{adapterId: string, models: Array<{id: string, displayName?: string}>}>} providerCatalogs
|
|
82
|
+
* - only the models each provider's own real catalog actually reports as
|
|
83
|
+
* available, e.g. `[{ adapterId: "codex", models: readCodexModels().models }]`
|
|
84
|
+
* @param {Array<object>} aaModels - readArtificialAnalysisModels().models
|
|
85
|
+
* @returns {Array<{adapterId: string, modelId: string, displayName: string|null, slug: string, name: string, intelligenceIndex: number|null, codingIndex: number|null, mathIndex: number|null}>}
|
|
86
|
+
* Only models Kairo actually has access to AND could confidently match — never a guessed score.
|
|
87
|
+
*/
|
|
88
|
+
export function scoreAvailableModels(providerCatalogs, aaModels) {
|
|
89
|
+
const results = [];
|
|
90
|
+
for (const { adapterId, models } of providerCatalogs) {
|
|
91
|
+
for (const model of models ?? []) {
|
|
92
|
+
const score = matchArtificialAnalysisScore(model.id, aaModels);
|
|
93
|
+
if (!score) continue;
|
|
94
|
+
results.push({
|
|
95
|
+
adapterId, modelId: model.id, displayName: model.displayName ?? null,
|
|
96
|
+
slug: score.slug, name: score.name,
|
|
97
|
+
intelligenceIndex: score.intelligenceIndex, codingIndex: score.codingIndex, mathIndex: score.mathIndex,
|
|
98
|
+
priceInputPerMTok: score.priceInputPerMTok ?? null, priceOutputPerMTok: score.priceOutputPerMTok ?? null,
|
|
99
|
+
outputTokensPerSecond: score.outputTokensPerSecond ?? null,
|
|
100
|
+
// Real per-benchmark scores AA's free API also returns — used as
|
|
101
|
+
// optional role-specific tie-breakers in AI_TEAM_ROLE_DEFINITIONS,
|
|
102
|
+
// never blended into the composite indices above.
|
|
103
|
+
gpqa: score.gpqa ?? null, hle: score.hle ?? null, sciCode: score.sciCode ?? null,
|
|
104
|
+
mmluPro: score.mmluPro ?? null, liveCodeBench: score.liveCodeBench ?? null, ifBench: score.ifBench ?? null,
|
|
105
|
+
terminalBenchHard: score.terminalBenchHard ?? null, terminalBenchV2: score.terminalBenchV2 ?? null,
|
|
106
|
+
tau2: score.tau2 ?? null, tauBanking: score.tauBanking ?? null
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return annotateBestFit(results);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* How much of each provider's real catalog Kairo could actually match to
|
|
115
|
+
* real Artificial Analysis data — separate from RUNTIME eligibility
|
|
116
|
+
* (checkCandidate's quota/availability check): a provider can be fully
|
|
117
|
+
* entitled and runtime-eligible yet still have unmatched models simply
|
|
118
|
+
* because AA doesn't track them, or Kairo's own catalog is only
|
|
119
|
+
* "documented" rather than a live discovery (Claude, today). Surfaces
|
|
120
|
+
* that distinction so "Fable is the best model available now" is never
|
|
121
|
+
* confused with "Fable is the only model Kairo could ever evaluate."
|
|
122
|
+
* @param {Array<{adapterId: string, catalogStatus: string, models: Array<object|string>}>} providerCatalogs
|
|
123
|
+
* @param {Array<object>} aaModels
|
|
124
|
+
* @returns {Array<{adapterId: string, catalogStatus: string, totalModels: number, matchedModels: number}>}
|
|
125
|
+
*/
|
|
126
|
+
export function summarizeCatalogCoverage(providerCatalogs, aaModels) {
|
|
127
|
+
return providerCatalogs.map(({ adapterId, catalogStatus, models }) => {
|
|
128
|
+
const list = models ?? [];
|
|
129
|
+
const matched = list.filter((entry) => {
|
|
130
|
+
const id = typeof entry === "string" ? entry : entry.id;
|
|
131
|
+
return matchArtificialAnalysisScore(id, aaModels) != null;
|
|
132
|
+
});
|
|
133
|
+
return { adapterId, catalogStatus, totalModels: list.length, matchedModels: matched.length };
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// listUnscoredModels used to live here — every real catalog model
|
|
138
|
+
// scoreAvailableModels() couldn't match to AA data, kept so /models
|
|
139
|
+
// --evidence could show it honestly instead of it just vanishing.
|
|
140
|
+
// model-candidate-catalog.js's buildCompleteCandidateCatalog now does
|
|
141
|
+
// this same real AA-match check as part of computing every candidate's
|
|
142
|
+
// evidenceStatus ("unscored" when unmatched) — a caller filters that
|
|
143
|
+
// catalog for evidenceStatus === "unscored" instead of calling a second,
|
|
144
|
+
// parallel function that duplicated the exact same real check.
|
|
145
|
+
|
|
146
|
+
// Which real, unweighted metric each model is best at among the models you
|
|
147
|
+
// actually have access to right now — never a blended/invented composite
|
|
148
|
+
// score. "better" says which direction wins for that metric (higher coding
|
|
149
|
+
// score is better; lower price is better).
|
|
150
|
+
const BEST_FIT_METRICS = [
|
|
151
|
+
{ key: "codingIndex", label: "best coding", better: "max" },
|
|
152
|
+
{ key: "intelligenceIndex", label: "best reasoning", better: "max" },
|
|
153
|
+
{ key: "outputTokensPerSecond", label: "fastest", better: "max" },
|
|
154
|
+
{ key: "priceInputPerMTok", label: "cheapest", better: "min" }
|
|
155
|
+
];
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* @param {Array<object>} models
|
|
159
|
+
* @param {(model: object) => number|null} getValue
|
|
160
|
+
* @param {"max"|"min"} better
|
|
161
|
+
*/
|
|
162
|
+
function bestIndexForValue(models, getValue, better) {
|
|
163
|
+
let bestIndex = -1;
|
|
164
|
+
let bestValue = null;
|
|
165
|
+
for (let i = 0; i < models.length; i += 1) {
|
|
166
|
+
const value = getValue(models[i]);
|
|
167
|
+
if (value == null) continue;
|
|
168
|
+
const wins = bestValue == null || (better === "max" ? value > bestValue : value < bestValue);
|
|
169
|
+
if (wins) { bestIndex = i; bestValue = value; }
|
|
170
|
+
}
|
|
171
|
+
return bestIndex;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function bestIndexFor(models, key, better) {
|
|
175
|
+
return bestIndexForValue(models, (model) => model[key], better);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* The bottleneck (worst-case) of two real metrics — never their average or
|
|
180
|
+
* a weighted blend. Used for roles that plausibly need both signals
|
|
181
|
+
* (Debugger, Reviewer) but have no distinct benchmark of their own: a
|
|
182
|
+
* model is only as good at the composite job as its weaker real skill.
|
|
183
|
+
* Null if either input is missing — never guesses with partial data.
|
|
184
|
+
*/
|
|
185
|
+
function minOfReal(a, b) {
|
|
186
|
+
return a == null || b == null ? null : Math.min(a, b);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Tags each model with which real metrics it wins, relative only to the
|
|
191
|
+
* other models actually in this list — a purely relative, computed fact,
|
|
192
|
+
* not a judgment call about which role/persona it "is."
|
|
193
|
+
* @param {Array<object>} models - scoreAvailableModels' output (pre-tagging)
|
|
194
|
+
*/
|
|
195
|
+
function annotateBestFit(models) {
|
|
196
|
+
const bestFor = models.map(() => []);
|
|
197
|
+
for (const metric of BEST_FIT_METRICS) {
|
|
198
|
+
const bestIndex = bestIndexFor(models, metric.key, metric.better);
|
|
199
|
+
if (bestIndex !== -1) bestFor[bestIndex].push(metric.label);
|
|
200
|
+
}
|
|
201
|
+
return models.map((model, i) => ({ ...model, bestFor: bestFor[i] }));
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// The seven reusable role profiles, each resolved from real metrics only —
|
|
205
|
+
// never a weighted blend, never an invented percentage. Three have a
|
|
206
|
+
// direct real benchmark (Architect/Planner, Implementer, Economy); the
|
|
207
|
+
// rest are honestly derived:
|
|
208
|
+
// - Explorer: same real signal as Architect/Planner (intelligence) —
|
|
209
|
+
// Kairo has no distinct "exploration" benchmark, so it doesn't
|
|
210
|
+
// pretend otherwise with a different-looking number.
|
|
211
|
+
// - Debugger / Reviewer: the bottleneck (minimum, not an average) of
|
|
212
|
+
// intelligence and coding — a model is only as good at either
|
|
213
|
+
// composite job as its weaker real skill.
|
|
214
|
+
// - Test Author: the same real coding signal as Implementer — there is
|
|
215
|
+
// no distinct testing benchmark in this data either.
|
|
216
|
+
// "Orchestrator" is deliberately not a role: it's Kairo itself, never a
|
|
217
|
+
// ranked model. "Terminal-required" and "autonomous execution" are real
|
|
218
|
+
// gaps (no Agentic Index at this API tier, verified against the live
|
|
219
|
+
// response) — logged as future work, not faked with a stand-in metric.
|
|
220
|
+
const ROLE_DEFINITIONS = [
|
|
221
|
+
{ role: "Explorer", compute: (m) => m.intelligenceIndex, better: "max" },
|
|
222
|
+
{ role: "Architect / Planner", compute: (m) => m.intelligenceIndex, better: "max" },
|
|
223
|
+
{ role: "Implementer", compute: (m) => m.codingIndex, better: "max" },
|
|
224
|
+
{ role: "Debugger", compute: (m) => minOfReal(m.intelligenceIndex, m.codingIndex), better: "max" },
|
|
225
|
+
{ role: "Test Author", compute: (m) => m.codingIndex, better: "max" },
|
|
226
|
+
{ role: "Reviewer", compute: (m) => minOfReal(m.intelligenceIndex, m.codingIndex), better: "max" },
|
|
227
|
+
{ role: "Economy", compute: (m) => m.priceInputPerMTok, better: "min" }
|
|
228
|
+
];
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* @param {Array<object>} models - scoreAvailableModels() output
|
|
232
|
+
* @returns {Array<{role: string, adapterId: string, modelId: string, displayName: string|null}>}
|
|
233
|
+
* One entry per role that has a real winner; a role is simply omitted
|
|
234
|
+
* when no available model reports the metric(s) it needs.
|
|
235
|
+
*/
|
|
236
|
+
export function bestModelPerRole(models) {
|
|
237
|
+
const entries = [];
|
|
238
|
+
for (const { role, compute, better } of ROLE_DEFINITIONS) {
|
|
239
|
+
const bestIndex = bestIndexForValue(models, compute, better);
|
|
240
|
+
if (bestIndex === -1) continue;
|
|
241
|
+
const winner = models[bestIndex];
|
|
242
|
+
entries.push({ role, adapterId: winner.adapterId, modelId: winner.modelId, displayName: winner.displayName });
|
|
243
|
+
}
|
|
244
|
+
return entries;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// Every metric a role definition might ask for — re-ingested into a
|
|
248
|
+
// throwaway registry (see ensureRegistry) when the caller doesn't pass a
|
|
249
|
+
// real one, so role compute() functions always have exactly one code path
|
|
250
|
+
// (resolve via the registry) regardless of whether richer evidence
|
|
251
|
+
// (Hugging Face, manufacturer snapshots, Kairo's own telemetry) is
|
|
252
|
+
// actually available for this call.
|
|
253
|
+
const KNOWN_MODEL_METRICS = [
|
|
254
|
+
"intelligenceIndex", "codingIndex", "mathIndex", "priceInputPerMTok", "outputTokensPerSecond",
|
|
255
|
+
"gpqa", "hle", "sciCode", "mmluPro", "liveCodeBench", "ifBench", "terminalBenchHard", "terminalBenchV2", "tau2", "tauBanking"
|
|
256
|
+
];
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Guarantees buildAiTeam always has a real registry to resolve role
|
|
260
|
+
* requirements against — a role's compute() must have exactly one code
|
|
261
|
+
* path (resolve via the registry) whether or not the caller supplied one.
|
|
262
|
+
* Always seeds the AA fields already present on `models`
|
|
263
|
+
* (scoreAvailableModels' output) into whichever registry ends up in
|
|
264
|
+
* use — a fresh throwaway one when none is given, or the caller's own
|
|
265
|
+
* real registry (service.js, wired to AA + Hugging Face + manufacturer
|
|
266
|
+
* snapshots + Kairo's own telemetry) otherwise. This has to seed the
|
|
267
|
+
* caller's registry too, not just the throwaway one: the robust
|
|
268
|
+
* multi-metric engine (capability-scoring.js) resolves every role
|
|
269
|
+
* purely through registry evidence, with no fallback to the raw
|
|
270
|
+
* `model[metric]` field the old single-metric `resolveMetric` used —
|
|
271
|
+
* so a real registry that hasn't separately ingested AA's
|
|
272
|
+
* intelligenceIndex/codingIndex would otherwise silently lose that
|
|
273
|
+
* evidence entirely. Never overwrites evidence the registry already
|
|
274
|
+
* has for a given identity/metric pair.
|
|
275
|
+
*/
|
|
276
|
+
export function ensureRegistry(models, registry) {
|
|
277
|
+
const effective = registry ?? createCapabilityRegistry();
|
|
278
|
+
for (const model of models) {
|
|
279
|
+
const id = effective.registerIdentity(model.adapterId, model.modelId, model.displayName ?? null);
|
|
280
|
+
const existingMetrics = new Set(effective.getEvidence(id).map((entry) => entry.metric));
|
|
281
|
+
for (const metric of KNOWN_MODEL_METRICS) {
|
|
282
|
+
const value = model[metric];
|
|
283
|
+
if (value == null || existingMetrics.has(metric)) continue;
|
|
284
|
+
effective.addEvidence(id, { metric, value, source: "artificial-analysis-free", benchmarkVersion: null, modelConfig: null, date: null, verified: false });
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
return effective;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/**
|
|
291
|
+
* Resolves one real metric for a model through the evidence registry
|
|
292
|
+
* first (bestEvidence already prefers verified/most-recent across every
|
|
293
|
+
* connected source — AA, Hugging Face, manufacturer snapshots, Kairo's
|
|
294
|
+
* own telemetry), falling back to the field already on `model` only if
|
|
295
|
+
* the registry somehow has nothing for it. This is what actually
|
|
296
|
+
* "connects the registry to each role's requirements" instead of only
|
|
297
|
+
* ever reading the one AA field baked onto the model object.
|
|
298
|
+
*
|
|
299
|
+
* Only safe for metrics with ONE real name across every source (today:
|
|
300
|
+
* intelligenceIndex, codingIndex, priceInputPerMTok — nothing else calls
|
|
301
|
+
* them anything different yet). A metric multiple sources name
|
|
302
|
+
* differently (GPQA as AA's "gpqa" vs a manufacturer table's
|
|
303
|
+
* "gpqa-diamond") is instead resolved through the robust multi-metric
|
|
304
|
+
* percentile engine (capability-scoring.js's BENCHMARK_IDENTITIES) —
|
|
305
|
+
* looking up one exact key here would silently miss every other source's
|
|
306
|
+
* real evidence for the same real thing.
|
|
307
|
+
*/
|
|
308
|
+
function resolveMetric(registry, model, key) {
|
|
309
|
+
const id = registry.registerIdentity(model.adapterId, model.modelId, model.displayName ?? null);
|
|
310
|
+
const best = bestEvidence(registry, id, key);
|
|
311
|
+
return best ? best.value : (model[key] ?? null);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// ROLE_CAPABILITIES's canonical home is role-profiles.js — it's the
|
|
315
|
+
// capabilities half of that module's RoleProfile (objective, allowed
|
|
316
|
+
// actions, risk, escalation — the OTHER half — live there too), imported
|
|
317
|
+
// above and re-exported here only so every existing caller of
|
|
318
|
+
// model-intelligence.js keeps working unchanged. Never edit the table
|
|
319
|
+
// itself here; see role-profiles.js for the real definition and its full
|
|
320
|
+
// reasoning (required-vs-optional split, softwareExecution/
|
|
321
|
+
// instructionFollowing measurements, etc.).
|
|
322
|
+
export { ROLE_CAPABILITIES };
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Normalizes a role's capability requirement — either the legacy plain
|
|
326
|
+
* array shape (every entry required; still used by
|
|
327
|
+
* conversation/project-strategy.js's project-derived roleCapabilities,
|
|
328
|
+
* which analyzes a real project and doesn't yet distinguish required from
|
|
329
|
+
* optional) or the {required, optional} shape above.
|
|
330
|
+
*/
|
|
331
|
+
function normalizeRoleCapabilities(capabilities) {
|
|
332
|
+
if (Array.isArray(capabilities)) return { required: capabilities, optional: [] };
|
|
333
|
+
return { required: capabilities.required ?? [], optional: capabilities.optional ?? [] };
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Builds one role definition per team-vocabulary role (Explorer /
|
|
338
|
+
* Architect / Builder / Debugger / Tester / Reviewer — the same six
|
|
339
|
+
* RoleProfile owns, see role-profiles.js), scored via the robust
|
|
340
|
+
* multi-metric percentile engine (capability-scoring.js). `compute()`
|
|
341
|
+
* per role is a real, precomputed RoleEvaluation.capabilityPercentile
|
|
342
|
+
* lookup (never recomputed per model — percentile is inherently relative
|
|
343
|
+
* to the WHOLE candidate pool, so it's computed once per role, batched,
|
|
344
|
+
* then looked up), and a model absent from that role's evaluations (zero
|
|
345
|
+
* real primary evidence for any of its relevant capabilities) never
|
|
346
|
+
* competes — same fail-closed contract `resolveMetric`-based compute()
|
|
347
|
+
* functions already had. Built fresh per buildAiTeam()/
|
|
348
|
+
* buildEfficientTeam() call (registry AND models differ per call).
|
|
349
|
+
*
|
|
350
|
+
* Economy is NOT one of these role definitions — it used to be a 7th
|
|
351
|
+
* role competing for its own slot here (ranked purely by real price,
|
|
352
|
+
* capability-floor-gated), but per the "PROJECT TEAM primero" plan it's
|
|
353
|
+
* an EXECUTION POLICY any of the six real roles can run under (cheapest
|
|
354
|
+
* real model that still clears that role's own requiredRoleFit),
|
|
355
|
+
* evaluated at task-routing time, never a 7th competitor in QUALITY
|
|
356
|
+
* TEAM/EFFICIENT TEAM's own rankings. Wiring that policy into real
|
|
357
|
+
* routing is a later increment; this function no longer knows Economy
|
|
358
|
+
* exists at all.
|
|
359
|
+
* @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>} registry
|
|
360
|
+
* @param {Array<object>} models
|
|
361
|
+
* @param {Record<string, string[]|{required: string[], optional?: string[]}>} [roleCapabilities] -
|
|
362
|
+
* which real capabilities each role needs, defaulting to the generic
|
|
363
|
+
* global table above ({required, optional} — see normalizeRoleCapabilities
|
|
364
|
+
* for what that distinction gates). A caller building a PROJECT-specific
|
|
365
|
+
* team (see conversation/project-strategy.js) passes the project's own
|
|
366
|
+
* real, detected roleRequirements here instead, as a plain array (legacy
|
|
367
|
+
* shape — every entry treated as required) — e.g. a project with no real
|
|
368
|
+
* test command drops terminalExecution from Tester/Debugger's real
|
|
369
|
+
* requirement entirely, which can genuinely change which model wins
|
|
370
|
+
* that role, not just whether the role is active at all.
|
|
371
|
+
* @returns {{roleDefinitions: Array<{role: string, compute: (model: object) => number|null, better: string}>, evaluationsByRole: Record<string, Map<string, import("./capability-scoring.js").RoleEvaluation>>, optionalEvaluationsByRole: Record<string, Map<string, import("./capability-scoring.js").RoleEvaluation>>, gapValueByRole: Record<string, Map<string, number>>}}
|
|
372
|
+
*/
|
|
373
|
+
function buildAiTeamRoleDefinitions(registry, models, roleCapabilities = ROLE_CAPABILITIES) {
|
|
374
|
+
const evaluationsByRole = {};
|
|
375
|
+
const optionalEvaluationsByRole = {};
|
|
376
|
+
const gapValueByRole = {};
|
|
377
|
+
const roleDefinitions = [];
|
|
378
|
+
for (const [role, rawCapabilities] of Object.entries(roleCapabilities)) {
|
|
379
|
+
const { required, optional } = normalizeRoleCapabilities(rawCapabilities);
|
|
380
|
+
// requiredRoleFit: capabilityPercentile computed ONLY from `required`
|
|
381
|
+
// — this is the number that decides both ranking ORDER (compute()
|
|
382
|
+
// below) and, via gapValueByRole, how CLOSE two real picks are for
|
|
383
|
+
// near-equivalence-band purposes. optional capabilities never enter
|
|
384
|
+
// either computation, so they can't smooth over a real required-
|
|
385
|
+
// capability gap the way folding them into one shared median used to.
|
|
386
|
+
const evaluations = computeRoleEvaluations(registry, models, role, required);
|
|
387
|
+
evaluationsByRole[role] = evaluations;
|
|
388
|
+
// Real, scale-normalized magnitude per model — NOT the percentile
|
|
389
|
+
// above. capabilityPercentile decides ORDER (robust, scale-invariant
|
|
390
|
+
// rank position); this decides HOW CLOSE two real picks are for
|
|
391
|
+
// near-equivalence-band/capability-floor purposes, which need real
|
|
392
|
+
// granularity that percentile alone can't provide with Kairo's
|
|
393
|
+
// typical 2-3-candidate pools (see capability-scoring.js). Same
|
|
394
|
+
// required-only capability list as evaluations above — order and
|
|
395
|
+
// closeness must agree on what "the role" actually means.
|
|
396
|
+
gapValueByRole[role] = computeRoleGapValue(registry, models, required);
|
|
397
|
+
// optionalRoleFit: a completely separate RoleEvaluation, scored only
|
|
398
|
+
// from `optional` capabilities. Never touches ranking order or
|
|
399
|
+
// gapValue — used purely as a tiebreak (sortByCapabilityPriority)
|
|
400
|
+
// among candidates already equally fit on required capabilities. A
|
|
401
|
+
// role with no optional capabilities (Tester, Reviewer) gets an empty
|
|
402
|
+
// map, never a crash — the tiebreak below treats a missing entry as
|
|
403
|
+
// "no optional signal for this model", which ranks equal to every
|
|
404
|
+
// other model with no entry (see sortByCapabilityPriority).
|
|
405
|
+
optionalEvaluationsByRole[role] = optional.length ? computeRoleEvaluations(registry, models, role, optional) : new Map();
|
|
406
|
+
roleDefinitions.push({
|
|
407
|
+
role, better: "max",
|
|
408
|
+
// A model with real evidence on every REQUIRED capability competes
|
|
409
|
+
// on its real requiredRoleFit (capabilityPercentile/gapValue).
|
|
410
|
+
// Missing even one required capability's evidence excludes it from
|
|
411
|
+
// the ranking entirely (null, filtered out by rankBy/rankEligible's
|
|
412
|
+
// existing `.filter((entry) => entry.value != null)`) — coverage
|
|
413
|
+
// stops being merely informational and becomes a real gate, so a
|
|
414
|
+
// model that "looks near-equivalent" on partial evidence can never
|
|
415
|
+
// quietly outrank a properly-measured generalist. Optional evidence
|
|
416
|
+
// never appears here at all — see optionalEvaluationsByRole above.
|
|
417
|
+
compute: (m) => {
|
|
418
|
+
const evaluation = evaluations.get(modelKey(m));
|
|
419
|
+
if (!evaluation) return null;
|
|
420
|
+
if (required.some((capability) => evaluation.capabilities[capability] == null)) return null;
|
|
421
|
+
return evaluation.capabilityPercentile;
|
|
422
|
+
}
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
return { roleDefinitions, evaluationsByRole, optionalEvaluationsByRole, gapValueByRole };
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
// modelName/candidateKey/accessMode/evidenceStatus/lineageKey/generation/
|
|
429
|
+
// lifecycle/resourceCost are real fields from a Recommendation Pool
|
|
430
|
+
// candidate (model-candidate-catalog.js) — passed through when present,
|
|
431
|
+
// never fabricated. A caller still passing raw scoreAvailableModels()
|
|
432
|
+
// output (no candidate-catalog join) simply gets null for all of them;
|
|
433
|
+
// this file never imports model-candidate-catalog.js itself, it just
|
|
434
|
+
// forwards whatever real identity fields the input model already
|
|
435
|
+
// carries, keeping the dependency one-directional.
|
|
436
|
+
function toTeamModel(model, available, registry = null) {
|
|
437
|
+
const base = {
|
|
438
|
+
adapterId: model.adapterId, modelId: model.modelId, displayName: model.displayName, available,
|
|
439
|
+
modelName: model.modelName ?? null, candidateKey: model.candidateKey ?? null,
|
|
440
|
+
accessMode: model.accessMode ?? null, evidenceStatus: model.evidenceStatus ?? null,
|
|
441
|
+
lineageKey: model.lineageKey ?? null, generation: model.generation ?? null, lifecycle: model.lifecycle ?? null,
|
|
442
|
+
resourceCost: model.resourceCost ?? null
|
|
443
|
+
};
|
|
444
|
+
return withCorroboration(base, registry);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function rankBy(models, compute, better) {
|
|
448
|
+
return models
|
|
449
|
+
.map((model) => ({ model, value: compute(model) }))
|
|
450
|
+
.filter((entry) => entry.value != null)
|
|
451
|
+
.sort((a, b) => (better === "max" ? b.value - a.value : a.value - b.value));
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function rankEligible(models, eligibility, compute, better) {
|
|
455
|
+
return rankBy(models.filter((m) => eligibility[m.adapterId]?.ok === true), compute, better);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* Comparable-before-provisional: real, distinct-benchmark coverage (see
|
|
460
|
+
* capability-scoring.js's isCapabilityComparable/RoleEvaluation.isProvisional)
|
|
461
|
+
* decides who's even allowed to compete BEFORE capability value does.
|
|
462
|
+
* "Provisional" means real evidence, just too thin on at least one
|
|
463
|
+
* required capability (e.g. one benchmark out of reasoning's three real
|
|
464
|
+
* active ones) to be genuinely comparable to a broadly-measured
|
|
465
|
+
* candidate — never excluded outright (a real, if thin, data point beats
|
|
466
|
+
* guessing), just never preferred. Filters `ranked` down to only
|
|
467
|
+
* comparable candidates whenever at least one exists; if EVERY real
|
|
468
|
+
* candidate is provisional, the full (all-provisional) list is kept as a
|
|
469
|
+
* real fallback — `usedProvisionalFallback` tells the caller this
|
|
470
|
+
* happened, so the eventual pick can be given an honest reason instead
|
|
471
|
+
* of looking like an ordinary capability win.
|
|
472
|
+
* @param {Array<{model: object, value: number}>} ranked
|
|
473
|
+
* @param {Map<string, import("./capability-scoring.js").RoleEvaluation>|undefined} roleEvaluations
|
|
474
|
+
* @returns {{pool: Array<{model: object, value: number}>, usedProvisionalFallback: boolean}}
|
|
475
|
+
*/
|
|
476
|
+
function preferComparableCandidates(ranked, roleEvaluations) {
|
|
477
|
+
if (!roleEvaluations || !ranked.length) return { pool: ranked, usedProvisionalFallback: false };
|
|
478
|
+
const comparable = ranked.filter((entry) => !roleEvaluations.get(modelKey(entry.model))?.isProvisional);
|
|
479
|
+
if (comparable.length) return { pool: comparable, usedProvisionalFallback: false };
|
|
480
|
+
return { pool: ranked, usedProvisionalFallback: true };
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/**
|
|
484
|
+
* Attaches each ranked entry's REAL, scale-normalized gap value (see
|
|
485
|
+
* capability-scoring.js's computeRoleGapValue) — a separate number from
|
|
486
|
+
* `.value` (the percentile compute() already produced), used only for
|
|
487
|
+
* near-equivalence-band/capability-floor magnitude comparisons (see
|
|
488
|
+
* capabilityPool/adequateCandidates/leaderAdvantage). `gapValueByModel`
|
|
489
|
+
* being undefined leaves entries unchanged — defensive, no current
|
|
490
|
+
* caller passes one without it.
|
|
491
|
+
*/
|
|
492
|
+
function attachGapValues(ranked, gapValueByModel) {
|
|
493
|
+
if (!gapValueByModel) return ranked;
|
|
494
|
+
return ranked.map((entry) => ({ ...entry, gapValue: gapValueByModel.get(modelKey(entry.model)) ?? null }));
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
// Per-role near-equivalence tolerance — replaces the single flat 8% band
|
|
499
|
+
// this codebase used before. Per explicit decision: capability alone
|
|
500
|
+
// isn't the only thing that matters — a model being capable of everything
|
|
501
|
+
// doesn't mean it should always be the one doing it, especially when a
|
|
502
|
+
// real, meaningfully cheaper alternative is genuinely close enough — but
|
|
503
|
+
// how close is "close enough" is NOT the same question for every role.
|
|
504
|
+
// Architect decides the whole plan every other role executes against —
|
|
505
|
+
// a real requiredRoleFit gap there compounds across the entire team, so
|
|
506
|
+
// its tolerance is the tightest. Debugger/Reviewer sit right behind it —
|
|
507
|
+
// Debugger needs real reasoning under a live failure, Reviewer is the
|
|
508
|
+
// team's only independent check on Builder's own work. Builder/Explorer/
|
|
509
|
+
// Tester tolerate more: Builder's real output is still checked by
|
|
510
|
+
// Reviewer, Explorer/Tester's mistakes are cheap to catch and retry.
|
|
511
|
+
// Two real data points anchored the old flat 8%: Claude Fable 5.1 vs
|
|
512
|
+
// OpenCode Go's Kimi K3 sit ~6.6% apart on codingIndex (real GPQA scores
|
|
513
|
+
// within 0.2 points of each other) at roughly a third of the price, while
|
|
514
|
+
// Fable 5.1 vs Codex GPT-5.6 Sol on coding sit ~5.2% apart with no price
|
|
515
|
+
// advantage. Those numbers describe Builder-tier closeness, not
|
|
516
|
+
// Architect-tier — kept as this file's Builder/Explorer/Tester tier
|
|
517
|
+
// value; Architect/Debugger/Reviewer are deliberately tighter than either
|
|
518
|
+
// anchor point. Re-verify against real registry data before changing any
|
|
519
|
+
// of these, never assume a ratio holds indefinitely (see this table's own
|
|
520
|
+
// review date).
|
|
521
|
+
const ROLE_NEAR_EQUIVALENCE_BAND = {
|
|
522
|
+
Architect: 0.03,
|
|
523
|
+
Debugger: 0.05,
|
|
524
|
+
Reviewer: 0.05,
|
|
525
|
+
Builder: 0.06,
|
|
526
|
+
Explorer: 0.06,
|
|
527
|
+
Tester: 0.06
|
|
528
|
+
};
|
|
529
|
+
const DEFAULT_NEAR_EQUIVALENCE_BAND = 0.06;
|
|
530
|
+
|
|
531
|
+
/** The real near-equivalence tolerance for a role — see ROLE_NEAR_EQUIVALENCE_BAND's own doc for why this isn't one flat number. A role missing from the table (e.g. a future addition) falls back to the Builder-tier default rather than crashing. */
|
|
532
|
+
function nearEquivalenceBandFor(role) {
|
|
533
|
+
return ROLE_NEAR_EQUIVALENCE_BAND[role] ?? DEFAULT_NEAR_EQUIVALENCE_BAND;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
// Portfolio-level concentration limits — applied to BOTH teams while
|
|
537
|
+
// assigning roles, not just a per-role decision. Six independent
|
|
538
|
+
// per-role winners don't form a team: without these, the same one or two
|
|
539
|
+
// real models/providers can end up covering every technical role, which
|
|
540
|
+
// is a monoculture risk (a single outage or rate-limit takes out the
|
|
541
|
+
// whole portfolio) even when each individual pick was locally correct.
|
|
542
|
+
const MAX_ROLES_PER_MODEL = 2;
|
|
543
|
+
const MAX_TECHNICAL_ROLES_PER_PROVIDER = 3;
|
|
544
|
+
const TECHNICAL_ROLES = ["Explorer", "Architect", "Builder", "Debugger", "Tester", "Reviewer"];
|
|
545
|
+
|
|
546
|
+
function modelKey(model) {
|
|
547
|
+
return `${model.adapterId}::${model.modelId}`;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
// Real, enumerable reasoning-effort/execution-mode tokens providers append
|
|
551
|
+
// to a model id (low/medium/high/xhigh/max/none/fast/thinking) — not part
|
|
552
|
+
// of the model's real identity, just how hard/fast it's asked to think.
|
|
553
|
+
// Confirmed empirically against Cursor's real ~220-model catalog (a single
|
|
554
|
+
// provider surfacing the same underlying model — e.g. Claude Opus 5 or
|
|
555
|
+
// Claude Fable 5.1 — under many ids like "claude-opus-5-low",
|
|
556
|
+
// "claude-opus-5-thinking-high", etc.) that MAX_ROLES_PER_MODEL's identity
|
|
557
|
+
// key (modelKey, exact adapterId::modelId) does NOT recognize these as the
|
|
558
|
+
// same underlying model, so two different reasoning-tier variants of the
|
|
559
|
+
// identical model could each separately reach the per-model role cap —
|
|
560
|
+
// real evasion of a real limit, verified by reading passesConcentration's
|
|
561
|
+
// own modelKey usage, not assumed.
|
|
562
|
+
//
|
|
563
|
+
// This does NOT mean AA scores them identically — live-tested against
|
|
564
|
+
// real Artificial Analysis data, each reasoning-tier variant matches its
|
|
565
|
+
// OWN distinct real AA benchmark entry (AA genuinely measures different
|
|
566
|
+
// effort settings separately), so RANKING must keep using the exact
|
|
567
|
+
// modelKey (capability evaluation, gapValue, confidence — all still keyed
|
|
568
|
+
// by modelKey below). Only CONCENTRATION/diversity accounting should
|
|
569
|
+
// collapse same-family variants — that's what familyKey is for, used
|
|
570
|
+
// exclusively in passesConcentration, the modelUsage tracking Map, and
|
|
571
|
+
// the diversity tiebreak in both team-builders' sort functions.
|
|
572
|
+
//
|
|
573
|
+
// Family grouping is also deliberately cross-adapter (no adapterId in the
|
|
574
|
+
// key): the same real model reachable via two access paths (e.g. Claude
|
|
575
|
+
// Fable 5.1 through the Claude subscription and through Cursor) is still
|
|
576
|
+
// one real model for concentration purposes — per-adapter monoculture
|
|
577
|
+
// risk is already covered separately by MAX_TECHNICAL_ROLES_PER_PROVIDER,
|
|
578
|
+
// which stays keyed by adapterId alone, unaffected by this change.
|
|
579
|
+
//
|
|
580
|
+
// The whitelist is intentionally narrow and never strips a token outside
|
|
581
|
+
// it — "mini"/"nano"/"sol"/"luna"/"terra"/"astra" etc. are real, distinct
|
|
582
|
+
// models or product lines, not effort settings, and must never be
|
|
583
|
+
// collapsed into the same family.
|
|
584
|
+
const CONCENTRATION_SUFFIX_TOKENS = new Set(["low", "medium", "high", "xhigh", "max", "none", "fast", "thinking"]);
|
|
585
|
+
|
|
586
|
+
function canonicalModelFamily(modelId) {
|
|
587
|
+
const tokens = String(modelId ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").split("-").filter(Boolean);
|
|
588
|
+
while (tokens.length > 1 && CONCENTRATION_SUFFIX_TOKENS.has(tokens[tokens.length - 1])) tokens.pop();
|
|
589
|
+
return tokens.join("-");
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
function familyKey(model) {
|
|
593
|
+
return canonicalModelFamily(model.modelId);
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/**
|
|
597
|
+
* The real capability leader's advantage over the rest of a pool, as a
|
|
598
|
+
* fraction of its own REAL, scale-normalized gap value (see
|
|
599
|
+
* capability-scoring.js's computeRoleGapValue — never the rank-only
|
|
600
|
+
* capabilityPercentile, which is scale-invariant by construction and
|
|
601
|
+
* would report every non-leader as "100% behind" with Kairo's typical
|
|
602
|
+
* 2-3-candidate pools). A single-candidate pool is trivially decisive
|
|
603
|
+
* (Infinity): there is nothing to concentrate away from. A leader with no
|
|
604
|
+
* real gap value at all is likewise treated as trivially decisive —
|
|
605
|
+
* there's no real magnitude to compare.
|
|
606
|
+
*/
|
|
607
|
+
function leaderAdvantage(pool, better) {
|
|
608
|
+
if (pool.length < 2) return Infinity;
|
|
609
|
+
const leader = pool[0];
|
|
610
|
+
if (leader.gapValue == null) return Infinity;
|
|
611
|
+
const scale = Math.abs(leader.gapValue) || 1;
|
|
612
|
+
let minDiff = Infinity;
|
|
613
|
+
for (let i = 1; i < pool.length; i += 1) {
|
|
614
|
+
if (pool[i].gapValue == null) continue;
|
|
615
|
+
const diff = better === "max" ? leader.gapValue - pool[i].gapValue : pool[i].gapValue - leader.gapValue;
|
|
616
|
+
minDiff = Math.min(minDiff, diff / scale);
|
|
617
|
+
}
|
|
618
|
+
return minDiff;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
/** A real decisive real-capability advantage (see ROLE_NEAR_EQUIVALENCE_BAND) is allowed to break the portfolio's concentration limits — a model that dramatically outclasses every other real candidate for a role should never be sacrificed just to spread load. */
|
|
622
|
+
function isDecisiveLeader(pool, better, role) {
|
|
623
|
+
return leaderAdvantage(pool, better) > nearEquivalenceBandFor(role);
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* Orders roles for coordinated assignment: fewer real alternatives first,
|
|
628
|
+
* so the most-constrained roles claim their pick before a more flexible
|
|
629
|
+
* role could have taken it instead. Builder is always resolved before
|
|
630
|
+
* Reviewer, regardless of pool-size ordering, since Reviewer's
|
|
631
|
+
* independence constraint depends on knowing Builder's chosen provider.
|
|
632
|
+
*/
|
|
633
|
+
function orderRolesForAssignment(rolePools) {
|
|
634
|
+
const ordered = [...rolePools].sort((a, b) => a.pool.length - b.pool.length);
|
|
635
|
+
const reviewerIndex = ordered.findIndex((r) => r.role === "Reviewer");
|
|
636
|
+
const builderIndex = ordered.findIndex((r) => r.role === "Builder");
|
|
637
|
+
if (reviewerIndex !== -1 && builderIndex !== -1 && reviewerIndex < builderIndex) {
|
|
638
|
+
const [reviewerEntry] = ordered.splice(reviewerIndex, 1);
|
|
639
|
+
ordered.push(reviewerEntry);
|
|
640
|
+
}
|
|
641
|
+
return ordered.map((r) => r.role);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
/**
|
|
645
|
+
* Stage 2 of buildAiTeam's per-role search: only reached when the narrow
|
|
646
|
+
* near-equivalence band (Stage 1) has NO real candidate that respects the
|
|
647
|
+
* portfolio's concentration limits. Before repeating the leader or
|
|
648
|
+
* invoking decisive-override, search the role's FULL real eligible pool
|
|
649
|
+
* (`fullRanked` — every candidate with real required-capability evidence,
|
|
650
|
+
* not just the ones inside the tight band) for a genuinely adequate real
|
|
651
|
+
* alternative: real requiredRoleFit gapValue still >= EFFICIENT_CAPABILITY_FLOOR
|
|
652
|
+
* (0.80) of the leader's own — the SAME real floor EFFICIENT TEAM already
|
|
653
|
+
* uses to mean "not near-identical, but still genuinely good enough",
|
|
654
|
+
* reused here rather than inventing a second threshold — AND respects
|
|
655
|
+
* concentration itself. Capability-mode only: EFFICIENT already builds
|
|
656
|
+
* its pool this wide from the very first stage (see buildEfficientTeam's
|
|
657
|
+
* own `adequateCandidates` call), so it never needs this widening and
|
|
658
|
+
* never reaches this function (guarded by the `mode === "capability"`
|
|
659
|
+
* check at both call sites below).
|
|
660
|
+
*/
|
|
661
|
+
function findWiderAlternative({ fullRanked, leader, better, role, modelUsage, providerTechnicalUsage, reviewerBuilderAdapter, sortWithinAllowed }) {
|
|
662
|
+
if (!fullRanked) return null;
|
|
663
|
+
const wide = adequateCandidates(fullRanked, leader, better, EFFICIENT_CAPABILITY_FLOOR)
|
|
664
|
+
.filter((candidate) => passesConcentration(candidate, role, modelUsage, providerTechnicalUsage, reviewerBuilderAdapter));
|
|
665
|
+
if (!wide.length) return null;
|
|
666
|
+
const chosen = sortWithinAllowed(wide)[0];
|
|
667
|
+
return { entry: chosen, reasonKind: "wider-search-diversity" };
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* Assigns one role's real winner under the portfolio's concentration
|
|
672
|
+
* limits. Never a benchmark or an invented diversity score — diversity is
|
|
673
|
+
* purely a hard constraint on an already-adequate real candidate pool,
|
|
674
|
+
* applied in this order:
|
|
675
|
+
* 1. Among the narrow near-equivalence band (`pool`), any real
|
|
676
|
+
* candidate that keeps every concentration limit intact competes;
|
|
677
|
+
* `sortWithinAllowed` picks among those (each team's own real
|
|
678
|
+
* priority order — see buildAiTeam/buildEfficientTeam). This is the
|
|
679
|
+
* common case — most roles have a clear leader with no real
|
|
680
|
+
* near-equivalent competitor at all.
|
|
681
|
+
* 2. If NO candidate in that narrow band respects every limit
|
|
682
|
+
* (capability mode only — see findWiderAlternative), widen the
|
|
683
|
+
* search to the role's FULL real eligible pool at
|
|
684
|
+
* EFFICIENT_CAPABILITY_FLOOR (0.80) — genuinely adequate, even if
|
|
685
|
+
* not near-equivalent — and use the best real, concentration-safe
|
|
686
|
+
* candidate there instead. A portfolio limit must never force an
|
|
687
|
+
* incapable model in, or silently exceed itself, while a real
|
|
688
|
+
* 80%+-adequate alternative sits unexamined outside the tight band.
|
|
689
|
+
* 3. If even THAT wide floor-filtered pool has no real,
|
|
690
|
+
* concentration-safe candidate, a decisive real leader (see
|
|
691
|
+
* isDecisiveLeader) is kept anyway rather than handing the role to a
|
|
692
|
+
* real-but-meaningfully-worse candidate from the narrow band.
|
|
693
|
+
* 4. Absolute last resort — nothing anywhere clears the floor and
|
|
694
|
+
* respects concentration, and the leader isn't decisively ahead of
|
|
695
|
+
* the narrow band either: the real leader is repeated anyway. A
|
|
696
|
+
* portfolio constraint must never force an incapable model in just
|
|
697
|
+
* to satisfy diversity for its own sake.
|
|
698
|
+
* @param {object} params
|
|
699
|
+
* @param {string} params.role
|
|
700
|
+
* @param {Array<{model: object, value: number}>} params.pool - already
|
|
701
|
+
* filtered to this role's near-equivalence band (capability mode) or
|
|
702
|
+
* capability-floor pool (efficient mode, already this wide — see
|
|
703
|
+
* buildEfficientTeam).
|
|
704
|
+
* @param {Array<{model: object, value: number}>|undefined} params.fullRanked -
|
|
705
|
+
* the role's FULL real eligible ranking (every candidate with required-
|
|
706
|
+
* capability evidence), used only by Stage 2's widened search.
|
|
707
|
+
* Capability mode only; efficient mode never reads this.
|
|
708
|
+
* @param {"max"|"min"} params.better
|
|
709
|
+
* @param {Map<string, number>} params.modelUsage
|
|
710
|
+
* @param {Map<string, number>} params.providerTechnicalUsage
|
|
711
|
+
* @param {string|null} params.reviewerBuilderAdapter - Builder's chosen
|
|
712
|
+
* adapterId, only when assigning Reviewer; null otherwise.
|
|
713
|
+
* @param {(candidates: Array<{model: object, value: number}>) => Array<{model: object, value: number}>} params.sortWithinAllowed
|
|
714
|
+
* @param {"capability"|"efficient"} params.mode
|
|
715
|
+
* @returns {{entry: {model: object, value: number}, reasonKind: string|null}|null}
|
|
716
|
+
*/
|
|
717
|
+
function assignOneRole({ role, pool, fullRanked, better, modelUsage, providerTechnicalUsage, reviewerBuilderAdapter, sortWithinAllowed, mode }) {
|
|
718
|
+
if (!pool.length) return null;
|
|
719
|
+
const leader = pool[0];
|
|
720
|
+
const widen = () => (mode === "capability"
|
|
721
|
+
? findWiderAlternative({ fullRanked, leader, better, role, modelUsage, providerTechnicalUsage, reviewerBuilderAdapter, sortWithinAllowed })
|
|
722
|
+
: null);
|
|
723
|
+
|
|
724
|
+
if (pool.length === 1) {
|
|
725
|
+
const passes = passesConcentration(leader, role, modelUsage, providerTechnicalUsage, reviewerBuilderAdapter);
|
|
726
|
+
if (passes) {
|
|
727
|
+
// capability mode: a lone real winner needs no explanation — this is
|
|
728
|
+
// the common case (most roles have a clear leader well outside their
|
|
729
|
+
// own, much narrower per-role band). efficient mode: a lone adequate
|
|
730
|
+
// candidate means nothing smaller cleared the capability floor —
|
|
731
|
+
// worth saying.
|
|
732
|
+
return { entry: leader, reasonKind: mode === "efficient" ? "only-adequate-floor" : null };
|
|
733
|
+
}
|
|
734
|
+
const wide = widen();
|
|
735
|
+
if (wide) return wide;
|
|
736
|
+
if (mode === "capability" && isDecisiveLeader(pool, better, role)) return { entry: leader, reasonKind: "decisive-override" };
|
|
737
|
+
return { entry: leader, reasonKind: "only-adequate-concentration" };
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
const allowed = pool.filter((candidate) => passesConcentration(candidate, role, modelUsage, providerTechnicalUsage, reviewerBuilderAdapter));
|
|
741
|
+
if (allowed.length) {
|
|
742
|
+
// The real capability leader doesn't even hit a concentration limit
|
|
743
|
+
// here (or a real near-equivalent alternative already does) — let the
|
|
744
|
+
// narrow band compete normally (CAPABILITY's own diversity priority,
|
|
745
|
+
// or EFFICIENT's real cost/duration/price/throughput chain), with no
|
|
746
|
+
// need to widen the search at all.
|
|
747
|
+
const chosen = sortWithinAllowed(allowed)[0];
|
|
748
|
+
return { entry: chosen, reasonKind: chosen === leader ? null : "diversity", poolSize: allowed.length };
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
const wide = widen();
|
|
752
|
+
if (wide) return wide;
|
|
753
|
+
if (mode === "capability" && isDecisiveLeader(pool, better, role)) {
|
|
754
|
+
// The leader IS blocked by concentration, and even the wide,
|
|
755
|
+
// floor-filtered search (Stage 2) found no real concentration-safe
|
|
756
|
+
// alternative — but its real capability advantage over the narrow
|
|
757
|
+
// band is decisive (> this role's own near-equivalence band, see
|
|
758
|
+
// ROLE_NEAR_EQUIVALENCE_BAND), so it's kept over handing the role to
|
|
759
|
+
// a real-but-meaningfully-worse narrow-band candidate.
|
|
760
|
+
return { entry: leader, reasonKind: "decisive-override" };
|
|
761
|
+
}
|
|
762
|
+
return { entry: leader, reasonKind: "only-adequate-concentration" };
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
function passesConcentration(candidate, role, modelUsage, providerTechnicalUsage, reviewerBuilderAdapter) {
|
|
766
|
+
const key = familyKey(candidate.model);
|
|
767
|
+
if ((modelUsage.get(key) ?? 0) >= MAX_ROLES_PER_MODEL) return false;
|
|
768
|
+
if (TECHNICAL_ROLES.includes(role) && (providerTechnicalUsage.get(candidate.model.adapterId) ?? 0) >= MAX_TECHNICAL_ROLES_PER_PROVIDER) return false;
|
|
769
|
+
if (role === "Reviewer" && reviewerBuilderAdapter != null && candidate.model.adapterId === reviewerBuilderAdapter) return false;
|
|
770
|
+
return true;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
/**
|
|
774
|
+
* Runs the coordinated portfolio assignment across every role's pool,
|
|
775
|
+
* tracking model/provider usage as it goes so later roles see the real
|
|
776
|
+
* concentration state left by earlier ones. Shared by buildAiTeam and
|
|
777
|
+
* buildEfficientTeam — they differ only in how each role's pool is built
|
|
778
|
+
* and how candidates are ordered within it (`makeSorter`).
|
|
779
|
+
* @param {Array<{role: string, better: string, pool: Array<{model: object, value: number}>, fullRanked?: Array<{model: object, value: number}>}>} rolePools -
|
|
780
|
+
* `fullRanked` (buildAiTeam only — see assignOneRole's Stage 2) is the
|
|
781
|
+
* role's full real eligible ranking, used only when `pool` (the narrow
|
|
782
|
+
* near-equivalence band) has no concentration-safe candidate.
|
|
783
|
+
* @param {(role: string, modelUsage: Map<string, number>, providerTechnicalUsage: Map<string, number>) => (candidates: Array<{model: object, value: number}>) => Array<{model: object, value: number}>} makeSorter -
|
|
784
|
+
* receives the SAME live Map instances this function mutates as it
|
|
785
|
+
* assigns roles, so a role's sort always sees the real concentration
|
|
786
|
+
* state left by every role assigned before it.
|
|
787
|
+
* @param {"capability"|"efficient"} mode
|
|
788
|
+
*/
|
|
789
|
+
function assignCoordinatedTeam(rolePools, makeSorter, mode) {
|
|
790
|
+
const order = orderRolesForAssignment(rolePools);
|
|
791
|
+
const modelUsage = new Map();
|
|
792
|
+
const providerTechnicalUsage = new Map();
|
|
793
|
+
const results = {};
|
|
794
|
+
let builderAdapter = null;
|
|
795
|
+
|
|
796
|
+
for (const role of order) {
|
|
797
|
+
const { better, pool, fullRanked } = rolePools.find((r) => r.role === role);
|
|
798
|
+
// Snapshot the concentration state as it stood BEFORE this role was
|
|
799
|
+
// assigned — describeEfficiencyDecision must explain a decision using
|
|
800
|
+
// the state that was actually true when it was made, never the
|
|
801
|
+
// portfolio's final state after every later role has also been
|
|
802
|
+
// assigned (which would misattribute a plain capability/price/etc.
|
|
803
|
+
// pick made before any concentration existed as if it had been a
|
|
804
|
+
// deliberate concentration-avoidance move).
|
|
805
|
+
const modelUsageSnapshot = new Map(modelUsage);
|
|
806
|
+
const providerUsageSnapshot = new Map(providerTechnicalUsage);
|
|
807
|
+
const result = assignOneRole({
|
|
808
|
+
role, pool, fullRanked, better, modelUsage, providerTechnicalUsage,
|
|
809
|
+
reviewerBuilderAdapter: role === "Reviewer" ? builderAdapter : null,
|
|
810
|
+
sortWithinAllowed: makeSorter(role, modelUsage, providerTechnicalUsage), mode
|
|
811
|
+
});
|
|
812
|
+
if (result) {
|
|
813
|
+
result.modelUsageSnapshot = modelUsageSnapshot;
|
|
814
|
+
result.providerUsageSnapshot = providerUsageSnapshot;
|
|
815
|
+
const key = familyKey(result.entry.model);
|
|
816
|
+
modelUsage.set(key, (modelUsage.get(key) ?? 0) + 1);
|
|
817
|
+
providerTechnicalUsage.set(result.entry.model.adapterId, (providerTechnicalUsage.get(result.entry.model.adapterId) ?? 0) + 1);
|
|
818
|
+
if (role === "Builder") builderAdapter = result.entry.model.adapterId;
|
|
819
|
+
}
|
|
820
|
+
results[role] = result;
|
|
821
|
+
}
|
|
822
|
+
return { results, modelUsage, providerTechnicalUsage };
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/**
|
|
826
|
+
* The real near-equivalence pool for a role: the percentile-ranked leader
|
|
827
|
+
* (`ranked[0]` — order comes from requiredRoleFit/capabilityPercentile)
|
|
828
|
+
* plus every other candidate within that role's own near-equivalence band
|
|
829
|
+
* (see ROLE_NEAR_EQUIVALENCE_BAND) of the leader's REAL, scale-normalized
|
|
830
|
+
* gap value (never the percentile itself — see leaderAdvantage's own
|
|
831
|
+
* comment). A candidate with no real gap value at all can't be honestly
|
|
832
|
+
* compared, so it's excluded from the pool rather than guessed into or
|
|
833
|
+
* out of it.
|
|
834
|
+
*/
|
|
835
|
+
function capabilityPool(ranked, better, role) {
|
|
836
|
+
if (!ranked.length) return [];
|
|
837
|
+
const leader = ranked[0];
|
|
838
|
+
if (leader.gapValue == null) return [leader];
|
|
839
|
+
const scale = Math.abs(leader.gapValue) || 1;
|
|
840
|
+
const band = nearEquivalenceBandFor(role);
|
|
841
|
+
return ranked.filter((entry) => entry.gapValue != null && Math.abs(leader.gapValue - entry.gapValue) / scale <= band);
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
/**
|
|
845
|
+
* CAPABILITY priority: real requiredRoleFit value first, then — for a
|
|
846
|
+
* real exact tie — which real pick has more trustworthy evidence behind
|
|
847
|
+
* it (RoleEvaluation.confidence: high beats medium beats low, never the
|
|
848
|
+
* score's own magnitude), then optionalRoleFit (a real pick with real
|
|
849
|
+
* evidence on the role's optional capabilities — e.g. instructionFollowing
|
|
850
|
+
* — beats one with none, purely as a tiebreak; never moves a model that's
|
|
851
|
+
* behind on required capabilities ahead of one that's tied or ahead —
|
|
852
|
+
* this only fires when `.value` is already an exact tie), then portfolio
|
|
853
|
+
* diversity (least-used model, then least-used provider), then a stable
|
|
854
|
+
* tiebreak.
|
|
855
|
+
* `getConfidenceRank`/`getOptionalFitRank` default to "always tied" for
|
|
856
|
+
* callers with no such signal (e.g. none was computed for this role).
|
|
857
|
+
*/
|
|
858
|
+
function sortByCapabilityPriority(candidates, better, modelUsage, providerTechnicalUsage, getConfidenceRank = () => 0, getOptionalFitRank = () => 0) {
|
|
859
|
+
return [...candidates].sort((a, b) => {
|
|
860
|
+
if (a.value !== b.value) return better === "max" ? b.value - a.value : a.value - b.value;
|
|
861
|
+
const aConfidence = getConfidenceRank(a.model);
|
|
862
|
+
const bConfidence = getConfidenceRank(b.model);
|
|
863
|
+
if (aConfidence !== bConfidence) return bConfidence - aConfidence; // higher confidence wins
|
|
864
|
+
const aOptionalFit = getOptionalFitRank(a.model);
|
|
865
|
+
const bOptionalFit = getOptionalFitRank(b.model);
|
|
866
|
+
if (aOptionalFit !== bOptionalFit) return bOptionalFit - aOptionalFit; // higher optionalRoleFit wins
|
|
867
|
+
const aModelUsage = modelUsage.get(familyKey(a.model)) ?? 0;
|
|
868
|
+
const bModelUsage = modelUsage.get(familyKey(b.model)) ?? 0;
|
|
869
|
+
if (aModelUsage !== bModelUsage) return aModelUsage - bModelUsage;
|
|
870
|
+
const aProviderUsage = providerTechnicalUsage.get(a.model.adapterId) ?? 0;
|
|
871
|
+
const bProviderUsage = providerTechnicalUsage.get(b.model.adapterId) ?? 0;
|
|
872
|
+
if (aProviderUsage !== bProviderUsage) return aProviderUsage - bProviderUsage;
|
|
873
|
+
const adapterCompare = a.model.adapterId.localeCompare(b.model.adapterId);
|
|
874
|
+
return adapterCompare !== 0 ? adapterCompare : a.model.modelId.localeCompare(b.model.modelId);
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
/**
|
|
879
|
+
* BEST FIT GLOBAL: el ganador de capability real por rol, sin ninguna
|
|
880
|
+
* coordinación de portafolio — nunca cede un rol a otro modelo por límite
|
|
881
|
+
* de familia, distribución por proveedor, o independencia Builder/
|
|
882
|
+
* Reviewer. Ese tipo de coordinación existe para PROJECT TEAM (buildAiTeam),
|
|
883
|
+
* un equipo real que se va a ejecutar en conjunto; esta función responde
|
|
884
|
+
* una pregunta distinta — "¿cuál es honestamente el mejor modelo para
|
|
885
|
+
* este rol, sin nada más en juego?" — así que Muse Spark nunca gana
|
|
886
|
+
* Architect aquí solo porque Astra ya esté "usado" en otro rol.
|
|
887
|
+
*
|
|
888
|
+
* Usa el mismo sistema de roles que buildAiTeam/buildEfficientTeam
|
|
889
|
+
* (buildAiTeamRoleDefinitions/ROLE_CAPABILITIES) — no el bestModelPerRole
|
|
890
|
+
* legado (ROLE_DEFINITIONS, con "Test Author" en vez de "Tester") — para
|
|
891
|
+
* que BEST FIT GLOBAL, EFFICIENT GLOBAL y PROJECT TEAM compartan
|
|
892
|
+
* exactamente el mismo conjunto de roles.
|
|
893
|
+
* @param {Array<object>} models - scoreAvailableModels() output
|
|
894
|
+
* @param {Record<string, {ok: boolean, reason?: string}>} [eligibility]
|
|
895
|
+
* @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>|null} [registry]
|
|
896
|
+
* @param {Record<string, string[]>} [roleCapabilities]
|
|
897
|
+
* @returns {Array<{role: string, primary: object, fallback: object|null, reason: string|null}>}
|
|
898
|
+
*/
|
|
899
|
+
export function bestModelPerRoleGlobal(models, eligibility = {}, registry = null, roleCapabilities = ROLE_CAPABILITIES) {
|
|
900
|
+
const effectiveRegistry = ensureRegistry(models, registry);
|
|
901
|
+
const { roleDefinitions } = buildAiTeamRoleDefinitions(effectiveRegistry, models, roleCapabilities);
|
|
902
|
+
const entries = [];
|
|
903
|
+
for (const { role, compute, better } of roleDefinitions) {
|
|
904
|
+
const globalRanked = rankBy(models, compute, better);
|
|
905
|
+
if (!globalRanked.length) continue;
|
|
906
|
+
const leader = globalRanked[0];
|
|
907
|
+
if (eligibility[leader.model.adapterId]?.ok === true) {
|
|
908
|
+
entries.push({ role, primary: toTeamModel(leader.model, true, effectiveRegistry), fallback: null, reason: null });
|
|
909
|
+
continue;
|
|
910
|
+
}
|
|
911
|
+
const eligibleRanked = rankEligible(models, eligibility, compute, better);
|
|
912
|
+
const fallback = eligibleRanked[0] ?? null;
|
|
913
|
+
entries.push({
|
|
914
|
+
role, primary: toTeamModel(leader.model, false, effectiveRegistry),
|
|
915
|
+
fallback: fallback ? toTeamModel(fallback.model, true, effectiveRegistry) : null,
|
|
916
|
+
reason: fallback
|
|
917
|
+
? `Real capability leader is temporarily unavailable (${eligibility[leader.model.adapterId]?.reason ?? "not eligible"}).`
|
|
918
|
+
: "No eligible provider currently covers this role."
|
|
919
|
+
});
|
|
920
|
+
}
|
|
921
|
+
return entries;
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
/**
|
|
925
|
+
* EFFICIENT GLOBAL: el ganador real de eficiencia por rol, con el mismo
|
|
926
|
+
* piso de capacidad (capabilityFloor) que buildEfficientTeam, pero sin
|
|
927
|
+
* ninguna coordinación de portafolio — el par natural de
|
|
928
|
+
* bestModelPerRoleGlobal. Pasa Maps de uso vacíos a sortByEfficiencyPriority
|
|
929
|
+
* a propósito: sin memoria de asignaciones previas, el desempate por
|
|
930
|
+
* "menos usado" nunca puede activarse, así que la elección cae siempre en
|
|
931
|
+
* la cadena real de eficiencia (costo/duración/precio/throughput) y,
|
|
932
|
+
* recién al final, en el desempate estable por adapterId/modelId.
|
|
933
|
+
* @param {Array<object>} models - scoreAvailableModels() output
|
|
934
|
+
* @param {Record<string, {ok: boolean, reason?: string}>} [eligibility]
|
|
935
|
+
* @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>|null} [registry]
|
|
936
|
+
* @param {{capabilityFloor?: number, providerCapacity?: object|null, roleCapabilities?: Record<string,string[]>}} [options]
|
|
937
|
+
* @returns {Array<{role: string, primary: object, fallback: object|null, reason: string|null}>}
|
|
938
|
+
*/
|
|
939
|
+
export function bestEfficientModelPerRoleGlobal(models, eligibility = {}, registry = null, options = {}) {
|
|
940
|
+
const { capabilityFloor = null, providerCapacity = null, roleCapabilities = ROLE_CAPABILITIES } = options;
|
|
941
|
+
const effectiveRegistry = ensureRegistry(models, registry);
|
|
942
|
+
const { roleDefinitions, gapValueByRole } = buildAiTeamRoleDefinitions(effectiveRegistry, models, roleCapabilities);
|
|
943
|
+
const noPortfolioUsage = new Map();
|
|
944
|
+
const entries = [];
|
|
945
|
+
for (const { role, compute, better } of roleDefinitions) {
|
|
946
|
+
const globalRanked = rankBy(models, compute, better);
|
|
947
|
+
if (!globalRanked.length) continue;
|
|
948
|
+
const eligibleRanked = attachGapValues(rankEligible(models, eligibility, compute, better), gapValueByRole[role]);
|
|
949
|
+
const globalLeader = globalRanked[0];
|
|
950
|
+
|
|
951
|
+
if (!eligibleRanked.length) {
|
|
952
|
+
entries.push({ role, primary: toTeamModel(globalLeader.model, false, effectiveRegistry), fallback: null, reason: "No eligible provider currently covers this role." });
|
|
953
|
+
continue;
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
const pool = adequateCandidates(eligibleRanked, eligibleRanked[0], better, resolveEfficientFloor(role, capabilityFloor));
|
|
957
|
+
const chosen = sortByEfficiencyPriority(pool, effectiveRegistry, providerCapacity, noPortfolioUsage, noPortfolioUsage, eligibleRanked[0])[0];
|
|
958
|
+
|
|
959
|
+
const globalLeaderEligible = eligibility[globalLeader.model.adapterId]?.ok === true;
|
|
960
|
+
const globalLeaderIsStrictlyBetter = better === "max" ? globalLeader.value > eligibleRanked[0].value : globalLeader.value < eligibleRanked[0].value;
|
|
961
|
+
if (!globalLeaderEligible && globalLeaderIsStrictlyBetter) {
|
|
962
|
+
entries.push({
|
|
963
|
+
role, primary: toTeamModel(globalLeader.model, false, effectiveRegistry), fallback: toTeamModel(chosen.model, true, effectiveRegistry),
|
|
964
|
+
reason: `Real capability leader is temporarily unavailable (${eligibility[globalLeader.model.adapterId]?.reason ?? "not eligible"}).`
|
|
965
|
+
});
|
|
966
|
+
continue;
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
const fallbackEntry = eligibleRanked.find((r) => r.model.adapterId !== chosen.model.adapterId);
|
|
970
|
+
entries.push({
|
|
971
|
+
role, primary: toTeamModel(chosen.model, true, effectiveRegistry),
|
|
972
|
+
fallback: fallbackEntry ? toTeamModel(fallbackEntry.model, true, effectiveRegistry) : null,
|
|
973
|
+
reason: describeEfficiencyDecision(chosen, eligibleRanked[0], effectiveRegistry, providerCapacity, noPortfolioUsage, noPortfolioUsage, pool.length).reason
|
|
974
|
+
});
|
|
975
|
+
}
|
|
976
|
+
return entries;
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
/**
|
|
980
|
+
* The "AI TEAM" distribution policy: decides which real, eligible provider
|
|
981
|
+
* actually gets reserved for each role, coordinated across the whole
|
|
982
|
+
* portfolio rather than seven independent per-role decisions — seven
|
|
983
|
+
* individual winners don't form a team. The policy, in order:
|
|
984
|
+
*
|
|
985
|
+
* 1. Capability floor — a role only considers models that report the real
|
|
986
|
+
* metric(s) it needs (unchanged from before: `rankBy` drops nulls).
|
|
987
|
+
* 2. Real capability decides — within each role's real near-equivalence
|
|
988
|
+
* pool (its own ROLE_NEAR_EQUIVALENCE_BAND — tighter for Architect/
|
|
989
|
+
* Reviewer than for Builder/Explorer/Tester), the highest-scoring
|
|
990
|
+
* eligible model wins, UNLESS the portfolio's concentration limits
|
|
991
|
+
* (max 2 roles per model, max 3 of 6 technical roles per provider)
|
|
992
|
+
* would be exceeded and a real, near-equivalent alternative exists —
|
|
993
|
+
* then the less-concentrated alternative is preferred instead.
|
|
994
|
+
* 3. Widened search (see assignOneRole's own doc) — if NO real candidate
|
|
995
|
+
* in that narrow band avoids concentration, the search widens to the
|
|
996
|
+
* role's full real eligible pool at the same 80% floor EFFICIENT TEAM
|
|
997
|
+
* uses, before ever resorting to a decisive-advantage override or
|
|
998
|
+
* repeating the leader. A portfolio limit must never silently exceed
|
|
999
|
+
* itself while a real, genuinely-adequate (if not near-identical)
|
|
1000
|
+
* alternative sits unexamined outside the tight band.
|
|
1001
|
+
* 4. A decisive real advantage (outside even that wide search) always
|
|
1002
|
+
* overrides the limits as a last resort: capability is never
|
|
1003
|
+
* sacrificed just to spread load.
|
|
1004
|
+
* 5. Review independence — Reviewer is additionally constrained off
|
|
1005
|
+
* Builder's own provider whenever a real, near-equivalent alternative
|
|
1006
|
+
* exists, so a model is never the sole judge of its own family's work.
|
|
1007
|
+
*
|
|
1008
|
+
* A temporarily unavailable real leader (quota/rate-limit) still never
|
|
1009
|
+
* just disappears: if the true global winner (across every candidate,
|
|
1010
|
+
* eligible or not) is stronger than the eligible pick, it's shown as the
|
|
1011
|
+
* primary, honestly flagged unavailable, with the eligible pick surfaced
|
|
1012
|
+
* as the fallback instead.
|
|
1013
|
+
* @param {Array<object>} models - scoreAvailableModels() output, computed
|
|
1014
|
+
* across every candidate provider regardless of current eligibility.
|
|
1015
|
+
* @param {Record<string, {ok: boolean, reason?: string}>} eligibility -
|
|
1016
|
+
* checkCandidate() results per adapterId.
|
|
1017
|
+
* @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>|null} [registry] -
|
|
1018
|
+
* when given, each primary/fallback also carries `corroboration` (real
|
|
1019
|
+
* Hugging Face / manufacturer-snapshot / Kairo-telemetry evidence for
|
|
1020
|
+
* that exact model) — purely informational, never part of the ranking.
|
|
1021
|
+
* @param {Record<string, string[]>} [roleCapabilities] - see
|
|
1022
|
+
* buildAiTeamRoleDefinitions's own doc — overrides the generic global
|
|
1023
|
+
* role->capability table with a PROJECT-specific one (project-strategy.js),
|
|
1024
|
+
* so the actual model selection responds to a real project's own
|
|
1025
|
+
* detected needs, not just which roles are active.
|
|
1026
|
+
* @returns {Array<{role: string, primary: object, fallback: object|null, reason: string|null}>}
|
|
1027
|
+
*/
|
|
1028
|
+
export function buildAiTeam(models, eligibility = {}, registry = null, roleCapabilities = ROLE_CAPABILITIES) {
|
|
1029
|
+
const effectiveRegistry = ensureRegistry(models, registry);
|
|
1030
|
+
const { roleDefinitions, evaluationsByRole, optionalEvaluationsByRole, gapValueByRole } = buildAiTeamRoleDefinitions(effectiveRegistry, models, roleCapabilities);
|
|
1031
|
+
const roleRankings = roleDefinitions.map(({ role, compute, better }) => {
|
|
1032
|
+
const eligibleRanked = attachGapValues(rankEligible(models, eligibility, compute, better), gapValueByRole[role]);
|
|
1033
|
+
// Comparable-before-provisional (see preferComparableCandidates's own
|
|
1034
|
+
// doc): a candidate with real evidence too thin on a required
|
|
1035
|
+
// capability to be genuinely comparable never outranks a broadly-
|
|
1036
|
+
// measured one, even at a higher raw capabilityPercentile — only
|
|
1037
|
+
// competes at all when every real eligible candidate is provisional.
|
|
1038
|
+
const { pool: ranked, usedProvisionalFallback } = preferComparableCandidates(eligibleRanked, evaluationsByRole[role]);
|
|
1039
|
+
return { role, compute, better, ranked, usedProvisionalFallback };
|
|
1040
|
+
});
|
|
1041
|
+
|
|
1042
|
+
const rolePools = roleRankings.map(({ role, better, ranked }) => ({
|
|
1043
|
+
role, better,
|
|
1044
|
+
pool: capabilityPool(ranked, better, role),
|
|
1045
|
+
fullRanked: ranked
|
|
1046
|
+
}));
|
|
1047
|
+
const makeSorter = (role, modelUsage, providerTechnicalUsage) => {
|
|
1048
|
+
const { better } = rolePools.find((r) => r.role === role);
|
|
1049
|
+
const roleEvaluations = evaluationsByRole[role];
|
|
1050
|
+
const roleOptionalEvaluations = optionalEvaluationsByRole[role];
|
|
1051
|
+
const getConfidenceRank = (model) => CONFIDENCE_RANK[roleEvaluations?.get(modelKey(model))?.confidence] ?? 0;
|
|
1052
|
+
const getOptionalFitRank = (model) => roleOptionalEvaluations?.get(modelKey(model))?.capabilityPercentile ?? 0;
|
|
1053
|
+
return (candidates) => sortByCapabilityPriority(candidates, better, modelUsage, providerTechnicalUsage, getConfidenceRank, getOptionalFitRank);
|
|
1054
|
+
};
|
|
1055
|
+
const { results } = assignCoordinatedTeam(rolePools, makeSorter, "capability");
|
|
1056
|
+
|
|
1057
|
+
const entries = [];
|
|
1058
|
+
for (const { role, compute, better } of roleDefinitions) {
|
|
1059
|
+
const result = results[role];
|
|
1060
|
+
const { ranked: eligibleRanked, usedProvisionalFallback } = roleRankings.find((r) => r.role === role);
|
|
1061
|
+
const globalRanked = rankBy(models, compute, better);
|
|
1062
|
+
if (!globalRanked.length) continue; // no model anywhere reports this role's real metric — never guessed
|
|
1063
|
+
// Real coverage/confidence for the model actually shown as primary
|
|
1064
|
+
// (see RoleEvaluation) — surfaced honestly as null rather than
|
|
1065
|
+
// fabricated when absent. Purely informational — /models --evidence's
|
|
1066
|
+
// own "UNSCORED"/incomplete-coverage detail, never part of the
|
|
1067
|
+
// ranking itself, which already happened above.
|
|
1068
|
+
const evalFor = (model) => evaluationsByRole[role]?.get(modelKey(model)) ?? null;
|
|
1069
|
+
|
|
1070
|
+
const requiredCapabilities = roleCapabilities[role]?.required ?? [];
|
|
1071
|
+
|
|
1072
|
+
if (!result) {
|
|
1073
|
+
const globalLeader = globalRanked[0];
|
|
1074
|
+
const evaluation = evalFor(globalLeader.model);
|
|
1075
|
+
entries.push({
|
|
1076
|
+
role, primary: toTeamModel(globalLeader.model, false, effectiveRegistry), fallback: null,
|
|
1077
|
+
reason: "No eligible provider currently covers this role.",
|
|
1078
|
+
coverage: evaluation?.coverage ?? null, confidence: evaluation?.confidence ?? null,
|
|
1079
|
+
decisionEvidence: buildDecisionEvidence({ evaluation, requiredCapabilities, decisionType: "fallback" })
|
|
1080
|
+
});
|
|
1081
|
+
continue;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
const chosen = result.entry;
|
|
1085
|
+
const globalLeader = globalRanked[0];
|
|
1086
|
+
const globalLeaderEligible = eligibility[globalLeader.model.adapterId]?.ok === true;
|
|
1087
|
+
const globalLeaderIsStrictlyBetter = better === "max" ? globalLeader.value > chosen.value : globalLeader.value < chosen.value;
|
|
1088
|
+
if (!globalLeaderEligible && globalLeaderIsStrictlyBetter) {
|
|
1089
|
+
const evaluation = evalFor(globalLeader.model);
|
|
1090
|
+
entries.push({
|
|
1091
|
+
role, primary: toTeamModel(globalLeader.model, false, effectiveRegistry), fallback: toTeamModel(chosen.model, true, effectiveRegistry),
|
|
1092
|
+
reason: `Real capability leader is temporarily unavailable (${eligibility[globalLeader.model.adapterId]?.reason ?? "not eligible"}).`,
|
|
1093
|
+
coverage: evaluation?.coverage ?? null, confidence: evaluation?.confidence ?? null,
|
|
1094
|
+
decisionEvidence: buildDecisionEvidence({ evaluation, requiredCapabilities, decisionType: "fallback" })
|
|
1095
|
+
});
|
|
1096
|
+
continue;
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
const fallbackEntry = eligibleRanked.find((r) => r.model.adapterId !== chosen.model.adapterId);
|
|
1100
|
+
const pool = rolePools.find((r) => r.role === role).pool;
|
|
1101
|
+
const reviewerLeaderWasBuilderAdapter = role === "Reviewer" && pool.length
|
|
1102
|
+
&& pool[0].model.adapterId === entries.find((e) => e.role === "Builder")?.primary.adapterId;
|
|
1103
|
+
let reason;
|
|
1104
|
+
let decisionType;
|
|
1105
|
+
if (usedProvisionalFallback) {
|
|
1106
|
+
// Every real eligible candidate for this role was provisional (real
|
|
1107
|
+
// evidence, just too thin on a required capability to be genuinely
|
|
1108
|
+
// comparable) — this pick is a real, honest fallback among them,
|
|
1109
|
+
// never presented as an ordinary capability win. Takes priority
|
|
1110
|
+
// over the other reason kinds below since it explains something
|
|
1111
|
+
// more fundamental about the WHOLE pool, not just this one pick.
|
|
1112
|
+
reason = "Only provisional evidence available for this role — no real candidate cleared comparable benchmark coverage.";
|
|
1113
|
+
decisionType = "fallback";
|
|
1114
|
+
} else if (reviewerLeaderWasBuilderAdapter && chosen.model.adapterId !== pool[0].model.adapterId) {
|
|
1115
|
+
reason = "Kept independent from Builder's provider.";
|
|
1116
|
+
decisionType = "diversity";
|
|
1117
|
+
} else if (result.reasonKind === "only-adequate-concentration") {
|
|
1118
|
+
reason = "Only adequate option — no real alternative avoids concentration without forcing a repeat.";
|
|
1119
|
+
decisionType = "fallback";
|
|
1120
|
+
} else if (result.reasonKind === "decisive-override") {
|
|
1121
|
+
reason = "Decisive real capability advantage — kept despite exceeding the concentration limit.";
|
|
1122
|
+
decisionType = "leader";
|
|
1123
|
+
} else if (result.reasonKind === "diversity") {
|
|
1124
|
+
reason = "Near-equivalent alternatives — assigned to a different model/provider to avoid concentration.";
|
|
1125
|
+
decisionType = "diversity";
|
|
1126
|
+
} else if (result.reasonKind === "wider-search-diversity") {
|
|
1127
|
+
reason = "No near-equivalent alternative avoided concentration — widened the search to the full real catalog and assigned a genuinely adequate model/provider instead.";
|
|
1128
|
+
decisionType = "diversity";
|
|
1129
|
+
} else {
|
|
1130
|
+
reason = null;
|
|
1131
|
+
decisionType = "leader";
|
|
1132
|
+
}
|
|
1133
|
+
const evaluation = evalFor(chosen.model);
|
|
1134
|
+
entries.push({
|
|
1135
|
+
role, primary: toTeamModel(chosen.model, true, effectiveRegistry),
|
|
1136
|
+
fallback: fallbackEntry ? toTeamModel(fallbackEntry.model, true, effectiveRegistry) : null,
|
|
1137
|
+
reason, coverage: evaluation?.coverage ?? null, confidence: evaluation?.confidence ?? null,
|
|
1138
|
+
decisionEvidence: buildDecisionEvidence({ evaluation, requiredCapabilities, decisionType })
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
1141
|
+
return entries;
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
// The real, per-MODEL signals EFFICIENT TEAM checks, in priority order, to
|
|
1145
|
+
// choose among candidates that already clear EFFICIENT_CAPABILITY_FLOOR
|
|
1146
|
+
// (see buildEfficientTeam) — never a blended score, each one only decides
|
|
1147
|
+
// when the previous ones don't (unknown or tied). This is the
|
|
1148
|
+
// "ModelEfficiency" side of the split: real per-task economics that
|
|
1149
|
+
// genuinely differ model to model.
|
|
1150
|
+
//
|
|
1151
|
+
// Provider quota (ProviderCapacity, see subscription-pressure-source.js)
|
|
1152
|
+
// is deliberately NOT one of these — it isn't a per-model measurement at
|
|
1153
|
+
// all (every model under a provider shares the exact same real number),
|
|
1154
|
+
// so it's resolved separately, by adapterId, and checked only as the very
|
|
1155
|
+
// last tiebreak (see sortByEfficiencyPriority/describeEfficiencyDecision) —
|
|
1156
|
+
// strictly after every real per-model signal AND the portfolio's own
|
|
1157
|
+
// concentration state have been exhausted. A provider's spare quota must
|
|
1158
|
+
// never, by itself, decide who wins a role over a model with genuinely
|
|
1159
|
+
// better per-task economics.
|
|
1160
|
+
const EFFICIENCY_DIMENSIONS = [
|
|
1161
|
+
{ key: "kairo.totalTokens", better: "min", label: "lower real observed token consumption" },
|
|
1162
|
+
{ key: "kairo.cost", better: "min", label: "lower real observed cost per task" },
|
|
1163
|
+
{ key: "kairo.durationMs", better: "min", label: "lower real observed duration" },
|
|
1164
|
+
// The real FULL price (input + output) — a public-price fallback, never
|
|
1165
|
+
// a fake stand-in for an already-paid subscription's real marginal
|
|
1166
|
+
// cost (the three kairo.* dimensions above are that real cost; this is
|
|
1167
|
+
// what's left when Kairo hasn't actually run the model yet). Falls
|
|
1168
|
+
// back to input-only when a real output price isn't known — never
|
|
1169
|
+
// invents one.
|
|
1170
|
+
{ key: "totalPricePerMTok", better: "min", label: "lower real full input+output price", resolve: resolveTotalPrice },
|
|
1171
|
+
{ key: "outputTokensPerSecond", better: "max", label: "higher reported throughput" }
|
|
1172
|
+
];
|
|
1173
|
+
|
|
1174
|
+
function resolveTotalPrice(registry, model) {
|
|
1175
|
+
const input = resolveMetric(registry, model, "priceInputPerMTok");
|
|
1176
|
+
if (input == null) return null;
|
|
1177
|
+
const output = resolveMetric(registry, model, "priceOutputPerMTok");
|
|
1178
|
+
return output == null ? input : input + output;
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
/** Resolves one EFFICIENCY_DIMENSIONS entry's real value for a model — its own `resolve` when it has one (a derived value, e.g. totalPricePerMTok), otherwise the plain registry/model field lookup every other dimension already used. */
|
|
1182
|
+
function resolveDimension(dimension, registry, model) {
|
|
1183
|
+
return dimension.resolve ? dimension.resolve(registry, model) : resolveMetric(registry, model, dimension.key);
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
/**
|
|
1187
|
+
* Resolves a real ProviderCapacity signal for a model's adapter — never
|
|
1188
|
+
* the model's own identity. Two models under the same adapter always
|
|
1189
|
+
* resolve to the exact same value here, because quota genuinely is an
|
|
1190
|
+
* account-wide, not per-model, real fact.
|
|
1191
|
+
* @param {Record<string, import("./subscription-pressure-source.js").ProviderCapacity>|null} providerCapacity
|
|
1192
|
+
* @param {{adapterId: string}} model
|
|
1193
|
+
*/
|
|
1194
|
+
function resolveProviderCapacity(providerCapacity, model) {
|
|
1195
|
+
return providerCapacity?.[model.adapterId]?.quotaRemainingPercent ?? null;
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
// EFFICIENT TEAM's capability floor: a candidate must retain at least this
|
|
1199
|
+
// fraction of the real capability leader's score (see "retention" below)
|
|
1200
|
+
// to be considered "adequate" for a role — a genuinely different policy
|
|
1201
|
+
// from NEAR_EQUIVALENCE_BAND's "almost identical" test. NEAR_EQUIVALENCE_BAND
|
|
1202
|
+
// (0.08, ~8%) keeps governing maximum-capability equivalence (AI TEAM's
|
|
1203
|
+
// near-equivalence pool, Reviewer independence) — it no longer governs
|
|
1204
|
+
// EFFICIENT TEAM. This floor is intentionally much wider: EFFICIENT
|
|
1205
|
+
// TEAM's job is "the minimum model that's still genuinely sufficient for
|
|
1206
|
+
// the role," not "whichever near-identical model happens to be cheaper."
|
|
1207
|
+
//
|
|
1208
|
+
// The floor is now risk-based, sourced from role-profiles.js's own
|
|
1209
|
+
// RoleProfile.riskLevel (already a real, deliberate per-role judgment —
|
|
1210
|
+
// see role-profiles.js's own doc for why Architect/Debugger/Reviewer are
|
|
1211
|
+
// "high" and Explorer is "low") rather than one flat number for every
|
|
1212
|
+
// role: a mistake from Architect/Debugger/Reviewer compounds across the
|
|
1213
|
+
// whole team or hits during a live failure, so EFFICIENT can afford to
|
|
1214
|
+
// give up less capability there than it can for Explorer, whose mistakes
|
|
1215
|
+
// are cheap to catch and retry. EFFICIENT_CAPABILITY_FLOOR (0.80) stays
|
|
1216
|
+
// exported as the "low"-risk value and the real fallback for a role with
|
|
1217
|
+
// no RoleProfile (a future addition, or a project-derived role name not
|
|
1218
|
+
// among the canonical six) — never calibrated against measured data the
|
|
1219
|
+
// way NEAR_EQUIVALENCE_BAND was, since "sufficient" is a product
|
|
1220
|
+
// decision, not something derivable from benchmark gaps alone.
|
|
1221
|
+
// `options.capabilityFloor` on buildEfficientTeam/bestEfficientModelPerRoleGlobal
|
|
1222
|
+
// still overrides ALL of this with one explicit number when a caller
|
|
1223
|
+
// wants that instead — resolveEfficientFloor only applies when it wasn't given.
|
|
1224
|
+
export const EFFICIENT_CAPABILITY_FLOOR = 0.80;
|
|
1225
|
+
|
|
1226
|
+
const EFFICIENT_FLOOR_BY_RISK = { high: 0.90, medium: 0.85, low: EFFICIENT_CAPABILITY_FLOOR };
|
|
1227
|
+
|
|
1228
|
+
/**
|
|
1229
|
+
* The real capability floor a role's EFFICIENT pick must clear — an
|
|
1230
|
+
* explicit `options.capabilityFloor` always wins (a caller's deliberate
|
|
1231
|
+
* override); otherwise resolved from the role's own real RoleProfile.riskLevel.
|
|
1232
|
+
* @param {string} role
|
|
1233
|
+
* @param {number|null} explicitFloor - `options.capabilityFloor`, or null/undefined when not overridden.
|
|
1234
|
+
* @returns {number}
|
|
1235
|
+
*/
|
|
1236
|
+
function resolveEfficientFloor(role, explicitFloor) {
|
|
1237
|
+
if (explicitFloor != null) return explicitFloor;
|
|
1238
|
+
const profile = getRoleProfile(role);
|
|
1239
|
+
return (profile && EFFICIENT_FLOOR_BY_RISK[profile.riskLevel]) ?? EFFICIENT_CAPABILITY_FLOOR;
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
/**
|
|
1243
|
+
* Picks the single real EFFICIENCY_DIMENSIONS entry to use as the Pareto
|
|
1244
|
+
* frontier's resource axis for one role's candidate pool — the
|
|
1245
|
+
* HIGHEST-PRIORITY dimension that at least one real candidate actually
|
|
1246
|
+
* has a value for (kairo.* real telemetry first, public price as
|
|
1247
|
+
* fallback, throughput last). Deliberately a SINGLE dimension, never a
|
|
1248
|
+
* blend: comparing retention against two different candidates' two
|
|
1249
|
+
* different real metrics would be comparing unlike things.
|
|
1250
|
+
* @returns {{key: string, better: "min"|"max", label: string, resolve?: Function}|null}
|
|
1251
|
+
*/
|
|
1252
|
+
function resolveResourceDimension(registry, candidates) {
|
|
1253
|
+
for (const dimension of EFFICIENCY_DIMENSIONS) {
|
|
1254
|
+
if (candidates.some((c) => resolveDimension(dimension, registry, c.model) != null)) return dimension;
|
|
1255
|
+
}
|
|
1256
|
+
return null;
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
/**
|
|
1260
|
+
* A real candidate's raw value on the chosen resource dimension — just a
|
|
1261
|
+
* thin resolveDimension wrapper kept separate so computeBalanceScores
|
|
1262
|
+
* reads clearly. Deliberately NOT normalized against another candidate's
|
|
1263
|
+
* value here (see computeBalanceScores's own doc for why a ratio against
|
|
1264
|
+
* the pool's cheapest candidate is wrong, and breaks outright on a real
|
|
1265
|
+
* free/zero-cost model).
|
|
1266
|
+
*/
|
|
1267
|
+
function resourceValue(dimension, registry, model) {
|
|
1268
|
+
return dimension ? resolveDimension(dimension, registry, model) : null;
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
/**
|
|
1272
|
+
* The real Pareto balance-point scores for one role's pool — computed
|
|
1273
|
+
* once across the whole pool, never pairwise, because a "balance point"
|
|
1274
|
+
* is inherently relative to the pool's own real extremes. For every
|
|
1275
|
+
* candidate with a real value on the chosen resource dimension, both real
|
|
1276
|
+
* retention (gapValue as a fraction of the leader's) and real resource
|
|
1277
|
+
* pressure are normalized DIRECTLY against the POOL's own real min/max on
|
|
1278
|
+
* each axis — `(value - min) / (max - min)` for a "min is better"
|
|
1279
|
+
* dimension (0 at the pool's own cheapest/fastest-draining, 1 at its
|
|
1280
|
+
* worst), or `(max - value) / (max - min)` for a "max is better" one
|
|
1281
|
+
* (e.g. throughput) — then scored `retentionNorm - pressureNorm`.
|
|
1282
|
+
* Maximizing this rewards the candidate closest to the "good corner" —
|
|
1283
|
+
* high real retention AND low real resource pressure RELATIVE TO ITS
|
|
1284
|
+
* PEERS — a genuine knee/balance point.
|
|
1285
|
+
*
|
|
1286
|
+
* This is deliberately NOT retention/pressure (a plain ratio dividing by
|
|
1287
|
+
* the pool's cheapest real value): besides always anchoring the cheapest
|
|
1288
|
+
* candidate's own pressure at 1.0 (collapsing EFFICIENT into ECONOMY —
|
|
1289
|
+
* see the git history for that bug), a real free/zero-cost model in the
|
|
1290
|
+
* pool (Artificial Analysis's own raw dataset carries hundreds of these,
|
|
1291
|
+
* even where none currently reach an eligible provider catalog) makes
|
|
1292
|
+
* that division either NaN (0/0, when it's also the cheapest) or Infinity
|
|
1293
|
+
* (anything/0 elsewhere), silently corrupting the whole pool's comparison
|
|
1294
|
+
* and always forcing the capability leader to win by default. Direct
|
|
1295
|
+
* min/max normalization has
|
|
1296
|
+
* no such division and handles a real zero exactly like any other value.
|
|
1297
|
+
*
|
|
1298
|
+
* With exactly two candidates, the two extremes always score identically
|
|
1299
|
+
* (0 each, by construction — there is no "middle" to find with only two
|
|
1300
|
+
* points), so a real two-way choice correctly falls through to
|
|
1301
|
+
* EFFICIENCY_DIMENSIONS' own cascade (see sortByEfficiencyPriority)
|
|
1302
|
+
* instead of this function arbitrarily favoring either endpoint.
|
|
1303
|
+
* @returns {Map<string, number>|null} modelKey -> balance score, or null
|
|
1304
|
+
* when there's no real leader/dimension/enough real data to compare.
|
|
1305
|
+
*/
|
|
1306
|
+
function computeBalanceScores(candidates, leader, dimension, registry) {
|
|
1307
|
+
if (!leader?.gapValue || !dimension) return null;
|
|
1308
|
+
const points = candidates
|
|
1309
|
+
.map((c) => ({
|
|
1310
|
+
key: modelKey(c.model),
|
|
1311
|
+
retention: (c.gapValue ?? 0) / leader.gapValue,
|
|
1312
|
+
value: resourceValue(dimension, registry, c.model)
|
|
1313
|
+
}))
|
|
1314
|
+
.filter((p) => p.value != null && Number.isFinite(p.value));
|
|
1315
|
+
if (points.length < 2) return null;
|
|
1316
|
+
const retentions = points.map((p) => p.retention);
|
|
1317
|
+
const values = points.map((p) => p.value);
|
|
1318
|
+
const minRet = Math.min(...retentions);
|
|
1319
|
+
const retRange = Math.max(...retentions) - minRet || 1;
|
|
1320
|
+
const minValue = Math.min(...values);
|
|
1321
|
+
const maxValue = Math.max(...values);
|
|
1322
|
+
const hasValueRange = maxValue > minValue;
|
|
1323
|
+
const valueRange = maxValue - minValue || 1;
|
|
1324
|
+
const scores = new Map();
|
|
1325
|
+
for (const p of points) {
|
|
1326
|
+
const retentionNorm = (p.retention - minRet) / retRange;
|
|
1327
|
+
const pressureNorm = !hasValueRange ? 0
|
|
1328
|
+
: dimension.better === "min" ? (p.value - minValue) / valueRange : (maxValue - p.value) / valueRange;
|
|
1329
|
+
scores.set(p.key, retentionNorm - pressureNorm);
|
|
1330
|
+
}
|
|
1331
|
+
return scores;
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
/**
|
|
1335
|
+
* Orders real adequate candidates by their Pareto balance-point score
|
|
1336
|
+
* first (see computeBalanceScores's own doc — a real knee/balance point
|
|
1337
|
+
* relative to the pool's own extremes, never just "cheapest wins" or
|
|
1338
|
+
* "closest to QUALITY wins") when a real role leader is given; the EFFICIENCY_DIMENSIONS
|
|
1339
|
+
* priority chain (real per-model economics) then only ever breaks a
|
|
1340
|
+
* genuine tie in that score. Falls through to the portfolio's own
|
|
1341
|
+
* concentration state (prefer the less-used model, then the less-used
|
|
1342
|
+
* provider), then a real ProviderCapacity signal (quota, resolved
|
|
1343
|
+
* per-adapter, never per-model), and only then a stable adapterId/modelId
|
|
1344
|
+
* tiebreak so the same real near-tie always resolves the same way run to
|
|
1345
|
+
* run.
|
|
1346
|
+
* @param {Array<{model: object, value: number, gapValue?: number}>} candidates
|
|
1347
|
+
* @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>} registry
|
|
1348
|
+
* @param {Record<string, import("./subscription-pressure-source.js").ProviderCapacity>|null} providerCapacity
|
|
1349
|
+
* @param {Map<string, number>} modelUsage
|
|
1350
|
+
* @param {Map<string, number>} providerTechnicalUsage
|
|
1351
|
+
* @param {{model: object, gapValue?: number}|null} [leader] - the role's real QUALITY leader (retention reference); when omitted, falls back to the plain EFFICIENCY_DIMENSIONS-only ordering (no real gapValue reference to compute retention against).
|
|
1352
|
+
*/
|
|
1353
|
+
function sortByEfficiencyPriority(candidates, registry, providerCapacity, modelUsage, providerTechnicalUsage, leader = null) {
|
|
1354
|
+
const dimension = leader ? resolveResourceDimension(registry, candidates) : null;
|
|
1355
|
+
const balanceScores = dimension ? computeBalanceScores(candidates, leader, dimension, registry) : null;
|
|
1356
|
+
return [...candidates].sort((a, b) => {
|
|
1357
|
+
if (balanceScores) {
|
|
1358
|
+
const aScore = balanceScores.get(modelKey(a.model));
|
|
1359
|
+
const bScore = balanceScores.get(modelKey(b.model));
|
|
1360
|
+
if (aScore != null && bScore != null && aScore !== bScore) return bScore - aScore; // higher real balance score wins
|
|
1361
|
+
}
|
|
1362
|
+
for (const dim of EFFICIENCY_DIMENSIONS) {
|
|
1363
|
+
const av = resolveDimension(dim, registry, a.model);
|
|
1364
|
+
const bv = resolveDimension(dim, registry, b.model);
|
|
1365
|
+
if (av == null || bv == null || av === bv) continue;
|
|
1366
|
+
return dim.better === "max" ? bv - av : av - bv;
|
|
1367
|
+
}
|
|
1368
|
+
const aModelUsage = modelUsage.get(familyKey(a.model)) ?? 0;
|
|
1369
|
+
const bModelUsage = modelUsage.get(familyKey(b.model)) ?? 0;
|
|
1370
|
+
if (aModelUsage !== bModelUsage) return aModelUsage - bModelUsage;
|
|
1371
|
+
const aProviderUsage = providerTechnicalUsage.get(a.model.adapterId) ?? 0;
|
|
1372
|
+
const bProviderUsage = providerTechnicalUsage.get(b.model.adapterId) ?? 0;
|
|
1373
|
+
if (aProviderUsage !== bProviderUsage) return aProviderUsage - bProviderUsage;
|
|
1374
|
+
const aQuota = resolveProviderCapacity(providerCapacity, a.model);
|
|
1375
|
+
const bQuota = resolveProviderCapacity(providerCapacity, b.model);
|
|
1376
|
+
if (aQuota != null && bQuota != null && aQuota !== bQuota) return bQuota - aQuota; // higher headroom wins
|
|
1377
|
+
const adapterCompare = a.model.adapterId.localeCompare(b.model.adapterId);
|
|
1378
|
+
return adapterCompare !== 0 ? adapterCompare : a.model.modelId.localeCompare(b.model.modelId);
|
|
1379
|
+
});
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
/**
|
|
1383
|
+
* Names the real dimension that actually decided an EFFICIENT TEAM pick
|
|
1384
|
+
* AND classifies the decision itself (for /models --evidence's structured
|
|
1385
|
+
* decisionEvidence — see buildDecisionEvidence) — never recalculated by
|
|
1386
|
+
* the UI, just labeled here where the real decision already happened.
|
|
1387
|
+
* Returns `{ reason: null, decisionType: "leader", savings: null }` when
|
|
1388
|
+
* chosen IS the unremarkable capability leader itself.
|
|
1389
|
+
* @param {{model: object, value: number}} chosen
|
|
1390
|
+
* @param {{model: object, value: number}} leader
|
|
1391
|
+
* @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>} registry
|
|
1392
|
+
* @param {Record<string, import("./subscription-pressure-source.js").ProviderCapacity>|null} providerCapacity
|
|
1393
|
+
* @param {Map<string, number>} modelUsage
|
|
1394
|
+
* @param {Map<string, number>} providerTechnicalUsage
|
|
1395
|
+
* @param {number} poolSize - how many real candidates were actually being
|
|
1396
|
+
* compared when this pick was made — a resource-dimension win over only
|
|
1397
|
+
* 2 real candidates is always a "tiebreak" (two points can never have a
|
|
1398
|
+
* real Pareto "middle" — see computeBalanceScores's own doc), while 3+
|
|
1399
|
+
* real candidates is a genuine "pareto" balance-point decision.
|
|
1400
|
+
* @returns {{reason: string|null, decisionType: "leader"|"pareto"|"tiebreak"|"diversity", savings: {dimension: string, label: string, from: number, to: number}|null}}
|
|
1401
|
+
*/
|
|
1402
|
+
function describeEfficiencyDecision(chosen, leader, registry, providerCapacity, modelUsage, providerTechnicalUsage, poolSize) {
|
|
1403
|
+
if (chosen.model.adapterId === leader.model.adapterId && chosen.model.modelId === leader.model.modelId) {
|
|
1404
|
+
return { reason: null, decisionType: "leader", savings: null };
|
|
1405
|
+
}
|
|
1406
|
+
// Real retention against the role's own QUALITY leader — "how much
|
|
1407
|
+
// real capability did this Pareto balance-point pick actually keep" —
|
|
1408
|
+
// prefixed onto every reason below, not just the resource-dimension
|
|
1409
|
+
// one, since it's real context for ANY reason a non-leader was chosen.
|
|
1410
|
+
const retentionPct = leader.gapValue ? Math.round(((chosen.gapValue ?? 0) / leader.gapValue) * 100) : null;
|
|
1411
|
+
const retentionPrefix = retentionPct != null ? `Retains ~${retentionPct}% of QUALITY's real capability — ` : "Adequate capability — ";
|
|
1412
|
+
const dimension = resolveResourceDimension(registry, [chosen, leader]);
|
|
1413
|
+
if (dimension) {
|
|
1414
|
+
const chosenValue = resolveDimension(dimension, registry, chosen.model);
|
|
1415
|
+
const leaderValue = resolveDimension(dimension, registry, leader.model);
|
|
1416
|
+
if (chosenValue != null && leaderValue != null && chosenValue !== leaderValue) {
|
|
1417
|
+
const chosenIsBetter = dimension.better === "max" ? chosenValue > leaderValue : chosenValue < leaderValue;
|
|
1418
|
+
if (chosenIsBetter) {
|
|
1419
|
+
return {
|
|
1420
|
+
reason: `${retentionPrefix}chosen for ${dimension.label}.`,
|
|
1421
|
+
decisionType: poolSize >= 3 ? "pareto" : "tiebreak",
|
|
1422
|
+
savings: { dimension: dimension.key, label: dimension.label, from: leaderValue, to: chosenValue }
|
|
1423
|
+
};
|
|
1424
|
+
}
|
|
1425
|
+
// this dimension didn't favor the switch; a later real signal must
|
|
1426
|
+
// have — no real number to report from THIS one, fall through.
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
const chosenModelUsage = modelUsage.get(familyKey(chosen.model)) ?? 0;
|
|
1430
|
+
const leaderModelUsage = modelUsage.get(familyKey(leader.model)) ?? 0;
|
|
1431
|
+
const chosenProviderUsage = providerTechnicalUsage.get(chosen.model.adapterId) ?? 0;
|
|
1432
|
+
const leaderProviderUsage = providerTechnicalUsage.get(leader.model.adapterId) ?? 0;
|
|
1433
|
+
if (chosenModelUsage < leaderModelUsage || chosenProviderUsage < leaderProviderUsage) {
|
|
1434
|
+
return { reason: `${retentionPrefix}assigned to a different model/provider to avoid concentration.`, decisionType: "diversity", savings: null };
|
|
1435
|
+
}
|
|
1436
|
+
const chosenQuota = resolveProviderCapacity(providerCapacity, chosen.model);
|
|
1437
|
+
const leaderQuota = resolveProviderCapacity(providerCapacity, leader.model);
|
|
1438
|
+
if (chosenQuota != null && leaderQuota != null && chosenQuota > leaderQuota) {
|
|
1439
|
+
return { reason: `${retentionPrefix}chosen for lower real provider quota pressure.`, decisionType: "tiebreak", savings: null };
|
|
1440
|
+
}
|
|
1441
|
+
return {
|
|
1442
|
+
reason: `${retentionPrefix}chosen by a stable tiebreak, no real consumption/cost/duration/price/speed/concentration/quota signal distinguished them.`,
|
|
1443
|
+
decisionType: "tiebreak", savings: null
|
|
1444
|
+
};
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1447
|
+
/**
|
|
1448
|
+
* The single, real decision receipt behind one team entry — real per-
|
|
1449
|
+
* capability benchmark coverage (identities, not sources — see
|
|
1450
|
+
* activeBenchmarkCountForCapability/isCapabilityComparable), real
|
|
1451
|
+
* confidence/provisional state, and — only for EFFICIENT, where these
|
|
1452
|
+
* concepts actually apply — real retention against the QUALITY leader,
|
|
1453
|
+
* the real risk-based floor that had to be cleared, and the real
|
|
1454
|
+
* decisionType/savings a Pareto/tiebreak/diversity pick actually used.
|
|
1455
|
+
* Computed once here, during selection, so /models --evidence only ever
|
|
1456
|
+
* RENDERS this — it never recalculates coverage, retention, or savings
|
|
1457
|
+
* itself.
|
|
1458
|
+
* @param {{evaluation: import("./capability-scoring.js").RoleEvaluation|null, requiredCapabilities: string[], decisionType: "leader"|"pareto"|"tiebreak"|"diversity"|"fallback", retention?: number|null, requiredFloor?: number|null, riskLevel?: string|null, savings?: object|null}} params
|
|
1459
|
+
*/
|
|
1460
|
+
function buildDecisionEvidence({ evaluation, requiredCapabilities, decisionType, retention = null, requiredFloor = null, riskLevel = null, savings = null }) {
|
|
1461
|
+
const coverage = {};
|
|
1462
|
+
for (const capability of requiredCapabilities ?? []) {
|
|
1463
|
+
const have = evaluation?.benchmarkCountsByCapability?.[capability] ?? 0;
|
|
1464
|
+
const active = activeBenchmarkCountForCapability(capability);
|
|
1465
|
+
coverage[capability] = { have, active, comparable: active > 0 ? isCapabilityComparable(capability, have) : true };
|
|
1466
|
+
}
|
|
1467
|
+
return {
|
|
1468
|
+
coverage,
|
|
1469
|
+
confidence: evaluation?.confidence ?? null,
|
|
1470
|
+
isProvisional: evaluation?.isProvisional ?? false,
|
|
1471
|
+
decisionType,
|
|
1472
|
+
retention,
|
|
1473
|
+
requiredFloor,
|
|
1474
|
+
riskLevel,
|
|
1475
|
+
savings
|
|
1476
|
+
};
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
/**
|
|
1480
|
+
* Which eligible, ranked candidates are "adequate" for a role under the
|
|
1481
|
+
* capability-floor policy — retain at least `capabilityFloor` fraction of
|
|
1482
|
+
* the real leader's score. Every current real role is ranked "max"
|
|
1483
|
+
* (higher capability wins); a defensive `better !== "max"` early return
|
|
1484
|
+
* exists below for any future "min"-ranked role (lower-is-better, e.g. a
|
|
1485
|
+
* real cost signal) — applying a capability floor to a value that isn't
|
|
1486
|
+
* a capability score at all would be applying it to the wrong axis
|
|
1487
|
+
* entirely, so such a role's every eligible candidate is treated as
|
|
1488
|
+
* already "adequate" rather than floor-filtered.
|
|
1489
|
+
*/
|
|
1490
|
+
function adequateCandidates(eligibleRanked, leader, better, capabilityFloor) {
|
|
1491
|
+
if (!eligibleRanked.length) return [];
|
|
1492
|
+
if (better !== "max") return eligibleRanked;
|
|
1493
|
+
// The floor compares REAL, scale-normalized gap values (never the
|
|
1494
|
+
// rank-only percentile — see leaderAdvantage's comment), so a genuinely
|
|
1495
|
+
// 85%-capable real alternative still clears an 80% floor even with only
|
|
1496
|
+
// 2 real candidates, instead of reading as a flat 0% (percentile's
|
|
1497
|
+
// runner-up value with 2 candidates).
|
|
1498
|
+
if (leader.gapValue == null) return [leader];
|
|
1499
|
+
const floorValue = leader.gapValue * capabilityFloor;
|
|
1500
|
+
return eligibleRanked.filter((entry) => entry.gapValue != null && entry.gapValue >= floorValue);
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
/**
|
|
1504
|
+
* EFFICIENT TEAM: a real Pareto balance point between capability and
|
|
1505
|
+
* resource cost for each role — deliberately NOT "whichever candidate is
|
|
1506
|
+
* cheapest" (that's ECONOMY, a separate concept EFFICIENT must never
|
|
1507
|
+
* collapse into — see computeBalanceScores's own doc for why a plain
|
|
1508
|
+
* retention/pressure ratio would do exactly that). Among eligible
|
|
1509
|
+
* candidates that clear the role's own risk-based capability floor (see
|
|
1510
|
+
* resolveEfficientFloor — 90% for high-risk roles, 85% medium, 80% low,
|
|
1511
|
+
* sourced from RoleProfile.riskLevel; `options.capabilityFloor` overrides
|
|
1512
|
+
* this for every role when explicitly given), the real candidate whose
|
|
1513
|
+
* capability retention and real resource pressure sit at the pool's own
|
|
1514
|
+
* genuine knee/balance point wins — never the pool's cheapest-adequate
|
|
1515
|
+
* extreme merely because it's cheapest, and never the raw capability
|
|
1516
|
+
* leader merely because it's most capable. EFFICIENCY_DIMENSIONS (real
|
|
1517
|
+
* token consumption, then cost, duration, price, throughput) only ever
|
|
1518
|
+
* breaks a genuine tie in that balance score, then the portfolio's own
|
|
1519
|
+
* concentration state, then a provider's real quota headroom. Never
|
|
1520
|
+
* invents a savings percentage or a blended score — only ever orders by a
|
|
1521
|
+
* real, already-connected signal, and falls back to a stable tiebreak
|
|
1522
|
+
* when none of them distinguish the candidates.
|
|
1523
|
+
*
|
|
1524
|
+
* Astra/Fable/Opus-class leaders can still appear here — precisely when
|
|
1525
|
+
* no smaller real model clears the floor, EFFICIENT TEAM shows the exact
|
|
1526
|
+
* same model as AI TEAM for that role, with an honest "Only adequate
|
|
1527
|
+
* option" reason rather than a fabricated savings claim.
|
|
1528
|
+
* @param {Array<object>} models - scoreAvailableModels() output, computed
|
|
1529
|
+
* across every candidate provider regardless of current eligibility.
|
|
1530
|
+
* @param {Record<string, {ok: boolean, reason?: string}>} eligibility
|
|
1531
|
+
* @param {ReturnType<import("./model-capability-registry.js").createCapabilityRegistry>|null} [registry]
|
|
1532
|
+
* @param {object} [options]
|
|
1533
|
+
* @param {number} [options.capabilityFloor] - fraction of the leader's
|
|
1534
|
+
* real score a candidate must retain to be considered adequate
|
|
1535
|
+
* (default 0.80).
|
|
1536
|
+
* @param {Record<string, import("./subscription-pressure-source.js").ProviderCapacity>|null} [options.providerCapacity] -
|
|
1537
|
+
* real per-adapter quota headroom (see subscription-pressure-source.js's
|
|
1538
|
+
* buildProviderCapacity) — a PROVIDER-level signal, checked only as the
|
|
1539
|
+
* very last tiebreak, strictly after every per-model EFFICIENCY_DIMENSIONS
|
|
1540
|
+
* signal AND the portfolio's own concentration state. Never blended
|
|
1541
|
+
* with or treated as evidence about a specific model's own efficiency.
|
|
1542
|
+
* @returns {Array<{role: string, primary: object, fallback: object|null, reason: string|null}>}
|
|
1543
|
+
*/
|
|
1544
|
+
export function buildEfficientTeam(models, eligibility = {}, registry = null, options = {}) {
|
|
1545
|
+
const { capabilityFloor = null, providerCapacity = null, roleCapabilities = ROLE_CAPABILITIES } = options;
|
|
1546
|
+
const effectiveRegistry = ensureRegistry(models, registry);
|
|
1547
|
+
const { roleDefinitions, evaluationsByRole, gapValueByRole } = buildAiTeamRoleDefinitions(effectiveRegistry, models, roleCapabilities);
|
|
1548
|
+
const roleRankings = roleDefinitions.map(({ role, compute, better }) => {
|
|
1549
|
+
const eligibleRanked = attachGapValues(rankEligible(models, eligibility, compute, better), gapValueByRole[role]);
|
|
1550
|
+
// Same comparable-before-provisional policy as buildAiTeam (see
|
|
1551
|
+
// preferComparableCandidates's own doc) — the real, risk-based
|
|
1552
|
+
// capability floor below applies WITHIN whichever tier this produces,
|
|
1553
|
+
// never across both at once, so a thin, provisional candidate's real
|
|
1554
|
+
// value can't let it clear the floor ahead of a genuinely comparable one.
|
|
1555
|
+
const { pool: ranked, usedProvisionalFallback } = preferComparableCandidates(eligibleRanked, evaluationsByRole[role]);
|
|
1556
|
+
return { role, compute, better, ranked, usedProvisionalFallback };
|
|
1557
|
+
});
|
|
1558
|
+
|
|
1559
|
+
const rolePools = roleRankings.map(({ role, better, ranked }) => ({
|
|
1560
|
+
role, better,
|
|
1561
|
+
pool: adequateCandidates(ranked, ranked[0], better, resolveEfficientFloor(role, capabilityFloor)),
|
|
1562
|
+
// The real QUALITY leader — retention reference for the Pareto
|
|
1563
|
+
// balance-point score (see sortByEfficiencyPriority's own doc). The
|
|
1564
|
+
// TRUE leader of the comparable-preferred `ranked` list, not just
|
|
1565
|
+
// `pool[0]` (pool is already floor-filtered, but preserves order —
|
|
1566
|
+
// ranked[0] and pool[0] are the same real model as long as the
|
|
1567
|
+
// leader itself clears its own floor, which it trivially always does).
|
|
1568
|
+
leader: ranked[0] ?? null
|
|
1569
|
+
}));
|
|
1570
|
+
const makeSorter = (role, modelUsage, providerTechnicalUsage) => {
|
|
1571
|
+
const { leader } = rolePools.find((r) => r.role === role);
|
|
1572
|
+
return (candidates) => sortByEfficiencyPriority(candidates, effectiveRegistry, providerCapacity, modelUsage, providerTechnicalUsage, leader);
|
|
1573
|
+
};
|
|
1574
|
+
const { results } = assignCoordinatedTeam(rolePools, makeSorter, "efficient");
|
|
1575
|
+
|
|
1576
|
+
const entries = [];
|
|
1577
|
+
for (const { role, compute, better } of roleDefinitions) {
|
|
1578
|
+
const result = results[role];
|
|
1579
|
+
const { ranked: eligibleRanked, usedProvisionalFallback } = roleRankings.find((r) => r.role === role);
|
|
1580
|
+
const globalRanked = rankBy(models, compute, better);
|
|
1581
|
+
if (!globalRanked.length) continue; // no model anywhere reports this role's real metric — never guessed
|
|
1582
|
+
|
|
1583
|
+
const requiredCapabilities = roleCapabilities[role]?.required ?? [];
|
|
1584
|
+
const requiredFloor = resolveEfficientFloor(role, capabilityFloor);
|
|
1585
|
+
const riskLevel = getRoleProfile(role)?.riskLevel ?? null;
|
|
1586
|
+
|
|
1587
|
+
if (!eligibleRanked.length) {
|
|
1588
|
+
const globalLeader = globalRanked[0];
|
|
1589
|
+
entries.push({
|
|
1590
|
+
role, primary: toTeamModel(globalLeader.model, false, effectiveRegistry), fallback: null,
|
|
1591
|
+
reason: "No eligible provider currently covers this role.",
|
|
1592
|
+
decisionEvidence: buildDecisionEvidence({ evaluation: null, requiredCapabilities, decisionType: "fallback", requiredFloor, riskLevel })
|
|
1593
|
+
});
|
|
1594
|
+
continue;
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
const leader = eligibleRanked[0];
|
|
1598
|
+
const chosen = result.entry;
|
|
1599
|
+
const evalFor = (model) => evaluationsByRole[role]?.get(modelKey(model)) ?? null;
|
|
1600
|
+
const retention = leader.gapValue ? (chosen.gapValue ?? 0) / leader.gapValue : null;
|
|
1601
|
+
|
|
1602
|
+
const globalLeader = globalRanked[0];
|
|
1603
|
+
const globalLeaderEligible = eligibility[globalLeader.model.adapterId]?.ok === true;
|
|
1604
|
+
const globalLeaderIsStrictlyBetter = better === "max" ? globalLeader.value > leader.value : globalLeader.value < leader.value;
|
|
1605
|
+
if (!globalLeaderEligible && globalLeaderIsStrictlyBetter) {
|
|
1606
|
+
// Same "unavailable real leader" transparency AI TEAM has — never
|
|
1607
|
+
// hidden, with the efficient real pick among the rest as fallback.
|
|
1608
|
+
entries.push({
|
|
1609
|
+
role, primary: toTeamModel(globalLeader.model, false, effectiveRegistry), fallback: toTeamModel(chosen.model, true, effectiveRegistry),
|
|
1610
|
+
reason: `Real capability leader is temporarily unavailable (${eligibility[globalLeader.model.adapterId]?.reason ?? "not eligible"}).`,
|
|
1611
|
+
decisionEvidence: buildDecisionEvidence({ evaluation: evalFor(chosen.model), requiredCapabilities, decisionType: "fallback", retention, requiredFloor, riskLevel })
|
|
1612
|
+
});
|
|
1613
|
+
continue;
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
const fallbackEntry = eligibleRanked.find((r) => r.model.adapterId !== chosen.model.adapterId);
|
|
1617
|
+
let reason;
|
|
1618
|
+
let decisionType;
|
|
1619
|
+
let savings = null;
|
|
1620
|
+
if (usedProvisionalFallback) {
|
|
1621
|
+
reason = "Only provisional evidence available for this role — no real candidate cleared comparable benchmark coverage.";
|
|
1622
|
+
decisionType = "fallback";
|
|
1623
|
+
} else if (result.reasonKind === "only-adequate-floor") {
|
|
1624
|
+
reason = "Only adequate option — no real alternative clears the capability floor.";
|
|
1625
|
+
decisionType = "fallback";
|
|
1626
|
+
} else if (result.reasonKind === "only-adequate-concentration") {
|
|
1627
|
+
reason = "Only adequate option — no real alternative avoids concentration without forcing a repeat.";
|
|
1628
|
+
decisionType = "fallback";
|
|
1629
|
+
} else if (result.reasonKind === "decisive-override") {
|
|
1630
|
+
reason = "Decisive real capability advantage — kept despite exceeding the concentration limit.";
|
|
1631
|
+
decisionType = "leader";
|
|
1632
|
+
} else {
|
|
1633
|
+
const decision = describeEfficiencyDecision(chosen, leader, effectiveRegistry, providerCapacity, result.modelUsageSnapshot, result.providerUsageSnapshot, result.poolSize ?? 1);
|
|
1634
|
+
reason = decision.reason;
|
|
1635
|
+
decisionType = decision.decisionType;
|
|
1636
|
+
savings = decision.savings;
|
|
1637
|
+
}
|
|
1638
|
+
entries.push({
|
|
1639
|
+
role, primary: toTeamModel(chosen.model, true, effectiveRegistry),
|
|
1640
|
+
fallback: fallbackEntry ? toTeamModel(fallbackEntry.model, true, effectiveRegistry) : null,
|
|
1641
|
+
reason,
|
|
1642
|
+
decisionEvidence: buildDecisionEvidence({ evaluation: evalFor(chosen.model), requiredCapabilities, decisionType, retention, requiredFloor, riskLevel, savings })
|
|
1643
|
+
});
|
|
1644
|
+
}
|
|
1645
|
+
return entries;
|
|
1646
|
+
}
|