lynkr 9.14.2 → 9.14.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/bin/cli.js +14 -6
- package/bin/lynkr-desktop-token.js +108 -0
- package/bin/lynkr-init.js +13 -1
- package/bin/lynkr-restart.js +125 -0
- package/bin/opencode-setup.js +175 -0
- package/bin/run.js +92 -0
- package/config/model-capabilities.json +20 -0
- package/config/model-capability-seeds.json +75 -0
- package/config/model-tiers.json +13 -1
- package/package.json +10 -4
- package/scripts/claude-desktop.js +212 -0
- package/scripts/ope-report.js +78 -0
- package/scripts/seed-capabilities.js +230 -0
- package/src/api/claude-desktop-gateway.js +123 -0
- package/src/api/mcp-broker.js +149 -0
- package/src/api/middleware/budget-enforcer.js +77 -7
- package/src/api/middleware/budget.js +36 -2
- package/src/api/openai-router.js +125 -4
- package/src/api/providers-handler.js +16 -0
- package/src/api/router.js +67 -103
- package/src/auth-mode.js +40 -4
- package/src/budget/index.js +86 -0
- package/src/cache/embeddings.js +125 -35
- package/src/cache/onnx-embedder.js +92 -0
- package/src/clients/databricks.js +285 -21
- package/src/clients/gpt-utils.js +83 -7
- package/src/clients/health-probe.js +157 -0
- package/src/clients/provider-capabilities.js +27 -0
- package/src/config/index.js +13 -1
- package/src/context/compression.js +127 -19
- package/src/dashboard/api.js +1 -0
- package/src/observability/metrics.js +52 -0
- package/src/observability/otel.js +180 -0
- package/src/orchestrator/azure-responses-sse.js +223 -0
- package/src/orchestrator/index.js +219 -136
- package/src/orchestrator/passthrough-stream.js +7 -1
- package/src/orchestrator/sse-transformer.js +62 -3
- package/src/routing/bandit.js +39 -0
- package/src/routing/capabilities.js +88 -0
- package/src/routing/capability-seeds/benchmarks-dir.js +58 -0
- package/src/routing/capability-seeds/family-heuristics.js +77 -0
- package/src/routing/capability-seeds/family.js +105 -0
- package/src/routing/capability-seeds/models-dev.js +64 -0
- package/src/routing/capability-seeds/normalize.js +160 -0
- package/src/routing/capability-seeds/registry.js +125 -0
- package/src/routing/capability-seeds/sources.js +69 -0
- package/src/routing/capability-seeds/swebench.js +66 -0
- package/src/routing/decide.js +149 -0
- package/src/routing/deescalator.js +37 -0
- package/src/routing/index.js +202 -86
- package/src/routing/model-registry.js +25 -0
- package/src/routing/model-slots.js +65 -0
- package/src/routing/model-tiers.js +2 -0
- package/src/routing/ope.js +255 -0
- package/src/routing/openai-model-slots.js +82 -0
- package/src/routing/shortfall.js +290 -0
- package/src/routing/stuck-detector.js +127 -0
- package/src/routing/telemetry.js +11 -2
- package/src/server.js +41 -3
- package/src/tools/web-search-exec.js +182 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capability requirement vector (item 1: HyDRA port, phase 1).
|
|
3
|
+
*
|
|
4
|
+
* Maps the existing 15-dim weighted analysis (complexity-analyzer.js
|
|
5
|
+
* calculateWeightedScore dimensions, each 0-100) onto 4 independent
|
|
6
|
+
* capability heads in [0,1]:
|
|
7
|
+
*
|
|
8
|
+
* - reasoning : multi-step / planning / tradeoff analysis load
|
|
9
|
+
* - codegen : code-writing demand (generation + technical + tool depth)
|
|
10
|
+
* - debugging : diagnostic load (analysis + technical + domain breadth)
|
|
11
|
+
* - tool_use : tool-orchestration load (count, complexity, chaining, history)
|
|
12
|
+
*
|
|
13
|
+
* Deliberately decoupled from the model catalog: this module never names a
|
|
14
|
+
* provider/model/tier. Model capabilities live in
|
|
15
|
+
* config/model-capabilities.json and matching lives in shortfall.js, so a
|
|
16
|
+
* catalog change is a config edit with zero retraining.
|
|
17
|
+
*
|
|
18
|
+
* Deliberately excludes risk/agentic overrides: risk-high forces REASONING
|
|
19
|
+
* and AUTONOMOUS sets a REASONING floor upstream (routing/index.js). Those
|
|
20
|
+
* stay as deterministic post-passes (like HyDRA's health veto), not as
|
|
21
|
+
* learned heads — keeping this predictor language- and catalog-invariant.
|
|
22
|
+
* The agentic flag only floors tool_use (agentic work always needs tools),
|
|
23
|
+
* it never lowers a requirement.
|
|
24
|
+
*
|
|
25
|
+
* Pure function, no I/O. Never throws: bad input yields the neutral prior
|
|
26
|
+
* (all 0.2 ≈ trivial) so callers fail open to cheap routing.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
const HEADS = ['reasoning', 'codegen', 'debugging', 'tool_use'];
|
|
30
|
+
|
|
31
|
+
function _clamp01(v) {
|
|
32
|
+
const n = Number(v);
|
|
33
|
+
if (!Number.isFinite(n)) return 0;
|
|
34
|
+
return Math.max(0, Math.min(1, n));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function _dim01(dimensions, name) {
|
|
38
|
+
const v = Number(dimensions?.[name]);
|
|
39
|
+
if (!Number.isFinite(v)) return 0;
|
|
40
|
+
return Math.max(0, Math.min(1, v / 100));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* @param {object} args
|
|
45
|
+
* @param {object} [args.dimensions] — calculateWeightedScore dimensions (0-100 each)
|
|
46
|
+
* @param {object} [args.agenticResult] — { isAgentic } (floors tool_use only)
|
|
47
|
+
* @returns {{ reasoning:number, codegen:number, debugging:number, tool_use:number }}
|
|
48
|
+
*/
|
|
49
|
+
function buildRequirementVector({ dimensions = {}, agenticResult = null } = {}) {
|
|
50
|
+
try {
|
|
51
|
+
const d = (name) => _dim01(dimensions, name);
|
|
52
|
+
|
|
53
|
+
let reasoning = d('multiStepReasoning') * 0.4
|
|
54
|
+
+ d('analysisDepth') * 0.4
|
|
55
|
+
+ d('promptComplexity') * 0.2;
|
|
56
|
+
|
|
57
|
+
let codegen = d('codeGeneration') * 0.5
|
|
58
|
+
+ d('technicalDepth') * 0.3
|
|
59
|
+
+ d('toolComplexity') * 0.2;
|
|
60
|
+
|
|
61
|
+
let debugging = d('analysisDepth') * 0.3
|
|
62
|
+
+ d('technicalDepth') * 0.2
|
|
63
|
+
+ d('domainSpecificity') * 0.3
|
|
64
|
+
+ d('promptComplexity') * 0.2;
|
|
65
|
+
|
|
66
|
+
let toolUse = d('toolCount') * 0.3
|
|
67
|
+
+ d('toolComplexity') * 0.3
|
|
68
|
+
+ d('toolChainPotential') * 0.2
|
|
69
|
+
+ d('priorToolUsage') * 0.2;
|
|
70
|
+
|
|
71
|
+
// Agentic work always needs tool orchestration — floor only, never lower.
|
|
72
|
+
if (agenticResult?.isAgentic) {
|
|
73
|
+
toolUse = Math.max(toolUse, 0.6);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const round3 = (v) => Math.round(_clamp01(v) * 1000) / 1000;
|
|
77
|
+
return {
|
|
78
|
+
reasoning: round3(reasoning),
|
|
79
|
+
codegen: round3(codegen),
|
|
80
|
+
debugging: round3(debugging),
|
|
81
|
+
tool_use: round3(toolUse),
|
|
82
|
+
};
|
|
83
|
+
} catch {
|
|
84
|
+
return { reasoning: 0.2, codegen: 0.2, debugging: 0.2, tool_use: 0.2 };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
module.exports = { HEADS, buildRequirementVector };
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drop-in benchmark directory loader: data/capability-benchmarks/*.json.
|
|
3
|
+
*
|
|
4
|
+
* For leaderboards with no stable raw endpoint (TerminalBench, LiveCodeBench
|
|
5
|
+
* snapshots, Artificial Analysis exports): download the export once, drop it
|
|
6
|
+
* here, and the seed script picks it up. Format per file:
|
|
7
|
+
*
|
|
8
|
+
* { "source": "terminalbench-2.1", "scale": "percent",
|
|
9
|
+
* "results": { "Kimi K2.5": 42.1, "GPT 5.2": 38.0 } }
|
|
10
|
+
*
|
|
11
|
+
* scale "percent" (0-100) or "fraction" (0-1). Model names are cleaned with
|
|
12
|
+
* the SWE-bench cleaner and family-mapped in normalize.js; unmapped names
|
|
13
|
+
* surface for review instead of being silently seeded. Directory absent or
|
|
14
|
+
* empty → { files: [] } (not an error).
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const { cleanName } = require('./swebench');
|
|
20
|
+
|
|
21
|
+
const BENCHMARKS_DIR = path.join(__dirname, '../../../data/capability-benchmarks');
|
|
22
|
+
|
|
23
|
+
function loadBenchmarksDir(dir = BENCHMARKS_DIR) {
|
|
24
|
+
try {
|
|
25
|
+
if (!fs.existsSync(dir)) return { files: [] };
|
|
26
|
+
const files = [];
|
|
27
|
+
for (const name of fs.readdirSync(dir)) {
|
|
28
|
+
if (!name.endsWith('.json')) continue;
|
|
29
|
+
try {
|
|
30
|
+
const raw = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8'));
|
|
31
|
+
if (!raw || typeof raw !== 'object' || !raw.results || typeof raw.results !== 'object') {
|
|
32
|
+
files.push({ file: name, status: 'skipped', reason: 'missing results object' });
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
const entries = [];
|
|
36
|
+
for (const [modelName, score] of Object.entries(raw.results)) {
|
|
37
|
+
const cleaned = cleanName(modelName);
|
|
38
|
+
const s = Number(score);
|
|
39
|
+
if (cleaned && Number.isFinite(s)) entries.push({ name: cleaned, score: s });
|
|
40
|
+
}
|
|
41
|
+
files.push({
|
|
42
|
+
file: name,
|
|
43
|
+
status: 'ok',
|
|
44
|
+
source: String(raw.source || name.replace(/\.json$/, '')),
|
|
45
|
+
scale: raw.scale === 'fraction' ? 'fraction' : 'percent',
|
|
46
|
+
entries,
|
|
47
|
+
});
|
|
48
|
+
} catch (err) {
|
|
49
|
+
files.push({ file: name, status: 'skipped', reason: err.message });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return { files };
|
|
53
|
+
} catch (err) {
|
|
54
|
+
return { files: [], error: err.message };
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
module.exports = { loadBenchmarksDir, BENCHMARKS_DIR };
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Family-ladder heuristics: zero-network capability estimates from model
|
|
3
|
+
* family names. Vendors ship in ladders (haiku<sonnet<opus, mini<full,
|
|
4
|
+
* flash<pro, air<max, 7b<32b<70b), so an unknown model usually reveals its
|
|
5
|
+
* rung by name. Deliberately pessimistic: heuristics never claim frontier
|
|
6
|
+
* (cap 0.85) — exact seeds do that. Returns null when nothing matches, so
|
|
7
|
+
* the resolver falls through to tier-slot caps (today's behavior).
|
|
8
|
+
*
|
|
9
|
+
* Tilts: coder-named models get +codegen, reasoner-named +reasoning.
|
|
10
|
+
* Everything rounded to 3 decimals. Pure, no I/O, never throws.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const CAP = 0.85;
|
|
14
|
+
const FLOOR = 0.15;
|
|
15
|
+
|
|
16
|
+
// Ordered: first match wins. Frontier names before size rules (a bare
|
|
17
|
+
// "deepseek-r1" carries no size suffix but is strong).
|
|
18
|
+
const LADDER = [
|
|
19
|
+
// Frontier (0.80)
|
|
20
|
+
{ re: /^(gpt-5(\.|$)|o1(-pro)?$|o3($|-)|claude-opus-4|gemini-2\.5-pro|deepseek-r1$|deepseek-reasoner|glm-5\.|kimi-k3($|-)|grok-4|llama-3\.1-405b|qwen3-max|qwen3-235b|doubao-seed-1\.6|minimax-m2($|-)|gpt-5\.2|gpt-5\.3|gpt-5\.4)/, base: 0.8 },
|
|
21
|
+
// Strong (0.65)
|
|
22
|
+
{ re: /(sonnet|gpt-4o$|gpt-4-turbo|gpt-5-mini|gpt-5-nano$|gemini-2\.[05].*pro|qwen3-3[02]b|qwen2\.5-3[23]b|deepseek-v3|glm-4\.7|glm-4\.6|glm-4\.5($|-)|kimi-k2($|-)|mistral-large|codestral|gpt-oss-120b|llama-3\.3-70b|qwen3-30b-a3b|devstral-large|kimi-k2\.5|gemini-3-pro|claude-4\.5)/, base: 0.65 },
|
|
23
|
+
// Mid (0.50)
|
|
24
|
+
{ re: /(gpt-4o-mini|haiku|o4-mini|qwen3-(4b|8b|14b)|qwen2\.5-(7b|14b)|gemma-3-27b|phi-4|mistral-small|ministral|glm-4-flash|glm-4\.5-air|-lite$|-air$|devstral-small|qwen3-4b|gpt-5\.1)/, base: 0.5 },
|
|
25
|
+
// Small (0.30)
|
|
26
|
+
{ re: /(-7b|-8b|-9b|^.*[^0-9](7|8|9)b$|mini|nano|gemma-?2|phi-3|llama-3\.2|flash-lite)/, base: 0.3 },
|
|
27
|
+
// Tiny (0.20)
|
|
28
|
+
{ re: /(-1b|-2b|-3b|-0\.5b|-1\.5b|tiny$|gpt-3\.5)/, base: 0.2 },
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
const CODER_RE = /(coder|codestral|starcoder|qwen3-coder|devstral)/;
|
|
32
|
+
const REASONER_RE = /(reason|r1$|qwq|\bo1\b|\bo3\b|thinking)/;
|
|
33
|
+
|
|
34
|
+
function _sizeBase(family) {
|
|
35
|
+
const m = String(family || '').match(/(\d+(?:\.\d+)?)\s*b(?:$|-)/);
|
|
36
|
+
if (!m) return null;
|
|
37
|
+
const gb = Number(m[1]);
|
|
38
|
+
if (!Number.isFinite(gb)) return null;
|
|
39
|
+
if (gb >= 100) return 0.8;
|
|
40
|
+
if (gb >= 30) return 0.65;
|
|
41
|
+
if (gb >= 10) return 0.5;
|
|
42
|
+
if (gb >= 4) return 0.35;
|
|
43
|
+
return 0.25;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* @param {string} family — normalized family id (see family.js)
|
|
48
|
+
* @returns {null | { reasoning:number, codegen:number, debugging:number, tool_use:number }}
|
|
49
|
+
*/
|
|
50
|
+
function heuristicCaps(family) {
|
|
51
|
+
try {
|
|
52
|
+
const f = String(family || '').toLowerCase().trim();
|
|
53
|
+
if (!f || f === 'unknown') return null;
|
|
54
|
+
let base = null;
|
|
55
|
+
for (const { re, base: b } of LADDER) {
|
|
56
|
+
if (re.test(f)) {
|
|
57
|
+
base = b;
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (base === null) base = _sizeBase(f);
|
|
62
|
+
if (base === null) return null;
|
|
63
|
+
const round3 = (v) => Math.max(FLOOR, Math.min(CAP, Math.round(v * 1000) / 1000));
|
|
64
|
+
const codegen = base + (CODER_RE.test(f) ? 0.05 : 0);
|
|
65
|
+
const reasoning = base + (REASONER_RE.test(f) ? 0.05 : 0);
|
|
66
|
+
return {
|
|
67
|
+
reasoning: round3(reasoning),
|
|
68
|
+
codegen: round3(codegen),
|
|
69
|
+
debugging: round3(base),
|
|
70
|
+
tool_use: round3(base),
|
|
71
|
+
};
|
|
72
|
+
} catch {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
module.exports = { heuristicCaps };
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model-family normalization for capability seeding.
|
|
3
|
+
*
|
|
4
|
+
* Capability is a property of the MODEL (weights), not the provider serving
|
|
5
|
+
* it: zai:glm-5.2, baidu:glm-5.2 and ollama:glm-5.2 must resolve to one
|
|
6
|
+
* family id ("glm-5.2") and therefore identical caps. Provider stays on the
|
|
7
|
+
* cost/invocation path only.
|
|
8
|
+
*
|
|
9
|
+
* Normalization (order matters):
|
|
10
|
+
* 1. lowercase + trim, drop @digest suffixes
|
|
11
|
+
* 2. last `/`-segment wins (strips org prefixes: zai-org/glm-5.2,
|
|
12
|
+
* openai/gpt-4o-mini, anthropic/claude-...)
|
|
13
|
+
* 3. Ollama `:tag` becomes `-tag` (tags often carry SIZE: qwen2.5-coder:7b
|
|
14
|
+
* → qwen2.5-coder-7b; :latest/:cloud are noise but harmless post-strip)
|
|
15
|
+
* 4. strip serving prefixes (databricks-, anthropic., bedrock/)
|
|
16
|
+
* 5. `_` → `-`, digit-dash-digit → digit.dot.digit
|
|
17
|
+
* (gpt-3-5-turbo → gpt-3.5-turbo, llama-3-1-70b → llama-3.1-70b),
|
|
18
|
+
* collapse repeats
|
|
19
|
+
*
|
|
20
|
+
* Quantization is detected, not normalized away: Q4_K_M / gguf / int4 / awq
|
|
21
|
+
* style markers set quant:true so the resolver can apply the small
|
|
22
|
+
* same-brain-smaller-body haircut. Full-precision self-hosted hits the same
|
|
23
|
+
* caps as the API.
|
|
24
|
+
*
|
|
25
|
+
* Pure functions, no I/O. Never throws.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
const SERVING_PREFIXES = ['databricks-', 'anthropic.', 'bedrock/'];
|
|
29
|
+
|
|
30
|
+
// Quant markers: matched against the full "model:tag" string BEFORE tag
|
|
31
|
+
// folding, and against the folded family after (tags like Q4_K_M survive as
|
|
32
|
+
// -q4-k-m, so one pass after folding suffices — but pre-tag names like
|
|
33
|
+
// "model-Q4" exist too, hence match post-fold only, on the whole string).
|
|
34
|
+
const QUANT_RE = /(q[23468](_|-|$)|_k_[sm]|[-_]q8_0|gguf|int[48]|fp[148]|awq|gptq|bnb|mlx|quantized)/i;
|
|
35
|
+
|
|
36
|
+
// Tags that carry no signal even as suffixes.
|
|
37
|
+
const NOISE_TAGS = new Set(['latest', 'cloud', 'instruct']);
|
|
38
|
+
|
|
39
|
+
function detectQuant(s) {
|
|
40
|
+
try {
|
|
41
|
+
return QUANT_RE.test(String(s || ''));
|
|
42
|
+
} catch {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @param {string} provider
|
|
49
|
+
* @param {string} model
|
|
50
|
+
* @returns {{ family:string, quant:boolean }}
|
|
51
|
+
*/
|
|
52
|
+
function normalizeFamily(provider, model) {
|
|
53
|
+
try {
|
|
54
|
+
let s = `${String(model || '').trim()}`;
|
|
55
|
+
// @sha256:... digests
|
|
56
|
+
s = s.split('@')[0];
|
|
57
|
+
// org prefix: last segment wins
|
|
58
|
+
if (s.includes('/')) s = s.split('/').pop();
|
|
59
|
+
const quant = detectQuant(`${provider || ''}/${model || ''}`) || detectQuant(s);
|
|
60
|
+
// Ollama :tag → -tag (size tags preserved: :7b → -7b)
|
|
61
|
+
if (s.includes(':')) {
|
|
62
|
+
const [base, ...tags] = s.split(':');
|
|
63
|
+
const kept = tags
|
|
64
|
+
.map((t) => t.trim().toLowerCase())
|
|
65
|
+
.filter((t) => t && !NOISE_TAGS.has(t));
|
|
66
|
+
s = kept.length > 0 ? `${base}-${kept.join('-')}` : base;
|
|
67
|
+
}
|
|
68
|
+
s = s.toLowerCase().trim();
|
|
69
|
+
for (const p of SERVING_PREFIXES) {
|
|
70
|
+
if (s.startsWith(p)) {
|
|
71
|
+
s = s.slice(p.length);
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
s = s.replace(/_/g, '-');
|
|
76
|
+
// digit-dash-digit → digit.dot.digit, but only for version runs: the
|
|
77
|
+
// second digit must be followed by another dash or end-of-string, so
|
|
78
|
+
// size suffixes survive (llama-3-1-70b → llama-3.1-70b, but
|
|
79
|
+
// qwen3-32b and gemma-3-27b keep their dash).
|
|
80
|
+
s = s.replace(/(\d)-(?=\d(?:-|$))/g, '$1.');
|
|
81
|
+
s = s.replace(/-+/g, '-').replace(/^-|-$/g, '');
|
|
82
|
+
return { family: s || 'unknown', quant };
|
|
83
|
+
} catch {
|
|
84
|
+
return { family: 'unknown', quant: false };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Quant haircut: same brain, smaller body. Applied by the resolver to
|
|
90
|
+
* seed/family/tier caps — never to explicit operator overrides (an exact
|
|
91
|
+
* provider:model override wins verbatim).
|
|
92
|
+
*/
|
|
93
|
+
const QUANT_HAIRCUT = 0.02;
|
|
94
|
+
const QUANT_FLOOR = 0.1;
|
|
95
|
+
|
|
96
|
+
function applyQuantHaircut(caps) {
|
|
97
|
+
const out = {};
|
|
98
|
+
for (const [k, v] of Object.entries(caps || {})) {
|
|
99
|
+
const n = Number(v);
|
|
100
|
+
out[k] = Number.isFinite(n) ? Math.max(QUANT_FLOOR, Math.round((n - QUANT_HAIRCUT) * 1000) / 1000) : v;
|
|
101
|
+
}
|
|
102
|
+
return out;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
module.exports = { normalizeFamily, detectQuant, applyQuantHaircut, QUANT_HAIRCUT };
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* models.dev adapter: boolean capability flags per model (tool_call,
|
|
3
|
+
* vision/image input, reasoning, context window). Same public endpoint the
|
|
4
|
+
* pricing registry already consumes (https://models.dev/api.json), same
|
|
5
|
+
* shape ({ providerId: { models: { modelId: {...} } } }).
|
|
6
|
+
*
|
|
7
|
+
* Flags don't set caps — they GATE them: a model with tool_call=false gets
|
|
8
|
+
* tool_use capped (it can still drive tools via XML extraction, which is why
|
|
9
|
+
* this is a cap, not a zero). Everything else passes through as provenance
|
|
10
|
+
* for operator review.
|
|
11
|
+
*
|
|
12
|
+
* Fail-soft: { status:'skipped', reason } on any problem.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const { fetchJson, getCached, setCached, skipped } = require('./sources');
|
|
16
|
+
const { normalizeFamily } = require('./family');
|
|
17
|
+
|
|
18
|
+
const MODELS_DEV_URL = 'https://models.dev/api.json';
|
|
19
|
+
|
|
20
|
+
// Conservative ceiling for models without function-calling. XML-extracted
|
|
21
|
+
// tool calls (see xml-tool-extractor.js) keep them usable, hence 0.4 not 0.
|
|
22
|
+
const NO_TOOLCALL_TOOL_USE_CAP = 0.4;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @param {object} [opts] — { refresh:boolean }
|
|
26
|
+
* @returns {Promise<{ status, flags?:Object<family,{toolCall,vision,reasoning,context}>, reason? }>}
|
|
27
|
+
*/
|
|
28
|
+
async function fetchModelsDevFlags({ refresh = true } = {}) {
|
|
29
|
+
const parse = (data) => {
|
|
30
|
+
const out = {};
|
|
31
|
+
for (const [providerId, providerData] of Object.entries(data || {})) {
|
|
32
|
+
if (!providerData?.models) continue;
|
|
33
|
+
for (const [modelId, info] of Object.entries(providerData.models)) {
|
|
34
|
+
if (!info || typeof info !== 'object') continue;
|
|
35
|
+
const { family } = normalizeFamily(providerId, modelId);
|
|
36
|
+
if (family === 'unknown' || out[family]) continue; // first provider wins
|
|
37
|
+
out[family] = {
|
|
38
|
+
toolCall: info.tool_call ?? null,
|
|
39
|
+
vision: Array.isArray(info.input) ? info.input.includes('image') : null,
|
|
40
|
+
reasoning: info.reasoning ?? null,
|
|
41
|
+
context: Number(info.context) || null,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
if (!refresh) {
|
|
49
|
+
const cached = getCached('modelsdev');
|
|
50
|
+
if (cached) return { status: 'ok', flags: cached, cached: true };
|
|
51
|
+
return skipped('no cache (run with --refresh)');
|
|
52
|
+
}
|
|
53
|
+
try {
|
|
54
|
+
const flags = parse(await fetchJson(MODELS_DEV_URL));
|
|
55
|
+
setCached('modelsdev', flags);
|
|
56
|
+
return { status: 'ok', flags };
|
|
57
|
+
} catch (err) {
|
|
58
|
+
const cached = getCached('modelsdev');
|
|
59
|
+
if (cached) return { status: 'ok', flags: cached, cached: true, stale: err.message };
|
|
60
|
+
return skipped(`fetch failed and no cache: ${err.message}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
module.exports = { fetchModelsDevFlags, MODELS_DEV_URL, NO_TOOLCALL_TOOL_USE_CAP };
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Score → capability-cap normalization + leaderboard-name → family mapping.
|
|
3
|
+
*
|
|
4
|
+
* Anchors (fixed permanently — τ absorbs residual error, consistency matters
|
|
5
|
+
* more than precision): benchmark fraction 0.8 (≈ flagship SWE-Verified
|
|
6
|
+
* territory) → caps 0.9; fraction 0 → 0.15 (never zero: even weak models do
|
|
7
|
+
* trivial turns). Linear between, clamped to [0.15, 0.9].
|
|
8
|
+
*
|
|
9
|
+
* Per-head source weights (v1 heuristic, documented):
|
|
10
|
+
* debugging = 0.6·swe + 0.4·terminal
|
|
11
|
+
* codegen = 0.5·livecode + 0.3·swe + 0.2·arena
|
|
12
|
+
* reasoning = 0.5·swe + 0.3·arena + 0.2·livecode
|
|
13
|
+
* tool_use = 0.6·terminal + 0.4·swe
|
|
14
|
+
* Missing sources renormalize over the present ones. A pessimistic haircut
|
|
15
|
+
* (-0.03) applies to all normalized caps: underestimating a new model costs
|
|
16
|
+
* money (extra escalations), overestimating costs quality.
|
|
17
|
+
*
|
|
18
|
+
* Family mapping: order-insensitive token overlap between the normalized
|
|
19
|
+
* family id and the leaderboard name. Every family token must hit an entry
|
|
20
|
+
* token exactly or as a version prefix ("4" matches "4.5", "k2" matches
|
|
21
|
+
* "k2.5" — but "32b" never matches "480b"). Best overlap wins; weak matches
|
|
22
|
+
* land in `unmapped` for operator review instead of being seeded silently.
|
|
23
|
+
* Pure functions, never throw.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
const { normalizeFamily } = require('./family');
|
|
27
|
+
|
|
28
|
+
const CAP_FLOOR = 0.15;
|
|
29
|
+
const CAP_CEIL = 0.9;
|
|
30
|
+
const NORMALIZE_HAIRCUT = 0.03;
|
|
31
|
+
|
|
32
|
+
// score fraction (0-1) at which caps hit the ceiling — flagship territory.
|
|
33
|
+
const ANCHOR_SCORE = 0.8;
|
|
34
|
+
|
|
35
|
+
const HEAD_WEIGHTS = {
|
|
36
|
+
debugging: { swe: 0.6, terminal: 0.4, livecode: 0, arena: 0 },
|
|
37
|
+
codegen: { swe: 0.3, terminal: 0, livecode: 0.5, arena: 0.2 },
|
|
38
|
+
reasoning: { swe: 0.5, terminal: 0, livecode: 0.2, arena: 0.3 },
|
|
39
|
+
tool_use: { swe: 0.4, terminal: 0.6, livecode: 0, arena: 0 },
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const STOPWORDS = new Set([
|
|
43
|
+
'the', 'model', 'preview', 'high', 'medium', 'low',
|
|
44
|
+
'agent', 'instruct', 'thinking',
|
|
45
|
+
'2024', '2025', '2026',
|
|
46
|
+
]);
|
|
47
|
+
|
|
48
|
+
function entryTokens(name) {
|
|
49
|
+
return String(name || '')
|
|
50
|
+
.toLowerCase()
|
|
51
|
+
.split(/[^a-z0-9.]+/)
|
|
52
|
+
.map((t) => t.replace(/^\.+|\.+$/g, ''))
|
|
53
|
+
.filter((t) => t && !STOPWORDS.has(t));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function _tokenHit(familyTok, entryToks) {
|
|
57
|
+
for (const tok of entryToks) {
|
|
58
|
+
if (tok === familyTok) return true;
|
|
59
|
+
// version prefix: family "4" hits entry "4.5"; family "k2" hits "k2.5".
|
|
60
|
+
// The char after the prefix must be a dot (never a letter/digit, so
|
|
61
|
+
// "32b" can't match "480b" and "gpt-5" can't match "gpt-50").
|
|
62
|
+
if (tok.length > familyTok.length && tok.startsWith(familyTok) && tok[familyTok.length] === '.') {
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
// reverse: entry "4" satisfies family "4.5"? No — a bare major never
|
|
66
|
+
// proves the minor. Skip.
|
|
67
|
+
}
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function familyTokens(family) {
|
|
72
|
+
return String(family || '')
|
|
73
|
+
.toLowerCase()
|
|
74
|
+
.split(/[^a-z0-9.]+/)
|
|
75
|
+
.map((t) => t.replace(/^\.+|\.+$/g, ''))
|
|
76
|
+
.filter((t) => t && !STOPWORDS.has(t));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* @param {string} entryName — cleaned leaderboard name
|
|
81
|
+
* @param {string[]} knownFamilies — candidate family ids (seed keys w/o wildcards)
|
|
82
|
+
* @returns {{ family:string|null, hits:number, of:number }}
|
|
83
|
+
*/
|
|
84
|
+
function mapEntryToFamily(entryName, knownFamilies) {
|
|
85
|
+
try {
|
|
86
|
+
const toks = entryTokens(entryName);
|
|
87
|
+
if (toks.length === 0 || !Array.isArray(knownFamilies)) {
|
|
88
|
+
return { family: null, hits: 0, of: 0 };
|
|
89
|
+
}
|
|
90
|
+
let best = { family: null, hits: 0, of: 0, ratio: 0 };
|
|
91
|
+
for (const fam of knownFamilies) {
|
|
92
|
+
const ftoks = familyTokens(fam);
|
|
93
|
+
if (ftoks.length === 0) continue;
|
|
94
|
+
let hits = 0;
|
|
95
|
+
for (const ft of ftoks) {
|
|
96
|
+
if (_tokenHit(ft, toks)) hits++;
|
|
97
|
+
}
|
|
98
|
+
const ratio = hits / ftoks.length;
|
|
99
|
+
// Full coverage required; ties prefer the more specific family.
|
|
100
|
+
if (ratio === 1 && (best.family === null || ftoks.length > best.of)) {
|
|
101
|
+
best = { family: fam, hits, of: ftoks.length, ratio };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return best.family ? best : { family: null, hits: 0, of: 0 };
|
|
105
|
+
} catch {
|
|
106
|
+
return { family: null, hits: 0, of: 0 };
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function scoreToCap(score) {
|
|
111
|
+
const s = Math.max(0, Math.min(1, Number(score)));
|
|
112
|
+
if (!Number.isFinite(s)) return null;
|
|
113
|
+
const cap = CAP_FLOOR + (s / ANCHOR_SCORE) * (CAP_CEIL - CAP_FLOOR);
|
|
114
|
+
return Math.max(CAP_FLOOR, Math.min(CAP_CEIL, cap));
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* @param {object} scores — { swe?, terminal?, livecode?, arena? } fractions 0-1
|
|
119
|
+
* @returns {null | { reasoning, codegen, debugging, tool_use }}
|
|
120
|
+
*/
|
|
121
|
+
function scoresToCaps(scores) {
|
|
122
|
+
try {
|
|
123
|
+
if (!scores || typeof scores !== 'object') return null;
|
|
124
|
+
const present = {};
|
|
125
|
+
for (const k of ['swe', 'terminal', 'livecode', 'arena']) {
|
|
126
|
+
const v = Number(scores[k]);
|
|
127
|
+
if (Number.isFinite(v)) present[k] = Math.max(0, Math.min(1, v));
|
|
128
|
+
}
|
|
129
|
+
if (Object.keys(present).length === 0) return null;
|
|
130
|
+
const caps = {};
|
|
131
|
+
for (const [head, weights] of Object.entries(HEAD_WEIGHTS)) {
|
|
132
|
+
let total = 0;
|
|
133
|
+
let wsum = 0;
|
|
134
|
+
for (const [src, w] of Object.entries(weights)) {
|
|
135
|
+
if (present[src] !== undefined && w > 0) {
|
|
136
|
+
total += scoreToCap(present[src]) * w;
|
|
137
|
+
wsum += w;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (wsum === 0) return null;
|
|
141
|
+
caps[head] = Math.round((total / wsum - NORMALIZE_HAIRCUT) * 1000) / 1000;
|
|
142
|
+
caps[head] = Math.max(CAP_FLOOR, Math.min(CAP_CEIL, caps[head]));
|
|
143
|
+
}
|
|
144
|
+
return caps;
|
|
145
|
+
} catch {
|
|
146
|
+
return null;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
module.exports = {
|
|
151
|
+
mapEntryToFamily,
|
|
152
|
+
scoresToCaps,
|
|
153
|
+
scoreToCap,
|
|
154
|
+
entryTokens,
|
|
155
|
+
familyTokens,
|
|
156
|
+
normalizeFamily,
|
|
157
|
+
CAP_FLOOR,
|
|
158
|
+
CAP_CEIL,
|
|
159
|
+
NORMALIZE_HAIRCUT,
|
|
160
|
+
};
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Seed registry: per-FAMILY capability profiles from online-derived data.
|
|
3
|
+
*
|
|
4
|
+
* Layers (first hit wins in shortfall.js resolveCapabilities):
|
|
5
|
+
* operator modelOverrides (provider:model, manual — supreme, lives in
|
|
6
|
+
* model-capabilities.json) → snapshot (data/, fetched, `source:'seed:snapshot'`)
|
|
7
|
+
* → shipped (config/, reviewed, `source:'seed:shipped'`) → family
|
|
8
|
+
* heuristic (`source:'family'`) → tier slot (`source:'tier'`).
|
|
9
|
+
*
|
|
10
|
+
* Keys are normalized family ids (see family.js), lowercased, with optional
|
|
11
|
+
* trailing `*` wildcards matched longest-first (gpt-5* , qwen3-*). Provider
|
|
12
|
+
* never appears in keys — zai/baidu/ollama servings of glm-5.2 share one entry.
|
|
13
|
+
*
|
|
14
|
+
* File lifecycles mirror model-tiers.json: read once at boot, restart to
|
|
15
|
+
* pick up edits. data/ is gitignored (operator-local), config/ is versioned.
|
|
16
|
+
* Pure lookups after load; load never throws (missing/malformed → empty).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const fs = require('fs');
|
|
20
|
+
const path = require('path');
|
|
21
|
+
const logger = require('../../logger');
|
|
22
|
+
const { HEADS } = require('../capabilities');
|
|
23
|
+
|
|
24
|
+
const SHIPPED_PATH = path.join(__dirname, '../../../config/model-capability-seeds.json');
|
|
25
|
+
const SNAPSHOT_PATH = path.join(__dirname, '../../../data/capability-seeds.snapshot.json');
|
|
26
|
+
|
|
27
|
+
let _cache = null;
|
|
28
|
+
|
|
29
|
+
function sanitizeCaps(raw) {
|
|
30
|
+
const caps = {};
|
|
31
|
+
for (const h of HEADS) {
|
|
32
|
+
const v = Number(raw?.[h]);
|
|
33
|
+
caps[h] = Number.isFinite(v) ? Math.max(0, Math.min(1, v)) : 0.5;
|
|
34
|
+
}
|
|
35
|
+
return caps;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function _loadSeedsFile(filePath, label) {
|
|
39
|
+
try {
|
|
40
|
+
if (!fs.existsSync(filePath)) return {};
|
|
41
|
+
const raw = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
42
|
+
const seeds = raw?.seeds && typeof raw.seeds === 'object' ? raw.seeds : {};
|
|
43
|
+
const out = {};
|
|
44
|
+
for (const [key, entry] of Object.entries(seeds)) {
|
|
45
|
+
const caps = entry?.caps && typeof entry.caps === 'object' ? entry.caps : entry;
|
|
46
|
+
if (!caps || typeof caps !== 'object') continue;
|
|
47
|
+
out[String(key).toLowerCase()] = {
|
|
48
|
+
caps: sanitizeCaps(caps),
|
|
49
|
+
sources: Array.isArray(entry?.sources) ? entry.sources : [],
|
|
50
|
+
note: typeof entry?.note === 'string' ? entry.note : '',
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
54
|
+
} catch (err) {
|
|
55
|
+
logger.debug({ err: err.message, label }, '[SeedRegistry] seed file load failed — skipping layer');
|
|
56
|
+
return {};
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function loadSeedLayers() {
|
|
61
|
+
if (_cache) return _cache;
|
|
62
|
+
_cache = {
|
|
63
|
+
snapshot: _loadSeedsFile(SNAPSHOT_PATH, 'snapshot'),
|
|
64
|
+
shipped: _loadSeedsFile(SHIPPED_PATH, 'shipped'),
|
|
65
|
+
};
|
|
66
|
+
return _cache;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function _resetSeedsCache() {
|
|
70
|
+
_cache = null;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Test hook: inject layers directly (Operations on the same cached object
|
|
74
|
+
// the resolver reads).
|
|
75
|
+
function _setSeedsForTests({ snapshot, shipped } = {}) {
|
|
76
|
+
const base = loadSeedLayers();
|
|
77
|
+
_cache = {
|
|
78
|
+
snapshot: snapshot ?? base.snapshot,
|
|
79
|
+
shipped: shipped ?? base.shipped,
|
|
80
|
+
};
|
|
81
|
+
return _cache;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function _matchWildcard(layer, family) {
|
|
85
|
+
let best = null;
|
|
86
|
+
for (const [key, entry] of Object.entries(layer)) {
|
|
87
|
+
if (!key.endsWith('*')) continue;
|
|
88
|
+
const prefix = key.slice(0, -1);
|
|
89
|
+
if (prefix && family.startsWith(prefix) && (!best || prefix.length > best.prefix.length)) {
|
|
90
|
+
best = { prefix, entry };
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return best?.entry ?? null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* @param {string} family — normalized family id
|
|
98
|
+
* @returns {null | { caps, source:'seed:snapshot'|'seed:shipped', sources, note }}
|
|
99
|
+
*/
|
|
100
|
+
function resolveSeedCaps(family) {
|
|
101
|
+
try {
|
|
102
|
+
const f = String(family || '').toLowerCase().trim();
|
|
103
|
+
if (!f || f === 'unknown') return null;
|
|
104
|
+
const { snapshot, shipped } = loadSeedLayers();
|
|
105
|
+
if (snapshot[f]) return { ...snapshot[f], source: 'seed:snapshot' };
|
|
106
|
+
if (shipped[f]) return { ...shipped[f], source: 'seed:shipped' };
|
|
107
|
+
const snapWild = _matchWildcard(snapshot, f);
|
|
108
|
+
if (snapWild) return { ...snapWild, source: 'seed:snapshot' };
|
|
109
|
+
const shipWild = _matchWildcard(shipped, f);
|
|
110
|
+
if (shipWild) return { ...shipWild, source: 'seed:shipped' };
|
|
111
|
+
return null;
|
|
112
|
+
} catch {
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
module.exports = {
|
|
118
|
+
loadSeedLayers,
|
|
119
|
+
_resetSeedsCache,
|
|
120
|
+
_setSeedsForTests,
|
|
121
|
+
resolveSeedCaps,
|
|
122
|
+
sanitizeCaps,
|
|
123
|
+
SHIPPED_PATH,
|
|
124
|
+
SNAPSHOT_PATH,
|
|
125
|
+
};
|