lynkr 9.14.1 → 9.14.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/bin/cli.js +14 -6
- package/bin/lynkr-desktop-token.js +108 -0
- package/bin/lynkr-init.js +13 -1
- package/bin/lynkr-restart.js +125 -0
- package/bin/opencode-setup.js +175 -0
- package/bin/run.js +92 -0
- package/config/model-capabilities.json +20 -0
- package/config/model-capability-seeds.json +75 -0
- package/config/model-tiers.json +13 -1
- package/package.json +16 -22
- package/scripts/claude-desktop.js +212 -0
- package/scripts/ope-report.js +78 -0
- package/scripts/seed-capabilities.js +230 -0
- package/src/api/claude-desktop-gateway.js +123 -0
- package/src/api/health.js +0 -1
- package/src/api/mcp-broker.js +149 -0
- package/src/api/middleware/budget-enforcer.js +77 -7
- package/src/api/middleware/budget.js +36 -2
- package/src/api/middleware/session.js +0 -0
- package/src/api/openai-router.js +210 -15
- package/src/api/providers-handler.js +16 -1
- package/src/api/router.js +73 -106
- package/src/auth-mode.js +40 -4
- package/src/budget/index.js +86 -0
- package/src/cache/embeddings.js +125 -35
- package/src/cache/onnx-embedder.js +92 -0
- package/src/cache/prompt.js +0 -7
- package/src/clients/bedrock-utils.js +2 -1
- package/src/clients/codex-process.js +1 -1
- package/src/clients/codex-utils.js +0 -2
- package/src/clients/databricks.js +337 -46
- package/src/clients/gpt-utils.js +84 -8
- package/src/clients/health-probe.js +157 -0
- package/src/clients/openai-format.js +73 -23
- package/src/clients/provider-capabilities.js +27 -2
- package/src/clients/resilience.js +0 -1
- package/src/clients/responses-format.js +0 -56
- package/src/clients/routing.js +1 -1
- package/src/config/index.js +13 -2
- package/src/context/budget.js +1 -1
- package/src/context/compression.js +127 -19
- package/src/context/tool-result-compressor.js +3 -5
- package/src/dashboard/api.js +1 -1
- package/src/logger/oversized-error-stream.js +1 -1
- package/src/memory/extractor.js +3 -3
- package/src/memory/search.js +2 -2
- package/src/memory/store.js +9 -24
- package/src/memory/tools.js +14 -0
- package/src/observability/metrics.js +52 -3
- package/src/observability/otel.js +180 -0
- package/src/orchestrator/azure-responses-sse.js +223 -0
- package/src/orchestrator/index.js +216 -212
- package/src/orchestrator/passthrough-stream.js +7 -1
- package/src/orchestrator/sse-transformer.js +130 -13
- package/src/policy/safe-commands.js +0 -1
- package/src/routing/bandit.js +39 -0
- package/src/routing/calibration.js +21 -1
- package/src/routing/capabilities.js +88 -0
- package/src/routing/capability-seeds/benchmarks-dir.js +58 -0
- package/src/routing/capability-seeds/family-heuristics.js +77 -0
- package/src/routing/capability-seeds/family.js +105 -0
- package/src/routing/capability-seeds/models-dev.js +64 -0
- package/src/routing/capability-seeds/normalize.js +160 -0
- package/src/routing/capability-seeds/registry.js +125 -0
- package/src/routing/capability-seeds/sources.js +69 -0
- package/src/routing/capability-seeds/swebench.js +66 -0
- package/src/routing/client-profiles.js +26 -0
- package/src/routing/complexity-analyzer.js +5 -8
- package/src/routing/confidence-scorer.js +2 -2
- package/src/routing/cost-optimizer.js +1 -2
- package/src/routing/decide.js +149 -0
- package/src/routing/deescalator.js +38 -1
- package/src/routing/embedding-cache.js +0 -1
- package/src/routing/index.js +202 -86
- package/src/routing/intent-score.js +0 -1
- package/src/routing/knn-router.js +27 -5
- package/src/routing/latency-tracker.js +1 -1
- package/src/routing/model-registry.js +25 -0
- package/src/routing/model-slots.js +65 -0
- package/src/routing/model-tiers.js +2 -0
- package/src/routing/ope.js +255 -0
- package/src/routing/openai-model-slots.js +82 -0
- package/src/routing/regret-estimator.js +1 -1
- package/src/routing/shortfall.js +290 -0
- package/src/routing/stuck-detector.js +127 -0
- package/src/routing/telemetry.js +11 -2
- package/src/server.js +41 -3
- package/src/stores/file-store.js +5 -1
- package/src/tools/web-search-exec.js +182 -0
- package/src/training/trajectory-compressor.js +1 -1
- package/src/workers/pool.js +1 -1
- package/src/workers/worker.js +1 -1
- package/scripts/convert-routellm.py +0 -105
- package/src/diff/comments.js +0 -153
- package/src/policy/web-fallback.js +0 -33
|
@@ -28,6 +28,14 @@ const logger = require("../logger");
|
|
|
28
28
|
// stream:false predated this transformer). Caveat: reasoning_content deltas
|
|
29
29
|
// (kimi thinking) are not reshaped — thinking text is dropped from streamed
|
|
30
30
|
// responses; the buffered path still lifts it into thinking blocks.
|
|
31
|
+
// baidu (Qianfan's /v2/chat/completions) already returns the raw stream the
|
|
32
|
+
// same way moonshot does (invokeBaidu's `if (response?.stream) return
|
|
33
|
+
// response;`) and its endpoint is documented as OpenAI-compatible — but
|
|
34
|
+
// unlike moonshot this hasn't been E2E-verified against a live key as of
|
|
35
|
+
// this addition. If Qianfan's SSE deltas turn out not to match
|
|
36
|
+
// choices[0].delta exactly, remove it from this list (or set
|
|
37
|
+
// LYNKR_STREAM_TRANSFORM_PROVIDERS to exclude it) rather than patching the
|
|
38
|
+
// shared transformer for one provider's quirk.
|
|
31
39
|
const DEFAULT_OPENAI_SSE_PROVIDERS = [
|
|
32
40
|
"openai",
|
|
33
41
|
"atlas",
|
|
@@ -37,12 +45,36 @@ const DEFAULT_OPENAI_SSE_PROVIDERS = [
|
|
|
37
45
|
"lmstudio",
|
|
38
46
|
"llamacpp",
|
|
39
47
|
"moonshot",
|
|
48
|
+
"baidu",
|
|
49
|
+
// fireworks (api.fireworks.ai/inference/v1/chat/completions) is documented
|
|
50
|
+
// as OpenAI-compatible SSE — same baidu caveat applies: E2E-unverified
|
|
51
|
+
// against a live key as of this addition. If its deltas turn out not to
|
|
52
|
+
// match choices[0].delta exactly, remove it from this list rather than
|
|
53
|
+
// patching the shared transformer for one provider's quirk.
|
|
54
|
+
"fireworks",
|
|
40
55
|
];
|
|
41
56
|
|
|
57
|
+
// llama.cpp specific: reasoning-capable local builds (live-confirmed on this
|
|
58
|
+
// deployment's GPT-OSS model) emit `delta.reasoning_content` before any
|
|
59
|
+
// `delta.content`. This transformer only reads delta.content (see the
|
|
60
|
+
// per-chunk loop below) — reasoning deltas are silently dropped, so a client
|
|
61
|
+
// streaming through this path sees nothing at all during the thinking phase
|
|
62
|
+
// and, if reasoning consumes the whole max_tokens budget, nothing ever.
|
|
63
|
+
// Default OFF (buffered instead) for exactly the same reason Ollama already
|
|
64
|
+
// defaults to buffering (LYNKR_OLLAMA_BUFFER_RESPONSES) — the buffered path's
|
|
65
|
+
// convertOpenRouterResponseToAnthropic already lifts reasoning_content into a
|
|
66
|
+
// proper `thinking` block, so no data is lost there. Opt into live streaming
|
|
67
|
+
// (and accept dropped thinking text) via LYNKR_LLAMACPP_BUFFER_RESPONSES=false.
|
|
42
68
|
function _transformProviders() {
|
|
43
69
|
const env = process.env.LYNKR_STREAM_TRANSFORM_PROVIDERS;
|
|
70
|
+
if (env) {
|
|
71
|
+
return new Set(env.split(",").map((s) => s.trim()).filter(Boolean));
|
|
72
|
+
}
|
|
73
|
+
const bufferLlamacpp = process.env.LYNKR_LLAMACPP_BUFFER_RESPONSES !== "false";
|
|
44
74
|
return new Set(
|
|
45
|
-
|
|
75
|
+
bufferLlamacpp
|
|
76
|
+
? DEFAULT_OPENAI_SSE_PROVIDERS.filter((p) => p !== "llamacpp")
|
|
77
|
+
: DEFAULT_OPENAI_SSE_PROVIDERS,
|
|
46
78
|
);
|
|
47
79
|
}
|
|
48
80
|
|
|
@@ -158,7 +190,62 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
|
|
|
158
190
|
}
|
|
159
191
|
};
|
|
160
192
|
|
|
193
|
+
// GLM-family reasoning leak filter (streaming twin of the fix in
|
|
194
|
+
// convertOpenAIToAnthropic): suppresses matched <think>…</think> spans and
|
|
195
|
+
// strips a bare orphan closing tag from visible text deltas, holding back
|
|
196
|
+
// trailing partial-tag fragments so tags split across chunk boundaries
|
|
197
|
+
// still match. Orphan pre-text has already been emitted by the time the
|
|
198
|
+
// closer arrives — a stream can't retract — so only the tag is removed.
|
|
199
|
+
let _inThink = false;
|
|
200
|
+
let _tagTail = "";
|
|
201
|
+
const _partialTagLen = (s) => {
|
|
202
|
+
for (let k = Math.min(8, s.length); k > 0; k--) {
|
|
203
|
+
const suf = s.slice(-k);
|
|
204
|
+
if ("</think>".startsWith(suf) || "<think>".startsWith(suf)) return k;
|
|
205
|
+
}
|
|
206
|
+
return 0;
|
|
207
|
+
};
|
|
208
|
+
const filterThink = (text) => {
|
|
209
|
+
let s = _tagTail + text;
|
|
210
|
+
_tagTail = "";
|
|
211
|
+
let out = "";
|
|
212
|
+
for (;;) {
|
|
213
|
+
if (_inThink) {
|
|
214
|
+
const close = s.indexOf("</think>");
|
|
215
|
+
if (close === -1) break; // still inside thinking — discard s (tail kept below)
|
|
216
|
+
s = s.slice(close + 8);
|
|
217
|
+
_inThink = false;
|
|
218
|
+
} else {
|
|
219
|
+
const open = s.indexOf("<think>");
|
|
220
|
+
const close = s.indexOf("</think>");
|
|
221
|
+
if (close !== -1 && (open === -1 || close < open)) {
|
|
222
|
+
out += s.slice(0, close); // orphan closer: drop the tag only
|
|
223
|
+
s = s.slice(close + 8);
|
|
224
|
+
} else if (open !== -1) {
|
|
225
|
+
out += s.slice(0, open);
|
|
226
|
+
s = s.slice(open + 7);
|
|
227
|
+
_inThink = true;
|
|
228
|
+
} else {
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
const keep = _partialTagLen(s);
|
|
234
|
+
if (keep > 0) _tagTail = s.slice(-keep);
|
|
235
|
+
if (!_inThink && s.length > keep) out += s.slice(0, s.length - keep);
|
|
236
|
+
return out;
|
|
237
|
+
};
|
|
238
|
+
|
|
161
239
|
const closeTextBlock = function* () {
|
|
240
|
+
// Flush a held-back fragment that turned out not to be a tag.
|
|
241
|
+
if (_tagTail && !_inThink && textIndex !== null) {
|
|
242
|
+
yield _sse("content_block_delta", {
|
|
243
|
+
type: "content_block_delta",
|
|
244
|
+
index: textIndex,
|
|
245
|
+
delta: { type: "text_delta", text: _tagTail },
|
|
246
|
+
});
|
|
247
|
+
_tagTail = "";
|
|
248
|
+
}
|
|
162
249
|
if (textIndex === null) return;
|
|
163
250
|
yield _sse("content_block_stop", { type: "content_block_stop", index: textIndex });
|
|
164
251
|
textIndex = null;
|
|
@@ -205,6 +292,8 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
|
|
|
205
292
|
if (chunk.usage) {
|
|
206
293
|
if (chunk.usage.prompt_tokens != null) stats.usage.input_tokens = chunk.usage.prompt_tokens;
|
|
207
294
|
if (chunk.usage.completion_tokens != null) stats.usage.output_tokens = chunk.usage.completion_tokens;
|
|
295
|
+
const cached = chunk.usage.prompt_tokens_details?.cached_tokens;
|
|
296
|
+
if (cached != null) stats.usage.cache_read_input_tokens = cached;
|
|
208
297
|
}
|
|
209
298
|
|
|
210
299
|
const choice = chunk.choices?.[0];
|
|
@@ -217,19 +306,22 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
|
|
|
217
306
|
// Text deltas are straightforward: open a block on first text, then
|
|
218
307
|
// emit a text_delta per chunk.
|
|
219
308
|
if (typeof delta.content === "string" && delta.content.length > 0) {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
309
|
+
const visible = filterThink(delta.content);
|
|
310
|
+
if (visible.length > 0) {
|
|
311
|
+
if (textIndex === null) {
|
|
312
|
+
textIndex = nextIndex++;
|
|
313
|
+
yield _sse("content_block_start", {
|
|
314
|
+
type: "content_block_start",
|
|
315
|
+
index: textIndex,
|
|
316
|
+
content_block: { type: "text", text: "" },
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
yield _sse("content_block_delta", {
|
|
320
|
+
type: "content_block_delta",
|
|
224
321
|
index: textIndex,
|
|
225
|
-
|
|
322
|
+
delta: { type: "text_delta", text: visible },
|
|
226
323
|
});
|
|
227
324
|
}
|
|
228
|
-
yield _sse("content_block_delta", {
|
|
229
|
-
type: "content_block_delta",
|
|
230
|
-
index: textIndex,
|
|
231
|
-
delta: { type: "text_delta", text: delta.content },
|
|
232
|
-
});
|
|
233
325
|
}
|
|
234
326
|
|
|
235
327
|
// Tool-call fragments: accumulate per OpenAI tool index. Fragments of
|
|
@@ -276,7 +368,11 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
|
|
|
276
368
|
// argument JSON may be truncated mid-string.
|
|
277
369
|
if (!finishReason) {
|
|
278
370
|
logger.warn(
|
|
279
|
-
{
|
|
371
|
+
{
|
|
372
|
+
droppedToolFragments: toolAcc.size,
|
|
373
|
+
inputTokens: stats.usage.input_tokens,
|
|
374
|
+
outputTokens: stats.usage.output_tokens,
|
|
375
|
+
},
|
|
280
376
|
"[SSETransform] Upstream ended without finish_reason — signaling stream error"
|
|
281
377
|
);
|
|
282
378
|
yield _sse("error", {
|
|
@@ -294,10 +390,26 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
|
|
|
294
390
|
const stopReason = _mapFinishReason(finishReason);
|
|
295
391
|
stats.stopReason = stopReason;
|
|
296
392
|
stats.finishReason = finishReason;
|
|
393
|
+
// Usage ordering problem, solved here: the Anthropic protocol reports
|
|
394
|
+
// input_tokens in message_start, but OpenAI-shaped upstreams (Azure
|
|
395
|
+
// Responses API especially) only deliver usage at END of stream — so our
|
|
396
|
+
// message_start necessarily went out with input_tokens: 0. Emit the real
|
|
397
|
+
// cumulative usage in the final message_delta instead: Anthropic's own
|
|
398
|
+
// newer streams carry input_tokens/cache fields here, and SDK clients
|
|
399
|
+
// (incl. opencode's @ai-sdk/anthropic) merge message_delta usage over
|
|
400
|
+
// message_start. Without this, clients see only output tokens (~tens) and
|
|
401
|
+
// render nonsense like "Context: 64 tokens, 0% used" for a 60k-token
|
|
402
|
+
// prompt — and misprice every turn.
|
|
297
403
|
yield _sse("message_delta", {
|
|
298
404
|
type: "message_delta",
|
|
299
405
|
delta: { stop_reason: stopReason, stop_sequence: null },
|
|
300
|
-
usage: {
|
|
406
|
+
usage: {
|
|
407
|
+
output_tokens: stats.usage.output_tokens ?? 0,
|
|
408
|
+
...(stats.usage.input_tokens != null ? { input_tokens: stats.usage.input_tokens } : {}),
|
|
409
|
+
...(stats.usage.cache_read_input_tokens != null
|
|
410
|
+
? { cache_read_input_tokens: stats.usage.cache_read_input_tokens }
|
|
411
|
+
: {}),
|
|
412
|
+
},
|
|
301
413
|
});
|
|
302
414
|
yield _sse("message_stop", { type: "message_stop" });
|
|
303
415
|
|
|
@@ -439,4 +551,9 @@ module.exports = {
|
|
|
439
551
|
// Exported for unit tests.
|
|
440
552
|
_openaiToAnthropicEvents,
|
|
441
553
|
_anthropicToOpenaiEvents,
|
|
554
|
+
// Exported for reuse by other SSE adapters (e.g. azure-responses-sse.js) —
|
|
555
|
+
// generic, provider-agnostic SSE byte-stream parsing with no OpenAI-specific
|
|
556
|
+
// behavior, worth sharing rather than reimplementing per adapter.
|
|
557
|
+
_iterateStream,
|
|
558
|
+
_sseDataLines,
|
|
442
559
|
};
|
package/src/routing/bandit.js
CHANGED
|
@@ -187,6 +187,45 @@ class LinUCBBandit {
|
|
|
187
187
|
return best;
|
|
188
188
|
}
|
|
189
189
|
|
|
190
|
+
/**
|
|
191
|
+
* Mean reward estimate for one arm in one context — θ·x with NO
|
|
192
|
+
* uncertainty bonus. This is the regression model r̂(x, a) the
|
|
193
|
+
* doubly-robust off-policy estimator consumes (routing/ope.js): the same
|
|
194
|
+
* ridge-regression state the live policy learns from, reused as the
|
|
195
|
+
* DR baseline instead of training a second model.
|
|
196
|
+
*
|
|
197
|
+
* Returns null when the arm has never been observed (identity-prior-only
|
|
198
|
+
* arms predict 0 by construction, which would bias DR toward "never
|
|
199
|
+
* tried" — better to let the estimator fall back to the IPS-only term).
|
|
200
|
+
*
|
|
201
|
+
* @param {string} tier
|
|
202
|
+
* @param {string} provider
|
|
203
|
+
* @param {string} model
|
|
204
|
+
* @param {number[]} context
|
|
205
|
+
* @returns {number|null} estimated reward in [0, 1], or null if unknown arm
|
|
206
|
+
*/
|
|
207
|
+
estimateReward(tier, provider, model, context) {
|
|
208
|
+
const key = this._armKey(tier, provider, model);
|
|
209
|
+
const arm = this.arms.get(key);
|
|
210
|
+
if (!arm || arm.count === 0) return null;
|
|
211
|
+
let ctx = context;
|
|
212
|
+
if (ctx.length !== this.dim) {
|
|
213
|
+
ctx = ctx.slice(0, this.dim);
|
|
214
|
+
while (ctx.length < this.dim) ctx.push(0);
|
|
215
|
+
}
|
|
216
|
+
let Ainv;
|
|
217
|
+
try {
|
|
218
|
+
Ainv = _inv(arm.A);
|
|
219
|
+
} catch {
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
const theta = _matVec(Ainv, arm.b);
|
|
223
|
+
const mean = _dot(theta, ctx);
|
|
224
|
+
// Rewards are trained in [0, 1]; clamp the linear extrapolation so a
|
|
225
|
+
// wild θ·x can't dominate the DR correction term.
|
|
226
|
+
return Math.max(0, Math.min(1, mean));
|
|
227
|
+
}
|
|
228
|
+
|
|
190
229
|
/**
|
|
191
230
|
* Update the chosen arm with the observed reward.
|
|
192
231
|
* @param {string} tier
|
|
@@ -19,6 +19,8 @@ const path = require('path');
|
|
|
19
19
|
|
|
20
20
|
const DEFAULT_DAYS = 7;
|
|
21
21
|
const MIN_SAMPLES = 100;
|
|
22
|
+
/** Narrower than this and a calibrated band is treated as collapse, not signal. */
|
|
23
|
+
const MIN_TIER_WIDTH = 10;
|
|
22
24
|
|
|
23
25
|
/** Quality score below which a complexity bucket is "underperforming" for its tier. */
|
|
24
26
|
const QUALITY_FLOOR = {
|
|
@@ -188,7 +190,24 @@ function runCalibration({ days = DEFAULT_DAYS, dryRun = false, dbPath, outputPat
|
|
|
188
190
|
const prev = ranges[tierOrder[i - 1]];
|
|
189
191
|
const cur = ranges[tierOrder[i]];
|
|
190
192
|
if (cur[0] !== prev[1] + 1) cur[0] = prev[1] + 1;
|
|
191
|
-
if (cur[0] > cur[1]) cur[1] = cur[0]; // collapsed;
|
|
193
|
+
if (cur[0] > cur[1]) cur[1] = cur[0]; // collapsed; see guardrail below
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Guardrail: a tier squeezed under MIN_TIER_WIDTH means calibration had
|
|
197
|
+
// degenerate signal, not a real finding — e.g. 2026-07-26 (MEDIUM →
|
|
198
|
+
// [20,20]) and 2026-08-22 (COMPLEX → [51,54], which re-stitched
|
|
199
|
+
// REASONING's floor down to 55 and sent ~all agentic traffic to the most
|
|
200
|
+
// expensive tier; skewed telemetry then re-taught the same bands every
|
|
201
|
+
// 24h). Collapsed bands revert the whole set to DEFAULT_RANGES — bands
|
|
202
|
+
// are interdependent after stitching, so partial reverts would leave
|
|
203
|
+
// gaps.
|
|
204
|
+
const collapsed = tierOrder.filter(
|
|
205
|
+
(t) => ranges[t][1] - ranges[t][0] < MIN_TIER_WIDTH
|
|
206
|
+
);
|
|
207
|
+
let guardrail = null;
|
|
208
|
+
if (collapsed.length > 0) {
|
|
209
|
+
guardrail = { reverted: true, reason: 'band_collapse', collapsed };
|
|
210
|
+
for (const t of tierOrder) ranges[t] = [...DEFAULT_RANGES[t]];
|
|
192
211
|
}
|
|
193
212
|
|
|
194
213
|
const out = {
|
|
@@ -197,6 +216,7 @@ function runCalibration({ days = DEFAULT_DAYS, dryRun = false, dbPath, outputPat
|
|
|
197
216
|
sampleCount: rows.length,
|
|
198
217
|
ranges,
|
|
199
218
|
stats,
|
|
219
|
+
...(guardrail ? { guardrail } : {}),
|
|
200
220
|
};
|
|
201
221
|
|
|
202
222
|
if (dryRun) return { ...out, dryRun: true };
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capability requirement vector (item 1: HyDRA port, phase 1).
|
|
3
|
+
*
|
|
4
|
+
* Maps the existing 15-dim weighted analysis (complexity-analyzer.js
|
|
5
|
+
* calculateWeightedScore dimensions, each 0-100) onto 4 independent
|
|
6
|
+
* capability heads in [0,1]:
|
|
7
|
+
*
|
|
8
|
+
* - reasoning : multi-step / planning / tradeoff analysis load
|
|
9
|
+
* - codegen : code-writing demand (generation + technical + tool depth)
|
|
10
|
+
* - debugging : diagnostic load (analysis + technical + domain breadth)
|
|
11
|
+
* - tool_use : tool-orchestration load (count, complexity, chaining, history)
|
|
12
|
+
*
|
|
13
|
+
* Deliberately decoupled from the model catalog: this module never names a
|
|
14
|
+
* provider/model/tier. Model capabilities live in
|
|
15
|
+
* config/model-capabilities.json and matching lives in shortfall.js, so a
|
|
16
|
+
* catalog change is a config edit with zero retraining.
|
|
17
|
+
*
|
|
18
|
+
* Deliberately excludes risk/agentic overrides: risk-high forces REASONING
|
|
19
|
+
* and AUTONOMOUS sets a REASONING floor upstream (routing/index.js). Those
|
|
20
|
+
* stay as deterministic post-passes (like HyDRA's health veto), not as
|
|
21
|
+
* learned heads — keeping this predictor language- and catalog-invariant.
|
|
22
|
+
* The agentic flag only floors tool_use (agentic work always needs tools),
|
|
23
|
+
* it never lowers a requirement.
|
|
24
|
+
*
|
|
25
|
+
* Pure function, no I/O. Never throws: bad input yields the neutral prior
|
|
26
|
+
* (all 0.2 ≈ trivial) so callers fail open to cheap routing.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
const HEADS = ['reasoning', 'codegen', 'debugging', 'tool_use'];
|
|
30
|
+
|
|
31
|
+
function _clamp01(v) {
|
|
32
|
+
const n = Number(v);
|
|
33
|
+
if (!Number.isFinite(n)) return 0;
|
|
34
|
+
return Math.max(0, Math.min(1, n));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function _dim01(dimensions, name) {
|
|
38
|
+
const v = Number(dimensions?.[name]);
|
|
39
|
+
if (!Number.isFinite(v)) return 0;
|
|
40
|
+
return Math.max(0, Math.min(1, v / 100));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* @param {object} args
|
|
45
|
+
* @param {object} [args.dimensions] — calculateWeightedScore dimensions (0-100 each)
|
|
46
|
+
* @param {object} [args.agenticResult] — { isAgentic } (floors tool_use only)
|
|
47
|
+
* @returns {{ reasoning:number, codegen:number, debugging:number, tool_use:number }}
|
|
48
|
+
*/
|
|
49
|
+
function buildRequirementVector({ dimensions = {}, agenticResult = null } = {}) {
|
|
50
|
+
try {
|
|
51
|
+
const d = (name) => _dim01(dimensions, name);
|
|
52
|
+
|
|
53
|
+
let reasoning = d('multiStepReasoning') * 0.4
|
|
54
|
+
+ d('analysisDepth') * 0.4
|
|
55
|
+
+ d('promptComplexity') * 0.2;
|
|
56
|
+
|
|
57
|
+
let codegen = d('codeGeneration') * 0.5
|
|
58
|
+
+ d('technicalDepth') * 0.3
|
|
59
|
+
+ d('toolComplexity') * 0.2;
|
|
60
|
+
|
|
61
|
+
let debugging = d('analysisDepth') * 0.3
|
|
62
|
+
+ d('technicalDepth') * 0.2
|
|
63
|
+
+ d('domainSpecificity') * 0.3
|
|
64
|
+
+ d('promptComplexity') * 0.2;
|
|
65
|
+
|
|
66
|
+
let toolUse = d('toolCount') * 0.3
|
|
67
|
+
+ d('toolComplexity') * 0.3
|
|
68
|
+
+ d('toolChainPotential') * 0.2
|
|
69
|
+
+ d('priorToolUsage') * 0.2;
|
|
70
|
+
|
|
71
|
+
// Agentic work always needs tool orchestration — floor only, never lower.
|
|
72
|
+
if (agenticResult?.isAgentic) {
|
|
73
|
+
toolUse = Math.max(toolUse, 0.6);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const round3 = (v) => Math.round(_clamp01(v) * 1000) / 1000;
|
|
77
|
+
return {
|
|
78
|
+
reasoning: round3(reasoning),
|
|
79
|
+
codegen: round3(codegen),
|
|
80
|
+
debugging: round3(debugging),
|
|
81
|
+
tool_use: round3(toolUse),
|
|
82
|
+
};
|
|
83
|
+
} catch {
|
|
84
|
+
return { reasoning: 0.2, codegen: 0.2, debugging: 0.2, tool_use: 0.2 };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
module.exports = { HEADS, buildRequirementVector };
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drop-in benchmark directory loader: data/capability-benchmarks/*.json.
|
|
3
|
+
*
|
|
4
|
+
* For leaderboards with no stable raw endpoint (TerminalBench, LiveCodeBench
|
|
5
|
+
* snapshots, Artificial Analysis exports): download the export once, drop it
|
|
6
|
+
* here, and the seed script picks it up. Format per file:
|
|
7
|
+
*
|
|
8
|
+
* { "source": "terminalbench-2.1", "scale": "percent",
|
|
9
|
+
* "results": { "Kimi K2.5": 42.1, "GPT 5.2": 38.0 } }
|
|
10
|
+
*
|
|
11
|
+
* scale "percent" (0-100) or "fraction" (0-1). Model names are cleaned with
|
|
12
|
+
* the SWE-bench cleaner and family-mapped in normalize.js; unmapped names
|
|
13
|
+
* surface for review instead of being silently seeded. Directory absent or
|
|
14
|
+
* empty → { files: [] } (not an error).
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const { cleanName } = require('./swebench');
|
|
20
|
+
|
|
21
|
+
const BENCHMARKS_DIR = path.join(__dirname, '../../../data/capability-benchmarks');
|
|
22
|
+
|
|
23
|
+
function loadBenchmarksDir(dir = BENCHMARKS_DIR) {
|
|
24
|
+
try {
|
|
25
|
+
if (!fs.existsSync(dir)) return { files: [] };
|
|
26
|
+
const files = [];
|
|
27
|
+
for (const name of fs.readdirSync(dir)) {
|
|
28
|
+
if (!name.endsWith('.json')) continue;
|
|
29
|
+
try {
|
|
30
|
+
const raw = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8'));
|
|
31
|
+
if (!raw || typeof raw !== 'object' || !raw.results || typeof raw.results !== 'object') {
|
|
32
|
+
files.push({ file: name, status: 'skipped', reason: 'missing results object' });
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
const entries = [];
|
|
36
|
+
for (const [modelName, score] of Object.entries(raw.results)) {
|
|
37
|
+
const cleaned = cleanName(modelName);
|
|
38
|
+
const s = Number(score);
|
|
39
|
+
if (cleaned && Number.isFinite(s)) entries.push({ name: cleaned, score: s });
|
|
40
|
+
}
|
|
41
|
+
files.push({
|
|
42
|
+
file: name,
|
|
43
|
+
status: 'ok',
|
|
44
|
+
source: String(raw.source || name.replace(/\.json$/, '')),
|
|
45
|
+
scale: raw.scale === 'fraction' ? 'fraction' : 'percent',
|
|
46
|
+
entries,
|
|
47
|
+
});
|
|
48
|
+
} catch (err) {
|
|
49
|
+
files.push({ file: name, status: 'skipped', reason: err.message });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return { files };
|
|
53
|
+
} catch (err) {
|
|
54
|
+
return { files: [], error: err.message };
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
module.exports = { loadBenchmarksDir, BENCHMARKS_DIR };
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Family-ladder heuristics: zero-network capability estimates from model
|
|
3
|
+
* family names. Vendors ship in ladders (haiku<sonnet<opus, mini<full,
|
|
4
|
+
* flash<pro, air<max, 7b<32b<70b), so an unknown model usually reveals its
|
|
5
|
+
* rung by name. Deliberately pessimistic: heuristics never claim frontier
|
|
6
|
+
* (cap 0.85) — exact seeds do that. Returns null when nothing matches, so
|
|
7
|
+
* the resolver falls through to tier-slot caps (today's behavior).
|
|
8
|
+
*
|
|
9
|
+
* Tilts: coder-named models get +codegen, reasoner-named +reasoning.
|
|
10
|
+
* Everything rounded to 3 decimals. Pure, no I/O, never throws.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const CAP = 0.85;
|
|
14
|
+
const FLOOR = 0.15;
|
|
15
|
+
|
|
16
|
+
// Ordered: first match wins. Frontier names before size rules (a bare
|
|
17
|
+
// "deepseek-r1" carries no size suffix but is strong).
|
|
18
|
+
const LADDER = [
|
|
19
|
+
// Frontier (0.80)
|
|
20
|
+
{ re: /^(gpt-5(\.|$)|o1(-pro)?$|o3($|-)|claude-opus-4|gemini-2\.5-pro|deepseek-r1$|deepseek-reasoner|glm-5\.|kimi-k3($|-)|grok-4|llama-3\.1-405b|qwen3-max|qwen3-235b|doubao-seed-1\.6|minimax-m2($|-)|gpt-5\.2|gpt-5\.3|gpt-5\.4)/, base: 0.8 },
|
|
21
|
+
// Strong (0.65)
|
|
22
|
+
{ re: /(sonnet|gpt-4o$|gpt-4-turbo|gpt-5-mini|gpt-5-nano$|gemini-2\.[05].*pro|qwen3-3[02]b|qwen2\.5-3[23]b|deepseek-v3|glm-4\.7|glm-4\.6|glm-4\.5($|-)|kimi-k2($|-)|mistral-large|codestral|gpt-oss-120b|llama-3\.3-70b|qwen3-30b-a3b|devstral-large|kimi-k2\.5|gemini-3-pro|claude-4\.5)/, base: 0.65 },
|
|
23
|
+
// Mid (0.50)
|
|
24
|
+
{ re: /(gpt-4o-mini|haiku|o4-mini|qwen3-(4b|8b|14b)|qwen2\.5-(7b|14b)|gemma-3-27b|phi-4|mistral-small|ministral|glm-4-flash|glm-4\.5-air|-lite$|-air$|devstral-small|qwen3-4b|gpt-5\.1)/, base: 0.5 },
|
|
25
|
+
// Small (0.30)
|
|
26
|
+
{ re: /(-7b|-8b|-9b|^.*[^0-9](7|8|9)b$|mini|nano|gemma-?2|phi-3|llama-3\.2|flash-lite)/, base: 0.3 },
|
|
27
|
+
// Tiny (0.20)
|
|
28
|
+
{ re: /(-1b|-2b|-3b|-0\.5b|-1\.5b|tiny$|gpt-3\.5)/, base: 0.2 },
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
const CODER_RE = /(coder|codestral|starcoder|qwen3-coder|devstral)/;
|
|
32
|
+
const REASONER_RE = /(reason|r1$|qwq|\bo1\b|\bo3\b|thinking)/;
|
|
33
|
+
|
|
34
|
+
function _sizeBase(family) {
|
|
35
|
+
const m = String(family || '').match(/(\d+(?:\.\d+)?)\s*b(?:$|-)/);
|
|
36
|
+
if (!m) return null;
|
|
37
|
+
const gb = Number(m[1]);
|
|
38
|
+
if (!Number.isFinite(gb)) return null;
|
|
39
|
+
if (gb >= 100) return 0.8;
|
|
40
|
+
if (gb >= 30) return 0.65;
|
|
41
|
+
if (gb >= 10) return 0.5;
|
|
42
|
+
if (gb >= 4) return 0.35;
|
|
43
|
+
return 0.25;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* @param {string} family — normalized family id (see family.js)
|
|
48
|
+
* @returns {null | { reasoning:number, codegen:number, debugging:number, tool_use:number }}
|
|
49
|
+
*/
|
|
50
|
+
function heuristicCaps(family) {
|
|
51
|
+
try {
|
|
52
|
+
const f = String(family || '').toLowerCase().trim();
|
|
53
|
+
if (!f || f === 'unknown') return null;
|
|
54
|
+
let base = null;
|
|
55
|
+
for (const { re, base: b } of LADDER) {
|
|
56
|
+
if (re.test(f)) {
|
|
57
|
+
base = b;
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (base === null) base = _sizeBase(f);
|
|
62
|
+
if (base === null) return null;
|
|
63
|
+
const round3 = (v) => Math.max(FLOOR, Math.min(CAP, Math.round(v * 1000) / 1000));
|
|
64
|
+
const codegen = base + (CODER_RE.test(f) ? 0.05 : 0);
|
|
65
|
+
const reasoning = base + (REASONER_RE.test(f) ? 0.05 : 0);
|
|
66
|
+
return {
|
|
67
|
+
reasoning: round3(reasoning),
|
|
68
|
+
codegen: round3(codegen),
|
|
69
|
+
debugging: round3(base),
|
|
70
|
+
tool_use: round3(base),
|
|
71
|
+
};
|
|
72
|
+
} catch {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
module.exports = { heuristicCaps };
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model-family normalization for capability seeding.
|
|
3
|
+
*
|
|
4
|
+
* Capability is a property of the MODEL (weights), not the provider serving
|
|
5
|
+
* it: zai:glm-5.2, baidu:glm-5.2 and ollama:glm-5.2 must resolve to one
|
|
6
|
+
* family id ("glm-5.2") and therefore identical caps. Provider stays on the
|
|
7
|
+
* cost/invocation path only.
|
|
8
|
+
*
|
|
9
|
+
* Normalization (order matters):
|
|
10
|
+
* 1. lowercase + trim, drop @digest suffixes
|
|
11
|
+
* 2. last `/`-segment wins (strips org prefixes: zai-org/glm-5.2,
|
|
12
|
+
* openai/gpt-4o-mini, anthropic/claude-...)
|
|
13
|
+
* 3. Ollama `:tag` becomes `-tag` (tags often carry SIZE: qwen2.5-coder:7b
|
|
14
|
+
* → qwen2.5-coder-7b; :latest/:cloud are noise but harmless post-strip)
|
|
15
|
+
* 4. strip serving prefixes (databricks-, anthropic., bedrock/)
|
|
16
|
+
* 5. `_` → `-`, digit-dash-digit → digit.dot.digit
|
|
17
|
+
* (gpt-3-5-turbo → gpt-3.5-turbo, llama-3-1-70b → llama-3.1-70b),
|
|
18
|
+
* collapse repeats
|
|
19
|
+
*
|
|
20
|
+
* Quantization is detected, not normalized away: Q4_K_M / gguf / int4 / awq
|
|
21
|
+
* style markers set quant:true so the resolver can apply the small
|
|
22
|
+
* same-brain-smaller-body haircut. Full-precision self-hosted hits the same
|
|
23
|
+
* caps as the API.
|
|
24
|
+
*
|
|
25
|
+
* Pure functions, no I/O. Never throws.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
const SERVING_PREFIXES = ['databricks-', 'anthropic.', 'bedrock/'];
|
|
29
|
+
|
|
30
|
+
// Quant markers: matched against the full "model:tag" string BEFORE tag
|
|
31
|
+
// folding, and against the folded family after (tags like Q4_K_M survive as
|
|
32
|
+
// -q4-k-m, so one pass after folding suffices — but pre-tag names like
|
|
33
|
+
// "model-Q4" exist too, hence match post-fold only, on the whole string).
|
|
34
|
+
const QUANT_RE = /(q[23468](_|-|$)|_k_[sm]|[-_]q8_0|gguf|int[48]|fp[148]|awq|gptq|bnb|mlx|quantized)/i;
|
|
35
|
+
|
|
36
|
+
// Tags that carry no signal even as suffixes.
|
|
37
|
+
const NOISE_TAGS = new Set(['latest', 'cloud', 'instruct']);
|
|
38
|
+
|
|
39
|
+
function detectQuant(s) {
|
|
40
|
+
try {
|
|
41
|
+
return QUANT_RE.test(String(s || ''));
|
|
42
|
+
} catch {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @param {string} provider
|
|
49
|
+
* @param {string} model
|
|
50
|
+
* @returns {{ family:string, quant:boolean }}
|
|
51
|
+
*/
|
|
52
|
+
function normalizeFamily(provider, model) {
|
|
53
|
+
try {
|
|
54
|
+
let s = `${String(model || '').trim()}`;
|
|
55
|
+
// @sha256:... digests
|
|
56
|
+
s = s.split('@')[0];
|
|
57
|
+
// org prefix: last segment wins
|
|
58
|
+
if (s.includes('/')) s = s.split('/').pop();
|
|
59
|
+
const quant = detectQuant(`${provider || ''}/${model || ''}`) || detectQuant(s);
|
|
60
|
+
// Ollama :tag → -tag (size tags preserved: :7b → -7b)
|
|
61
|
+
if (s.includes(':')) {
|
|
62
|
+
const [base, ...tags] = s.split(':');
|
|
63
|
+
const kept = tags
|
|
64
|
+
.map((t) => t.trim().toLowerCase())
|
|
65
|
+
.filter((t) => t && !NOISE_TAGS.has(t));
|
|
66
|
+
s = kept.length > 0 ? `${base}-${kept.join('-')}` : base;
|
|
67
|
+
}
|
|
68
|
+
s = s.toLowerCase().trim();
|
|
69
|
+
for (const p of SERVING_PREFIXES) {
|
|
70
|
+
if (s.startsWith(p)) {
|
|
71
|
+
s = s.slice(p.length);
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
s = s.replace(/_/g, '-');
|
|
76
|
+
// digit-dash-digit → digit.dot.digit, but only for version runs: the
|
|
77
|
+
// second digit must be followed by another dash or end-of-string, so
|
|
78
|
+
// size suffixes survive (llama-3-1-70b → llama-3.1-70b, but
|
|
79
|
+
// qwen3-32b and gemma-3-27b keep their dash).
|
|
80
|
+
s = s.replace(/(\d)-(?=\d(?:-|$))/g, '$1.');
|
|
81
|
+
s = s.replace(/-+/g, '-').replace(/^-|-$/g, '');
|
|
82
|
+
return { family: s || 'unknown', quant };
|
|
83
|
+
} catch {
|
|
84
|
+
return { family: 'unknown', quant: false };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Quant haircut: same brain, smaller body. Applied by the resolver to
|
|
90
|
+
* seed/family/tier caps — never to explicit operator overrides (an exact
|
|
91
|
+
* provider:model override wins verbatim).
|
|
92
|
+
*/
|
|
93
|
+
const QUANT_HAIRCUT = 0.02;
|
|
94
|
+
const QUANT_FLOOR = 0.1;
|
|
95
|
+
|
|
96
|
+
function applyQuantHaircut(caps) {
|
|
97
|
+
const out = {};
|
|
98
|
+
for (const [k, v] of Object.entries(caps || {})) {
|
|
99
|
+
const n = Number(v);
|
|
100
|
+
out[k] = Number.isFinite(n) ? Math.max(QUANT_FLOOR, Math.round((n - QUANT_HAIRCUT) * 1000) / 1000) : v;
|
|
101
|
+
}
|
|
102
|
+
return out;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
module.exports = { normalizeFamily, detectQuant, applyQuantHaircut, QUANT_HAIRCUT };
|