lynkr 9.14.1 → 9.14.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +5 -1
  2. package/bin/cli.js +14 -6
  3. package/bin/lynkr-desktop-token.js +108 -0
  4. package/bin/lynkr-init.js +13 -1
  5. package/bin/lynkr-restart.js +125 -0
  6. package/bin/opencode-setup.js +175 -0
  7. package/bin/run.js +92 -0
  8. package/config/model-capabilities.json +20 -0
  9. package/config/model-capability-seeds.json +75 -0
  10. package/config/model-tiers.json +13 -1
  11. package/package.json +16 -22
  12. package/scripts/claude-desktop.js +212 -0
  13. package/scripts/ope-report.js +78 -0
  14. package/scripts/seed-capabilities.js +230 -0
  15. package/src/api/claude-desktop-gateway.js +123 -0
  16. package/src/api/health.js +0 -1
  17. package/src/api/mcp-broker.js +149 -0
  18. package/src/api/middleware/budget-enforcer.js +77 -7
  19. package/src/api/middleware/budget.js +36 -2
  20. package/src/api/middleware/session.js +0 -0
  21. package/src/api/openai-router.js +210 -15
  22. package/src/api/providers-handler.js +16 -1
  23. package/src/api/router.js +73 -106
  24. package/src/auth-mode.js +40 -4
  25. package/src/budget/index.js +86 -0
  26. package/src/cache/embeddings.js +125 -35
  27. package/src/cache/onnx-embedder.js +92 -0
  28. package/src/cache/prompt.js +0 -7
  29. package/src/clients/bedrock-utils.js +2 -1
  30. package/src/clients/codex-process.js +1 -1
  31. package/src/clients/codex-utils.js +0 -2
  32. package/src/clients/databricks.js +337 -46
  33. package/src/clients/gpt-utils.js +84 -8
  34. package/src/clients/health-probe.js +157 -0
  35. package/src/clients/openai-format.js +73 -23
  36. package/src/clients/provider-capabilities.js +27 -2
  37. package/src/clients/resilience.js +0 -1
  38. package/src/clients/responses-format.js +0 -56
  39. package/src/clients/routing.js +1 -1
  40. package/src/config/index.js +13 -2
  41. package/src/context/budget.js +1 -1
  42. package/src/context/compression.js +127 -19
  43. package/src/context/tool-result-compressor.js +3 -5
  44. package/src/dashboard/api.js +1 -1
  45. package/src/logger/oversized-error-stream.js +1 -1
  46. package/src/memory/extractor.js +3 -3
  47. package/src/memory/search.js +2 -2
  48. package/src/memory/store.js +9 -24
  49. package/src/memory/tools.js +14 -0
  50. package/src/observability/metrics.js +52 -3
  51. package/src/observability/otel.js +180 -0
  52. package/src/orchestrator/azure-responses-sse.js +223 -0
  53. package/src/orchestrator/index.js +216 -212
  54. package/src/orchestrator/passthrough-stream.js +7 -1
  55. package/src/orchestrator/sse-transformer.js +130 -13
  56. package/src/policy/safe-commands.js +0 -1
  57. package/src/routing/bandit.js +39 -0
  58. package/src/routing/calibration.js +21 -1
  59. package/src/routing/capabilities.js +88 -0
  60. package/src/routing/capability-seeds/benchmarks-dir.js +58 -0
  61. package/src/routing/capability-seeds/family-heuristics.js +77 -0
  62. package/src/routing/capability-seeds/family.js +105 -0
  63. package/src/routing/capability-seeds/models-dev.js +64 -0
  64. package/src/routing/capability-seeds/normalize.js +160 -0
  65. package/src/routing/capability-seeds/registry.js +125 -0
  66. package/src/routing/capability-seeds/sources.js +69 -0
  67. package/src/routing/capability-seeds/swebench.js +66 -0
  68. package/src/routing/client-profiles.js +26 -0
  69. package/src/routing/complexity-analyzer.js +5 -8
  70. package/src/routing/confidence-scorer.js +2 -2
  71. package/src/routing/cost-optimizer.js +1 -2
  72. package/src/routing/decide.js +149 -0
  73. package/src/routing/deescalator.js +38 -1
  74. package/src/routing/embedding-cache.js +0 -1
  75. package/src/routing/index.js +202 -86
  76. package/src/routing/intent-score.js +0 -1
  77. package/src/routing/knn-router.js +27 -5
  78. package/src/routing/latency-tracker.js +1 -1
  79. package/src/routing/model-registry.js +25 -0
  80. package/src/routing/model-slots.js +65 -0
  81. package/src/routing/model-tiers.js +2 -0
  82. package/src/routing/ope.js +255 -0
  83. package/src/routing/openai-model-slots.js +82 -0
  84. package/src/routing/regret-estimator.js +1 -1
  85. package/src/routing/shortfall.js +290 -0
  86. package/src/routing/stuck-detector.js +127 -0
  87. package/src/routing/telemetry.js +11 -2
  88. package/src/server.js +41 -3
  89. package/src/stores/file-store.js +5 -1
  90. package/src/tools/web-search-exec.js +182 -0
  91. package/src/training/trajectory-compressor.js +1 -1
  92. package/src/workers/pool.js +1 -1
  93. package/src/workers/worker.js +1 -1
  94. package/scripts/convert-routellm.py +0 -105
  95. package/src/diff/comments.js +0 -153
  96. package/src/policy/web-fallback.js +0 -33
@@ -28,6 +28,14 @@ const logger = require("../logger");
28
28
  // stream:false predated this transformer). Caveat: reasoning_content deltas
29
29
  // (kimi thinking) are not reshaped — thinking text is dropped from streamed
30
30
  // responses; the buffered path still lifts it into thinking blocks.
31
+ // baidu (Qianfan's /v2/chat/completions) already returns the raw stream the
32
+ // same way moonshot does (invokeBaidu's `if (response?.stream) return
33
+ // response;`) and its endpoint is documented as OpenAI-compatible — but
34
+ // unlike moonshot this hasn't been E2E-verified against a live key as of
35
+ // this addition. If Qianfan's SSE deltas turn out not to match
36
+ // choices[0].delta exactly, remove it from this list (or set
37
+ // LYNKR_STREAM_TRANSFORM_PROVIDERS to exclude it) rather than patching the
38
+ // shared transformer for one provider's quirk.
31
39
  const DEFAULT_OPENAI_SSE_PROVIDERS = [
32
40
  "openai",
33
41
  "atlas",
@@ -37,12 +45,36 @@ const DEFAULT_OPENAI_SSE_PROVIDERS = [
37
45
  "lmstudio",
38
46
  "llamacpp",
39
47
  "moonshot",
48
+ "baidu",
49
+ // fireworks (api.fireworks.ai/inference/v1/chat/completions) is documented
50
+ // as OpenAI-compatible SSE — same baidu caveat applies: E2E-unverified
51
+ // against a live key as of this addition. If its deltas turn out not to
52
+ // match choices[0].delta exactly, remove it from this list rather than
53
+ // patching the shared transformer for one provider's quirk.
54
+ "fireworks",
40
55
  ];
41
56
 
57
+ // llama.cpp specific: reasoning-capable local builds (live-confirmed on this
58
+ // deployment's GPT-OSS model) emit `delta.reasoning_content` before any
59
+ // `delta.content`. This transformer only reads delta.content (see the
60
+ // per-chunk loop below) — reasoning deltas are silently dropped, so a client
61
+ // streaming through this path sees nothing at all during the thinking phase
62
+ // and, if reasoning consumes the whole max_tokens budget, nothing ever.
63
+ // Default OFF (buffered instead) for exactly the same reason Ollama already
64
+ // defaults to buffering (LYNKR_OLLAMA_BUFFER_RESPONSES) — the buffered path's
65
+ // convertOpenRouterResponseToAnthropic already lifts reasoning_content into a
66
+ // proper `thinking` block, so no data is lost there. Opt into live streaming
67
+ // (and accept dropped thinking text) via LYNKR_LLAMACPP_BUFFER_RESPONSES=false.
42
68
  function _transformProviders() {
43
69
  const env = process.env.LYNKR_STREAM_TRANSFORM_PROVIDERS;
70
+ if (env) {
71
+ return new Set(env.split(",").map((s) => s.trim()).filter(Boolean));
72
+ }
73
+ const bufferLlamacpp = process.env.LYNKR_LLAMACPP_BUFFER_RESPONSES !== "false";
44
74
  return new Set(
45
- env ? env.split(",").map((s) => s.trim()).filter(Boolean) : DEFAULT_OPENAI_SSE_PROVIDERS,
75
+ bufferLlamacpp
76
+ ? DEFAULT_OPENAI_SSE_PROVIDERS.filter((p) => p !== "llamacpp")
77
+ : DEFAULT_OPENAI_SSE_PROVIDERS,
46
78
  );
47
79
  }
48
80
 
@@ -158,7 +190,62 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
158
190
  }
159
191
  };
160
192
 
193
+ // GLM-family reasoning leak filter (streaming twin of the fix in
194
+ // convertOpenAIToAnthropic): suppresses matched <think>…</think> spans and
195
+ // strips a bare orphan closing tag from visible text deltas, holding back
196
+ // trailing partial-tag fragments so tags split across chunk boundaries
197
+ // still match. Orphan pre-text has already been emitted by the time the
198
+ // closer arrives — a stream can't retract — so only the tag is removed.
199
+ let _inThink = false;
200
+ let _tagTail = "";
201
+ const _partialTagLen = (s) => {
202
+ for (let k = Math.min(8, s.length); k > 0; k--) {
203
+ const suf = s.slice(-k);
204
+ if ("</think>".startsWith(suf) || "<think>".startsWith(suf)) return k;
205
+ }
206
+ return 0;
207
+ };
208
+ const filterThink = (text) => {
209
+ let s = _tagTail + text;
210
+ _tagTail = "";
211
+ let out = "";
212
+ for (;;) {
213
+ if (_inThink) {
214
+ const close = s.indexOf("</think>");
215
+ if (close === -1) break; // still inside thinking — discard s (tail kept below)
216
+ s = s.slice(close + 8);
217
+ _inThink = false;
218
+ } else {
219
+ const open = s.indexOf("<think>");
220
+ const close = s.indexOf("</think>");
221
+ if (close !== -1 && (open === -1 || close < open)) {
222
+ out += s.slice(0, close); // orphan closer: drop the tag only
223
+ s = s.slice(close + 8);
224
+ } else if (open !== -1) {
225
+ out += s.slice(0, open);
226
+ s = s.slice(open + 7);
227
+ _inThink = true;
228
+ } else {
229
+ break;
230
+ }
231
+ }
232
+ }
233
+ const keep = _partialTagLen(s);
234
+ if (keep > 0) _tagTail = s.slice(-keep);
235
+ if (!_inThink && s.length > keep) out += s.slice(0, s.length - keep);
236
+ return out;
237
+ };
238
+
161
239
  const closeTextBlock = function* () {
240
+ // Flush a held-back fragment that turned out not to be a tag.
241
+ if (_tagTail && !_inThink && textIndex !== null) {
242
+ yield _sse("content_block_delta", {
243
+ type: "content_block_delta",
244
+ index: textIndex,
245
+ delta: { type: "text_delta", text: _tagTail },
246
+ });
247
+ _tagTail = "";
248
+ }
162
249
  if (textIndex === null) return;
163
250
  yield _sse("content_block_stop", { type: "content_block_stop", index: textIndex });
164
251
  textIndex = null;
@@ -205,6 +292,8 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
205
292
  if (chunk.usage) {
206
293
  if (chunk.usage.prompt_tokens != null) stats.usage.input_tokens = chunk.usage.prompt_tokens;
207
294
  if (chunk.usage.completion_tokens != null) stats.usage.output_tokens = chunk.usage.completion_tokens;
295
+ const cached = chunk.usage.prompt_tokens_details?.cached_tokens;
296
+ if (cached != null) stats.usage.cache_read_input_tokens = cached;
208
297
  }
209
298
 
210
299
  const choice = chunk.choices?.[0];
@@ -217,19 +306,22 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
217
306
  // Text deltas are straightforward: open a block on first text, then
218
307
  // emit a text_delta per chunk.
219
308
  if (typeof delta.content === "string" && delta.content.length > 0) {
220
- if (textIndex === null) {
221
- textIndex = nextIndex++;
222
- yield _sse("content_block_start", {
223
- type: "content_block_start",
309
+ const visible = filterThink(delta.content);
310
+ if (visible.length > 0) {
311
+ if (textIndex === null) {
312
+ textIndex = nextIndex++;
313
+ yield _sse("content_block_start", {
314
+ type: "content_block_start",
315
+ index: textIndex,
316
+ content_block: { type: "text", text: "" },
317
+ });
318
+ }
319
+ yield _sse("content_block_delta", {
320
+ type: "content_block_delta",
224
321
  index: textIndex,
225
- content_block: { type: "text", text: "" },
322
+ delta: { type: "text_delta", text: visible },
226
323
  });
227
324
  }
228
- yield _sse("content_block_delta", {
229
- type: "content_block_delta",
230
- index: textIndex,
231
- delta: { type: "text_delta", text: delta.content },
232
- });
233
325
  }
234
326
 
235
327
  // Tool-call fragments: accumulate per OpenAI tool index. Fragments of
@@ -276,7 +368,11 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
276
368
  // argument JSON may be truncated mid-string.
277
369
  if (!finishReason) {
278
370
  logger.warn(
279
- { droppedToolFragments: toolAcc.size, outputTokens: stats.usage.output_tokens },
371
+ {
372
+ droppedToolFragments: toolAcc.size,
373
+ inputTokens: stats.usage.input_tokens,
374
+ outputTokens: stats.usage.output_tokens,
375
+ },
280
376
  "[SSETransform] Upstream ended without finish_reason — signaling stream error"
281
377
  );
282
378
  yield _sse("error", {
@@ -294,10 +390,26 @@ async function* _openaiToAnthropicEvents(upstream, opts = {}) {
294
390
  const stopReason = _mapFinishReason(finishReason);
295
391
  stats.stopReason = stopReason;
296
392
  stats.finishReason = finishReason;
393
+ // Usage ordering problem, solved here: the Anthropic protocol reports
394
+ // input_tokens in message_start, but OpenAI-shaped upstreams (Azure
395
+ // Responses API especially) only deliver usage at END of stream — so our
396
+ // message_start necessarily went out with input_tokens: 0. Emit the real
397
+ // cumulative usage in the final message_delta instead: Anthropic's own
398
+ // newer streams carry input_tokens/cache fields here, and SDK clients
399
+ // (incl. opencode's @ai-sdk/anthropic) merge message_delta usage over
400
+ // message_start. Without this, clients see only output tokens (~tens) and
401
+ // render nonsense like "Context: 64 tokens, 0% used" for a 60k-token
402
+ // prompt — and misprice every turn.
297
403
  yield _sse("message_delta", {
298
404
  type: "message_delta",
299
405
  delta: { stop_reason: stopReason, stop_sequence: null },
300
- usage: { output_tokens: stats.usage.output_tokens ?? 0 },
406
+ usage: {
407
+ output_tokens: stats.usage.output_tokens ?? 0,
408
+ ...(stats.usage.input_tokens != null ? { input_tokens: stats.usage.input_tokens } : {}),
409
+ ...(stats.usage.cache_read_input_tokens != null
410
+ ? { cache_read_input_tokens: stats.usage.cache_read_input_tokens }
411
+ : {}),
412
+ },
301
413
  });
302
414
  yield _sse("message_stop", { type: "message_stop" });
303
415
 
@@ -439,4 +551,9 @@ module.exports = {
439
551
  // Exported for unit tests.
440
552
  _openaiToAnthropicEvents,
441
553
  _anthropicToOpenaiEvents,
554
+ // Exported for reuse by other SSE adapters (e.g. azure-responses-sse.js) —
555
+ // generic, provider-agnostic SSE byte-stream parsing with no OpenAI-specific
556
+ // behavior, worth sharing rather than reimplementing per adapter.
557
+ _iterateStream,
558
+ _sseDataLines,
442
559
  };
@@ -1,4 +1,3 @@
1
- const logger = require("../logger");
2
1
  const config = require("../config");
3
2
 
4
3
  /**
@@ -187,6 +187,45 @@ class LinUCBBandit {
187
187
  return best;
188
188
  }
189
189
 
190
+ /**
191
+ * Mean reward estimate for one arm in one context — θ·x with NO
192
+ * uncertainty bonus. This is the regression model r̂(x, a) the
193
+ * doubly-robust off-policy estimator consumes (routing/ope.js): the same
194
+ * ridge-regression state the live policy learns from, reused as the
195
+ * DR baseline instead of training a second model.
196
+ *
197
+ * Returns null when the arm has never been observed (identity-prior-only
198
+ * arms predict 0 by construction, which would bias DR toward "never
199
+ * tried" — better to let the estimator fall back to the IPS-only term).
200
+ *
201
+ * @param {string} tier
202
+ * @param {string} provider
203
+ * @param {string} model
204
+ * @param {number[]} context
205
+ * @returns {number|null} estimated reward in [0, 1], or null if unknown arm
206
+ */
207
+ estimateReward(tier, provider, model, context) {
208
+ const key = this._armKey(tier, provider, model);
209
+ const arm = this.arms.get(key);
210
+ if (!arm || arm.count === 0) return null;
211
+ let ctx = context;
212
+ if (ctx.length !== this.dim) {
213
+ ctx = ctx.slice(0, this.dim);
214
+ while (ctx.length < this.dim) ctx.push(0);
215
+ }
216
+ let Ainv;
217
+ try {
218
+ Ainv = _inv(arm.A);
219
+ } catch {
220
+ return null;
221
+ }
222
+ const theta = _matVec(Ainv, arm.b);
223
+ const mean = _dot(theta, ctx);
224
+ // Rewards are trained in [0, 1]; clamp the linear extrapolation so a
225
+ // wild θ·x can't dominate the DR correction term.
226
+ return Math.max(0, Math.min(1, mean));
227
+ }
228
+
190
229
  /**
191
230
  * Update the chosen arm with the observed reward.
192
231
  * @param {string} tier
@@ -19,6 +19,8 @@ const path = require('path');
19
19
 
20
20
  const DEFAULT_DAYS = 7;
21
21
  const MIN_SAMPLES = 100;
22
+ /** Narrower than this and a calibrated band is treated as collapse, not signal. */
23
+ const MIN_TIER_WIDTH = 10;
22
24
 
23
25
  /** Quality score below which a complexity bucket is "underperforming" for its tier. */
24
26
  const QUALITY_FLOOR = {
@@ -188,7 +190,24 @@ function runCalibration({ days = DEFAULT_DAYS, dryRun = false, dbPath, outputPat
188
190
  const prev = ranges[tierOrder[i - 1]];
189
191
  const cur = ranges[tierOrder[i]];
190
192
  if (cur[0] !== prev[1] + 1) cur[0] = prev[1] + 1;
191
- if (cur[0] > cur[1]) cur[1] = cur[0]; // collapsed; tier disabled in practice
193
+ if (cur[0] > cur[1]) cur[1] = cur[0]; // collapsed; see guardrail below
194
+ }
195
+
196
+ // Guardrail: a tier squeezed under MIN_TIER_WIDTH means calibration had
197
+ // degenerate signal, not a real finding — e.g. 2026-07-26 (MEDIUM →
198
+ // [20,20]) and 2026-08-22 (COMPLEX → [51,54], which re-stitched
199
+ // REASONING's floor down to 55 and sent ~all agentic traffic to the most
200
+ // expensive tier; skewed telemetry then re-taught the same bands every
201
+ // 24h). Collapsed bands revert the whole set to DEFAULT_RANGES — bands
202
+ // are interdependent after stitching, so partial reverts would leave
203
+ // gaps.
204
+ const collapsed = tierOrder.filter(
205
+ (t) => ranges[t][1] - ranges[t][0] < MIN_TIER_WIDTH
206
+ );
207
+ let guardrail = null;
208
+ if (collapsed.length > 0) {
209
+ guardrail = { reverted: true, reason: 'band_collapse', collapsed };
210
+ for (const t of tierOrder) ranges[t] = [...DEFAULT_RANGES[t]];
192
211
  }
193
212
 
194
213
  const out = {
@@ -197,6 +216,7 @@ function runCalibration({ days = DEFAULT_DAYS, dryRun = false, dbPath, outputPat
197
216
  sampleCount: rows.length,
198
217
  ranges,
199
218
  stats,
219
+ ...(guardrail ? { guardrail } : {}),
200
220
  };
201
221
 
202
222
  if (dryRun) return { ...out, dryRun: true };
@@ -0,0 +1,88 @@
1
+ /**
2
+ * Capability requirement vector (item 1: HyDRA port, phase 1).
3
+ *
4
+ * Maps the existing 15-dim weighted analysis (complexity-analyzer.js
5
+ * calculateWeightedScore dimensions, each 0-100) onto 4 independent
6
+ * capability heads in [0,1]:
7
+ *
8
+ * - reasoning : multi-step / planning / tradeoff analysis load
9
+ * - codegen : code-writing demand (generation + technical + tool depth)
10
+ * - debugging : diagnostic load (analysis + technical + domain breadth)
11
+ * - tool_use : tool-orchestration load (count, complexity, chaining, history)
12
+ *
13
+ * Deliberately decoupled from the model catalog: this module never names a
14
+ * provider/model/tier. Model capabilities live in
15
+ * config/model-capabilities.json and matching lives in shortfall.js, so a
16
+ * catalog change is a config edit with zero retraining.
17
+ *
18
+ * Deliberately excludes risk/agentic overrides: risk-high forces REASONING
19
+ * and AUTONOMOUS sets a REASONING floor upstream (routing/index.js). Those
20
+ * stay as deterministic post-passes (like HyDRA's health veto), not as
21
+ * learned heads — keeping this predictor language- and catalog-invariant.
22
+ * The agentic flag only floors tool_use (agentic work always needs tools),
23
+ * it never lowers a requirement.
24
+ *
25
+ * Pure function, no I/O. Never throws: bad input yields the neutral prior
26
+ * (all 0.2 ≈ trivial) so callers fail open to cheap routing.
27
+ */
28
+
29
+ const HEADS = ['reasoning', 'codegen', 'debugging', 'tool_use'];
30
+
31
+ function _clamp01(v) {
32
+ const n = Number(v);
33
+ if (!Number.isFinite(n)) return 0;
34
+ return Math.max(0, Math.min(1, n));
35
+ }
36
+
37
+ function _dim01(dimensions, name) {
38
+ const v = Number(dimensions?.[name]);
39
+ if (!Number.isFinite(v)) return 0;
40
+ return Math.max(0, Math.min(1, v / 100));
41
+ }
42
+
43
+ /**
44
+ * @param {object} args
45
+ * @param {object} [args.dimensions] — calculateWeightedScore dimensions (0-100 each)
46
+ * @param {object} [args.agenticResult] — { isAgentic } (floors tool_use only)
47
+ * @returns {{ reasoning:number, codegen:number, debugging:number, tool_use:number }}
48
+ */
49
+ function buildRequirementVector({ dimensions = {}, agenticResult = null } = {}) {
50
+ try {
51
+ const d = (name) => _dim01(dimensions, name);
52
+
53
+ let reasoning = d('multiStepReasoning') * 0.4
54
+ + d('analysisDepth') * 0.4
55
+ + d('promptComplexity') * 0.2;
56
+
57
+ let codegen = d('codeGeneration') * 0.5
58
+ + d('technicalDepth') * 0.3
59
+ + d('toolComplexity') * 0.2;
60
+
61
+ let debugging = d('analysisDepth') * 0.3
62
+ + d('technicalDepth') * 0.2
63
+ + d('domainSpecificity') * 0.3
64
+ + d('promptComplexity') * 0.2;
65
+
66
+ let toolUse = d('toolCount') * 0.3
67
+ + d('toolComplexity') * 0.3
68
+ + d('toolChainPotential') * 0.2
69
+ + d('priorToolUsage') * 0.2;
70
+
71
+ // Agentic work always needs tool orchestration — floor only, never lower.
72
+ if (agenticResult?.isAgentic) {
73
+ toolUse = Math.max(toolUse, 0.6);
74
+ }
75
+
76
+ const round3 = (v) => Math.round(_clamp01(v) * 1000) / 1000;
77
+ return {
78
+ reasoning: round3(reasoning),
79
+ codegen: round3(codegen),
80
+ debugging: round3(debugging),
81
+ tool_use: round3(toolUse),
82
+ };
83
+ } catch {
84
+ return { reasoning: 0.2, codegen: 0.2, debugging: 0.2, tool_use: 0.2 };
85
+ }
86
+ }
87
+
88
+ module.exports = { HEADS, buildRequirementVector };
@@ -0,0 +1,58 @@
1
+ /**
2
+ * Drop-in benchmark directory loader: data/capability-benchmarks/*.json.
3
+ *
4
+ * For leaderboards with no stable raw endpoint (TerminalBench, LiveCodeBench
5
+ * snapshots, Artificial Analysis exports): download the export once, drop it
6
+ * here, and the seed script picks it up. Format per file:
7
+ *
8
+ * { "source": "terminalbench-2.1", "scale": "percent",
9
+ * "results": { "Kimi K2.5": 42.1, "GPT 5.2": 38.0 } }
10
+ *
11
+ * scale "percent" (0-100) or "fraction" (0-1). Model names are cleaned with
12
+ * the SWE-bench cleaner and family-mapped in normalize.js; unmapped names
13
+ * surface for review instead of being silently seeded. Directory absent or
14
+ * empty → { files: [] } (not an error).
15
+ */
16
+
17
+ const fs = require('fs');
18
+ const path = require('path');
19
+ const { cleanName } = require('./swebench');
20
+
21
+ const BENCHMARKS_DIR = path.join(__dirname, '../../../data/capability-benchmarks');
22
+
23
+ function loadBenchmarksDir(dir = BENCHMARKS_DIR) {
24
+ try {
25
+ if (!fs.existsSync(dir)) return { files: [] };
26
+ const files = [];
27
+ for (const name of fs.readdirSync(dir)) {
28
+ if (!name.endsWith('.json')) continue;
29
+ try {
30
+ const raw = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8'));
31
+ if (!raw || typeof raw !== 'object' || !raw.results || typeof raw.results !== 'object') {
32
+ files.push({ file: name, status: 'skipped', reason: 'missing results object' });
33
+ continue;
34
+ }
35
+ const entries = [];
36
+ for (const [modelName, score] of Object.entries(raw.results)) {
37
+ const cleaned = cleanName(modelName);
38
+ const s = Number(score);
39
+ if (cleaned && Number.isFinite(s)) entries.push({ name: cleaned, score: s });
40
+ }
41
+ files.push({
42
+ file: name,
43
+ status: 'ok',
44
+ source: String(raw.source || name.replace(/\.json$/, '')),
45
+ scale: raw.scale === 'fraction' ? 'fraction' : 'percent',
46
+ entries,
47
+ });
48
+ } catch (err) {
49
+ files.push({ file: name, status: 'skipped', reason: err.message });
50
+ }
51
+ }
52
+ return { files };
53
+ } catch (err) {
54
+ return { files: [], error: err.message };
55
+ }
56
+ }
57
+
58
+ module.exports = { loadBenchmarksDir, BENCHMARKS_DIR };
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Family-ladder heuristics: zero-network capability estimates from model
3
+ * family names. Vendors ship in ladders (haiku<sonnet<opus, mini<full,
4
+ * flash<pro, air<max, 7b<32b<70b), so an unknown model usually reveals its
5
+ * rung by name. Deliberately pessimistic: heuristics never claim frontier
6
+ * (cap 0.85) — exact seeds do that. Returns null when nothing matches, so
7
+ * the resolver falls through to tier-slot caps (today's behavior).
8
+ *
9
+ * Tilts: coder-named models get +codegen, reasoner-named +reasoning.
10
+ * Everything rounded to 3 decimals. Pure, no I/O, never throws.
11
+ */
12
+
13
+ const CAP = 0.85;
14
+ const FLOOR = 0.15;
15
+
16
+ // Ordered: first match wins. Frontier names before size rules (a bare
17
+ // "deepseek-r1" carries no size suffix but is strong).
18
+ const LADDER = [
19
+ // Frontier (0.80)
20
+ { re: /^(gpt-5(\.|$)|o1(-pro)?$|o3($|-)|claude-opus-4|gemini-2\.5-pro|deepseek-r1$|deepseek-reasoner|glm-5\.|kimi-k3($|-)|grok-4|llama-3\.1-405b|qwen3-max|qwen3-235b|doubao-seed-1\.6|minimax-m2($|-)|gpt-5\.2|gpt-5\.3|gpt-5\.4)/, base: 0.8 },
21
+ // Strong (0.65)
22
+ { re: /(sonnet|gpt-4o$|gpt-4-turbo|gpt-5-mini|gpt-5-nano$|gemini-2\.[05].*pro|qwen3-3[02]b|qwen2\.5-3[23]b|deepseek-v3|glm-4\.7|glm-4\.6|glm-4\.5($|-)|kimi-k2($|-)|mistral-large|codestral|gpt-oss-120b|llama-3\.3-70b|qwen3-30b-a3b|devstral-large|kimi-k2\.5|gemini-3-pro|claude-4\.5)/, base: 0.65 },
23
+ // Mid (0.50)
24
+ { re: /(gpt-4o-mini|haiku|o4-mini|qwen3-(4b|8b|14b)|qwen2\.5-(7b|14b)|gemma-3-27b|phi-4|mistral-small|ministral|glm-4-flash|glm-4\.5-air|-lite$|-air$|devstral-small|qwen3-4b|gpt-5\.1)/, base: 0.5 },
25
+ // Small (0.30)
26
+ { re: /(-7b|-8b|-9b|^.*[^0-9](7|8|9)b$|mini|nano|gemma-?2|phi-3|llama-3\.2|flash-lite)/, base: 0.3 },
27
+ // Tiny (0.20)
28
+ { re: /(-1b|-2b|-3b|-0\.5b|-1\.5b|tiny$|gpt-3\.5)/, base: 0.2 },
29
+ ];
30
+
31
+ const CODER_RE = /(coder|codestral|starcoder|qwen3-coder|devstral)/;
32
+ const REASONER_RE = /(reason|r1$|qwq|\bo1\b|\bo3\b|thinking)/;
33
+
34
+ function _sizeBase(family) {
35
+ const m = String(family || '').match(/(\d+(?:\.\d+)?)\s*b(?:$|-)/);
36
+ if (!m) return null;
37
+ const gb = Number(m[1]);
38
+ if (!Number.isFinite(gb)) return null;
39
+ if (gb >= 100) return 0.8;
40
+ if (gb >= 30) return 0.65;
41
+ if (gb >= 10) return 0.5;
42
+ if (gb >= 4) return 0.35;
43
+ return 0.25;
44
+ }
45
+
46
+ /**
47
+ * @param {string} family — normalized family id (see family.js)
48
+ * @returns {null | { reasoning:number, codegen:number, debugging:number, tool_use:number }}
49
+ */
50
+ function heuristicCaps(family) {
51
+ try {
52
+ const f = String(family || '').toLowerCase().trim();
53
+ if (!f || f === 'unknown') return null;
54
+ let base = null;
55
+ for (const { re, base: b } of LADDER) {
56
+ if (re.test(f)) {
57
+ base = b;
58
+ break;
59
+ }
60
+ }
61
+ if (base === null) base = _sizeBase(f);
62
+ if (base === null) return null;
63
+ const round3 = (v) => Math.max(FLOOR, Math.min(CAP, Math.round(v * 1000) / 1000));
64
+ const codegen = base + (CODER_RE.test(f) ? 0.05 : 0);
65
+ const reasoning = base + (REASONER_RE.test(f) ? 0.05 : 0);
66
+ return {
67
+ reasoning: round3(reasoning),
68
+ codegen: round3(codegen),
69
+ debugging: round3(base),
70
+ tool_use: round3(base),
71
+ };
72
+ } catch {
73
+ return null;
74
+ }
75
+ }
76
+
77
+ module.exports = { heuristicCaps };
@@ -0,0 +1,105 @@
1
+ /**
2
+ * Model-family normalization for capability seeding.
3
+ *
4
+ * Capability is a property of the MODEL (weights), not the provider serving
5
+ * it: zai:glm-5.2, baidu:glm-5.2 and ollama:glm-5.2 must resolve to one
6
+ * family id ("glm-5.2") and therefore identical caps. Provider stays on the
7
+ * cost/invocation path only.
8
+ *
9
+ * Normalization (order matters):
10
+ * 1. lowercase + trim, drop @digest suffixes
11
+ * 2. last `/`-segment wins (strips org prefixes: zai-org/glm-5.2,
12
+ * openai/gpt-4o-mini, anthropic/claude-...)
13
+ * 3. Ollama `:tag` becomes `-tag` (tags often carry SIZE: qwen2.5-coder:7b
14
+ * → qwen2.5-coder-7b; :latest/:cloud are noise but harmless post-strip)
15
+ * 4. strip serving prefixes (databricks-, anthropic., bedrock/)
16
+ * 5. `_` → `-`, digit-dash-digit → digit.dot.digit
17
+ * (gpt-3-5-turbo → gpt-3.5-turbo, llama-3-1-70b → llama-3.1-70b),
18
+ * collapse repeats
19
+ *
20
+ * Quantization is detected, not normalized away: Q4_K_M / gguf / int4 / awq
21
+ * style markers set quant:true so the resolver can apply the small
22
+ * same-brain-smaller-body haircut. Full-precision self-hosted hits the same
23
+ * caps as the API.
24
+ *
25
+ * Pure functions, no I/O. Never throws.
26
+ */
27
+
28
+ const SERVING_PREFIXES = ['databricks-', 'anthropic.', 'bedrock/'];
29
+
30
+ // Quant markers: matched against the full "model:tag" string BEFORE tag
31
+ // folding, and against the folded family after (tags like Q4_K_M survive as
32
+ // -q4-k-m, so one pass after folding suffices — but pre-tag names like
33
+ // "model-Q4" exist too, hence match post-fold only, on the whole string).
34
+ const QUANT_RE = /(q[23468](_|-|$)|_k_[sm]|[-_]q8_0|gguf|int[48]|fp[148]|awq|gptq|bnb|mlx|quantized)/i;
35
+
36
+ // Tags that carry no signal even as suffixes.
37
+ const NOISE_TAGS = new Set(['latest', 'cloud', 'instruct']);
38
+
39
+ function detectQuant(s) {
40
+ try {
41
+ return QUANT_RE.test(String(s || ''));
42
+ } catch {
43
+ return false;
44
+ }
45
+ }
46
+
47
+ /**
48
+ * @param {string} provider
49
+ * @param {string} model
50
+ * @returns {{ family:string, quant:boolean }}
51
+ */
52
+ function normalizeFamily(provider, model) {
53
+ try {
54
+ let s = `${String(model || '').trim()}`;
55
+ // @sha256:... digests
56
+ s = s.split('@')[0];
57
+ // org prefix: last segment wins
58
+ if (s.includes('/')) s = s.split('/').pop();
59
+ const quant = detectQuant(`${provider || ''}/${model || ''}`) || detectQuant(s);
60
+ // Ollama :tag → -tag (size tags preserved: :7b → -7b)
61
+ if (s.includes(':')) {
62
+ const [base, ...tags] = s.split(':');
63
+ const kept = tags
64
+ .map((t) => t.trim().toLowerCase())
65
+ .filter((t) => t && !NOISE_TAGS.has(t));
66
+ s = kept.length > 0 ? `${base}-${kept.join('-')}` : base;
67
+ }
68
+ s = s.toLowerCase().trim();
69
+ for (const p of SERVING_PREFIXES) {
70
+ if (s.startsWith(p)) {
71
+ s = s.slice(p.length);
72
+ break;
73
+ }
74
+ }
75
+ s = s.replace(/_/g, '-');
76
+ // digit-dash-digit → digit.dot.digit, but only for version runs: the
77
+ // second digit must be followed by another dash or end-of-string, so
78
+ // size suffixes survive (llama-3-1-70b → llama-3.1-70b, but
79
+ // qwen3-32b and gemma-3-27b keep their dash).
80
+ s = s.replace(/(\d)-(?=\d(?:-|$))/g, '$1.');
81
+ s = s.replace(/-+/g, '-').replace(/^-|-$/g, '');
82
+ return { family: s || 'unknown', quant };
83
+ } catch {
84
+ return { family: 'unknown', quant: false };
85
+ }
86
+ }
87
+
88
+ /**
89
+ * Quant haircut: same brain, smaller body. Applied by the resolver to
90
+ * seed/family/tier caps — never to explicit operator overrides (an exact
91
+ * provider:model override wins verbatim).
92
+ */
93
+ const QUANT_HAIRCUT = 0.02;
94
+ const QUANT_FLOOR = 0.1;
95
+
96
+ function applyQuantHaircut(caps) {
97
+ const out = {};
98
+ for (const [k, v] of Object.entries(caps || {})) {
99
+ const n = Number(v);
100
+ out[k] = Number.isFinite(n) ? Math.max(QUANT_FLOOR, Math.round((n - QUANT_HAIRCUT) * 1000) / 1000) : v;
101
+ }
102
+ return out;
103
+ }
104
+
105
+ module.exports = { normalizeFamily, detectQuant, applyQuantHaircut, QUANT_HAIRCUT };