lynkr 9.14.1 → 9.14.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +5 -1
  2. package/bin/cli.js +14 -6
  3. package/bin/lynkr-desktop-token.js +108 -0
  4. package/bin/lynkr-init.js +13 -1
  5. package/bin/lynkr-restart.js +125 -0
  6. package/bin/opencode-setup.js +175 -0
  7. package/bin/run.js +92 -0
  8. package/config/model-capabilities.json +20 -0
  9. package/config/model-capability-seeds.json +75 -0
  10. package/config/model-tiers.json +13 -1
  11. package/package.json +16 -22
  12. package/scripts/claude-desktop.js +212 -0
  13. package/scripts/ope-report.js +78 -0
  14. package/scripts/seed-capabilities.js +230 -0
  15. package/src/api/claude-desktop-gateway.js +123 -0
  16. package/src/api/health.js +0 -1
  17. package/src/api/mcp-broker.js +149 -0
  18. package/src/api/middleware/budget-enforcer.js +77 -7
  19. package/src/api/middleware/budget.js +36 -2
  20. package/src/api/middleware/session.js +0 -0
  21. package/src/api/openai-router.js +210 -15
  22. package/src/api/providers-handler.js +16 -1
  23. package/src/api/router.js +73 -106
  24. package/src/auth-mode.js +40 -4
  25. package/src/budget/index.js +86 -0
  26. package/src/cache/embeddings.js +125 -35
  27. package/src/cache/onnx-embedder.js +92 -0
  28. package/src/cache/prompt.js +0 -7
  29. package/src/clients/bedrock-utils.js +2 -1
  30. package/src/clients/codex-process.js +1 -1
  31. package/src/clients/codex-utils.js +0 -2
  32. package/src/clients/databricks.js +337 -46
  33. package/src/clients/gpt-utils.js +84 -8
  34. package/src/clients/health-probe.js +157 -0
  35. package/src/clients/openai-format.js +73 -23
  36. package/src/clients/provider-capabilities.js +27 -2
  37. package/src/clients/resilience.js +0 -1
  38. package/src/clients/responses-format.js +0 -56
  39. package/src/clients/routing.js +1 -1
  40. package/src/config/index.js +13 -2
  41. package/src/context/budget.js +1 -1
  42. package/src/context/compression.js +127 -19
  43. package/src/context/tool-result-compressor.js +3 -5
  44. package/src/dashboard/api.js +1 -1
  45. package/src/logger/oversized-error-stream.js +1 -1
  46. package/src/memory/extractor.js +3 -3
  47. package/src/memory/search.js +2 -2
  48. package/src/memory/store.js +9 -24
  49. package/src/memory/tools.js +14 -0
  50. package/src/observability/metrics.js +52 -3
  51. package/src/observability/otel.js +180 -0
  52. package/src/orchestrator/azure-responses-sse.js +223 -0
  53. package/src/orchestrator/index.js +216 -212
  54. package/src/orchestrator/passthrough-stream.js +7 -1
  55. package/src/orchestrator/sse-transformer.js +130 -13
  56. package/src/policy/safe-commands.js +0 -1
  57. package/src/routing/bandit.js +39 -0
  58. package/src/routing/calibration.js +21 -1
  59. package/src/routing/capabilities.js +88 -0
  60. package/src/routing/capability-seeds/benchmarks-dir.js +58 -0
  61. package/src/routing/capability-seeds/family-heuristics.js +77 -0
  62. package/src/routing/capability-seeds/family.js +105 -0
  63. package/src/routing/capability-seeds/models-dev.js +64 -0
  64. package/src/routing/capability-seeds/normalize.js +160 -0
  65. package/src/routing/capability-seeds/registry.js +125 -0
  66. package/src/routing/capability-seeds/sources.js +69 -0
  67. package/src/routing/capability-seeds/swebench.js +66 -0
  68. package/src/routing/client-profiles.js +26 -0
  69. package/src/routing/complexity-analyzer.js +5 -8
  70. package/src/routing/confidence-scorer.js +2 -2
  71. package/src/routing/cost-optimizer.js +1 -2
  72. package/src/routing/decide.js +149 -0
  73. package/src/routing/deescalator.js +38 -1
  74. package/src/routing/embedding-cache.js +0 -1
  75. package/src/routing/index.js +202 -86
  76. package/src/routing/intent-score.js +0 -1
  77. package/src/routing/knn-router.js +27 -5
  78. package/src/routing/latency-tracker.js +1 -1
  79. package/src/routing/model-registry.js +25 -0
  80. package/src/routing/model-slots.js +65 -0
  81. package/src/routing/model-tiers.js +2 -0
  82. package/src/routing/ope.js +255 -0
  83. package/src/routing/openai-model-slots.js +82 -0
  84. package/src/routing/regret-estimator.js +1 -1
  85. package/src/routing/shortfall.js +290 -0
  86. package/src/routing/stuck-detector.js +127 -0
  87. package/src/routing/telemetry.js +11 -2
  88. package/src/server.js +41 -3
  89. package/src/stores/file-store.js +5 -1
  90. package/src/tools/web-search-exec.js +182 -0
  91. package/src/training/trajectory-compressor.js +1 -1
  92. package/src/workers/pool.js +1 -1
  93. package/src/workers/worker.js +1 -1
  94. package/scripts/convert-routellm.py +0 -105
  95. package/src/diff/comments.js +0 -153
  96. package/src/policy/web-fallback.js +0 -33
package/src/api/router.js CHANGED
@@ -7,7 +7,9 @@ const logger = require("../logger");
7
7
  const { createRateLimiter } = require("./middleware/rate-limiter");
8
8
  const openaiRouter = require("./openai-router");
9
9
  const providersRouter = require("./providers-handler");
10
+ const claudeDesktopGatewayRouter = require("./claude-desktop-gateway");
10
11
  const { getRoutingHeaders, getRoutingStats, analyzeComplexity, getModelTierSelector, analyzeRisk, checkSessionPin, writeSessionPin, checkPinScoreDrift } = require("../routing");
12
+ const { resolveTierForModelId } = require("../routing/model-slots");
11
13
 
12
14
  // Upstream streams can die without a clean end (reader.read() never
13
15
  // resolves on a dropped socket), hanging the client forever. Every
@@ -36,7 +38,7 @@ const { detectClient } = require("../routing/client-profiles");
36
38
  const { buildInteractionBlock } = require("../routing/interaction");
37
39
  const { validateCwd } = require("../workspace");
38
40
  const { renderText } = require("../utils/markdown-ansi");
39
- const { classifyAuthMode } = require("../auth-mode");
41
+ const { classifyAuthMode, isFirstPartyAnthropicClient } = require("../auth-mode");
40
42
 
41
43
  const router = express.Router();
42
44
 
@@ -306,21 +308,16 @@ async function handleOauthPassthrough(req, res, opts = {}) {
306
308
  const upstream = process.env.LYNKR_OAUTH_PASSTHROUGH_URL
307
309
  || "https://api.anthropic.com/v1/messages";
308
310
 
309
- // === Optional: memory injection at last-user-message tail ===
310
- // Headroom's P0-1 pattern: append memory context to the latest user
311
- // message's first text block. NEVER touches system prompt or frozen-prefix
312
- // messages, so the cache-hot zone Anthropic fingerprints stays intact.
313
- // Opt-in via LYNKR_OAUTH_MEMORY_INJECTION=true since any body mutation on
314
- // a subscription request has nonzero anti-abuse risk.
311
+ // NO body mutation on the passthrough — deliberate policy (2026-09). The
312
+ // memory-injection option that lived here (LYNKR_OAUTH_MEMORY_INJECTION)
313
+ // was removed: this function's whole contract is byte-for-byte fidelity
314
+ // to what the first-party client sent, and any mutation of subscription
315
+ // traffic both breaks that claim and carries anti-abuse risk. Memory
316
+ // injection remains available on the orchestrator paths, where Lynkr is
317
+ // openly transforming requests. The only change made below is stripping
318
+ // Lynkr-internal underscore-prefixed fields, which Anthropic would reject
319
+ // as unknown keys — protocol necessity, not content mutation.
315
320
  let bodyToSend = req.body;
316
- if (process.env.LYNKR_OAUTH_MEMORY_INJECTION === 'true' && config.memory?.enabled !== false) {
317
- try {
318
- bodyToSend = maybeInjectMemoryIntoUserTail(req.body);
319
- } catch (err) {
320
- logger.debug({ err: err.message }, "Memory injection skipped (non-fatal)");
321
- bodyToSend = req.body;
322
- }
323
- }
324
321
 
325
322
  // === Observability: start ===
326
323
  const startedAt = Date.now();
@@ -390,8 +387,13 @@ async function handleOauthPassthrough(req, res, opts = {}) {
390
387
  // Lynkr's own decision headers so callers can see which model answered.
391
388
  res.set("X-Lynkr-Provider", "azure-anthropic-passthrough");
392
389
  if (opts.tier?.tier) res.set("X-Lynkr-Tier", opts.tier.tier);
393
- if (req.body?.model) res.set("X-Lynkr-Model", req.body.model);
394
- res.set("X-Lynkr-Routing-Method", "oauth-subscription-stealth");
390
+ if (req.body?.model) {
391
+ res.set("X-Lynkr-Model", req.body.model);
392
+ const { contextWindowFor } = require("../routing/model-registry");
393
+ const contextWindow = contextWindowFor(req.body.model);
394
+ if (contextWindow) res.set("X-Lynkr-Context-Window", String(contextWindow));
395
+ }
396
+ res.set("X-Lynkr-Routing-Method", "oauth-subscription-passthrough");
395
397
 
396
398
  // Capture the response (buffered or streamed) so we can do observability hooks
397
399
  // on the way back without changing what the client sees.
@@ -450,7 +452,7 @@ async function handleOauthPassthrough(req, res, opts = {}) {
450
452
  try { reader.cancel(); } catch { /* already dead */ }
451
453
  try { res.write(SSE_STALL_EVENT); } catch { /* client gone */ }
452
454
  } finally {
453
- try { reader.releaseLock(); } catch {}
455
+ try { reader.releaseLock(); } catch { /* lock already released */ }
454
456
  }
455
457
  res.end();
456
458
  } else {
@@ -495,7 +497,7 @@ async function handleOauthPassthrough(req, res, opts = {}) {
495
497
  const tier = opts.tier || {};
496
498
  let parsedResponse = null;
497
499
  if (contentType.includes("application/json")) {
498
- try { parsedResponse = JSON.parse(responseTextForObservability); } catch {}
500
+ try { parsedResponse = JSON.parse(responseTextForObservability); } catch { /* not JSON — telemetry proceeds without a parsed body */ }
499
501
  } else if (contentType.includes("text/event-stream")) {
500
502
  // Extract a usable response object from the SSE stream by finding the
501
503
  // final message_delta / message_stop events.
@@ -730,86 +732,8 @@ function extractAnthropicMessageFromSSE(sseText) {
730
732
  return result;
731
733
  }
732
734
 
733
- /**
734
- * Append relevant memories to the FIRST TEXT BLOCK of the LATEST USER MESSAGE.
735
- *
736
- * Headroom's P0-1 pattern (`_append_context_to_latest_non_frozen_user_turn`).
737
- * The cache hot zone (system + frozen prefix) is NEVER touched. Mutating only
738
- * the latest user message — which is the request's "live zone" — keeps the
739
- * prompt-cache identity stable and avoids Anthropic anti-abuse fingerprint
740
- * divergence for subscription tokens.
741
- *
742
- * Returns the body unchanged if:
743
- * - Memory is disabled
744
- * - No memories retrieved
745
- * - Latest message is not a user turn (could be tool_result, assistant)
746
- *
747
- * Returns a new body with appended context otherwise. Original body never
748
- * mutated (returns a shallow-cloned messages array).
749
- */
750
- function maybeInjectMemoryIntoUserTail(body) {
751
- if (!body || !Array.isArray(body.messages) || body.messages.length === 0) return body;
752
-
753
- const lastIdx = body.messages.length - 1;
754
- const lastMsg = body.messages[lastIdx];
755
- if (!lastMsg || lastMsg.role !== "user") return body;
756
-
757
- const { retrieveRelevantMemories, formatMemoriesForContext, extractQueryFromMessage } =
758
- require("../memory/retriever");
759
- const query = extractQueryFromMessage(lastMsg);
760
- if (!query || query.length < 10) return body; // too short to be a useful query
761
-
762
- const memories = retrieveRelevantMemories(query, {
763
- limit: Math.min(parseInt(process.env.MEMORY_RETRIEVAL_LIMIT, 10) || 5, 10),
764
- sessionId: body._sessionId || null,
765
- includeGlobal: process.env.MEMORY_INCLUDE_GLOBAL !== "false",
766
- });
767
- if (!memories || memories.length === 0) return body;
768
-
769
- const formatted = formatMemoriesForContext(memories);
770
- if (!formatted) return body;
771
-
772
- const contextText = `\n\n## Relevant context from earlier sessions:\n${formatted}`;
773
-
774
- // Bound the injection size (Headroom uses a MemoryInjectionBudget; we use
775
- // a simpler char cap — ~1024 tokens * 4 chars/token = 4096 chars).
776
- const MAX_INJECTION_CHARS = 4096;
777
- const boundedContext = contextText.length > MAX_INJECTION_CHARS
778
- ? contextText.slice(0, MAX_INJECTION_CHARS) + "\n…"
779
- : contextText;
780
-
781
- // Clone messages array (shallow) so we don't mutate the caller's body.
782
- const newMessages = body.messages.slice();
783
-
784
- if (typeof lastMsg.content === "string") {
785
- newMessages[lastIdx] = { ...lastMsg, content: lastMsg.content + boundedContext };
786
- } else if (Array.isArray(lastMsg.content) && lastMsg.content.length > 0) {
787
- // Append to the FIRST text block, preserving every other block (images,
788
- // tool_use, etc.) untouched.
789
- const newContent = [];
790
- let appended = false;
791
- for (const block of lastMsg.content) {
792
- if (!appended && block && typeof block === "object" && block.type === "text") {
793
- newContent.push({ ...block, text: (block.text || "") + boundedContext });
794
- appended = true;
795
- } else {
796
- newContent.push(block);
797
- }
798
- }
799
- if (!appended) return body; // no text block to append to
800
- newMessages[lastIdx] = { ...lastMsg, content: newContent };
801
- } else {
802
- return body;
803
- }
804
-
805
- logger.debug({
806
- memoryCount: memories.length,
807
- appendedChars: boundedContext.length,
808
- }, "Memory injected into last-user-message tail");
809
-
810
- return { ...body, messages: newMessages };
811
- }
812
-
735
+ // (maybeInjectMemoryIntoUserTail was removed with the passthrough
736
+ // memory-injection option — the passthrough is byte-for-byte by contract.)
813
737
  /**
814
738
  * Estimate token count for messages.
815
739
  *
@@ -1027,7 +951,10 @@ router.post("/v1/messages", rateLimiter, async (req, res, next) => {
1027
951
  try {
1028
952
  const { getMetricsCollector } = require("../observability/metrics");
1029
953
  getMetricsCollector().recordRequest("POST", "/v1/messages", null, null);
1030
- } catch (_) {}
954
+ } catch (err) {
955
+ // If this fails silently, the wrap UI's "No requests tracked" bug returns.
956
+ logger.debug({ err: err?.message }, "observability collector recordRequest failed");
957
+ }
1031
958
 
1032
959
  messagesRequestCount += 1;
1033
960
 
@@ -1089,7 +1016,7 @@ router.post("/v1/messages", rateLimiter, async (req, res, next) => {
1089
1016
  //
1090
1017
  // All three paths now share window-scored intent tier picking
1091
1018
  // (`pickTierByIntent`). Subscription still has the additional
1092
- // azure-anthropic passthrough fork for anti-abuse stealth; everything
1019
+ // azure-anthropic passthrough fork (first-party clients only); everything
1093
1020
  // else just falls through to the orchestrator with the picked tier
1094
1021
  // pinned via _forceProvider/_tierModel. The reason all paths share the
1095
1022
  // scorer is that determineProviderSmart's full-body analysis inflates
@@ -1114,6 +1041,37 @@ router.post("/v1/messages", rateLimiter, async (req, res, next) => {
1114
1041
  // in a session skip the whole window-scored intent pipeline.
1115
1042
  let tier;
1116
1043
  const pinCheck = checkSessionPin(req.body);
1044
+
1045
+ // Explicit model-id pin: Claude Desktop's model picker (or any Anthropic-
1046
+ // format caller) can name one of the fixed ids claude-desktop-gateway.js
1047
+ // advertises (see ../routing/model-slots.js). Unlike every signal below,
1048
+ // this one is EXPLICIT user intent, not an inference — it wins over
1049
+ // session pins, side-request detection, and content scoring alike.
1050
+ // "Lynkr Auto" (claude-fable-5) and any unrecognized id resolve to null
1051
+ // and fall through to the existing cascade unchanged.
1052
+ const modelPinTier = resolveTierForModelId(req.body?.model);
1053
+ if (modelPinTier) {
1054
+ const _sel = getModelTierSelector().selectModel(modelPinTier, null);
1055
+ tier = {
1056
+ tier: modelPinTier,
1057
+ provider: _sel.provider,
1058
+ model: _sel.model || null,
1059
+ score: null,
1060
+ method: 'model_id_pin',
1061
+ reason: 'client_selected_model',
1062
+ base_tier: null,
1063
+ escalation_source: null,
1064
+ pinned: false,
1065
+ switch_reason: null,
1066
+ propensity: 1.0,
1067
+ candidates: [{ provider: _sel.provider, model: _sel.model || null }],
1068
+ };
1069
+ logger.debug({
1070
+ model: req.body.model,
1071
+ tier: modelPinTier,
1072
+ provider: _sel.provider,
1073
+ }, "[Routing] Explicit model-id pin — scoring bypassed");
1074
+ } else {
1117
1075
  // Side-request detection. Claude Code fires internal background calls
1118
1076
  // (title generation, summarization, memory extraction, suggestion-mode
1119
1077
  // autocomplete) that REPLAY the conversation — so they share the
@@ -1340,11 +1298,13 @@ router.post("/v1/messages", rateLimiter, async (req, res, next) => {
1340
1298
  }, "OAuth intent — side request (no tools), pin write skipped");
1341
1299
  }
1342
1300
  }
1301
+ } // end else — no explicit model-id pin, scored normally above
1343
1302
 
1344
- // Subscription-only fork: anti-abuse stealth passthrough when the picked
1345
- // tier resolves to azure-anthropic. Bypasses the orchestrator entirely
1346
- // so the inbound bytes hit api.anthropic.com unchanged (Anthropic
1347
- // fingerprints subscription clients; any mutation gets flagged).
1303
+ // Subscription-only fork: byte-for-byte passthrough to api.anthropic.com
1304
+ // when the picked tier resolves to azure-anthropic — FIRST-PARTY
1305
+ // Anthropic clients only (Claude Code / Claude Desktop wrapping their
1306
+ // own traffic; see auth-mode.js for the policy). Gated here at the
1307
+ // dispatch point, not just in classification, so the two can't drift.
1348
1308
  // Passthrough forwards the CLIENT's credentials; GUI harnesses that
1349
1309
  // spawn claude headless inject placeholder keys ("dummy") that Anthropic
1350
1310
  // 401s. Placeholder auth ⇒ serve REASONING via the provider's own
@@ -1354,13 +1314,14 @@ router.post("/v1/messages", rateLimiter, async (req, res, next) => {
1354
1314
  const _placeholderAuth =
1355
1315
  !_clientAuthHdr &&
1356
1316
  (['dummy', 'test', 'placeholder', 'none', 'x', 'sk-dummy'].includes(_clientApiKey) || _clientApiKey.length < 8);
1317
+ const _firstParty = isFirstPartyAnthropicClient(req.headers);
1357
1318
  if (_placeholderAuth && authMode === 'subscription' && tier.provider === 'azure-anthropic') {
1358
1319
  logger.info({
1359
1320
  reqNumber: messagesRequestCount,
1360
1321
  xApiKeyShape: _clientApiKey.slice(0, 12),
1361
1322
  }, 'Placeholder client auth — skipping passthrough, serving REASONING via provider credentials');
1362
1323
  }
1363
- if (authMode === 'subscription' && tier.provider === 'azure-anthropic' && !_placeholderAuth) {
1324
+ if (authMode === 'subscription' && tier.provider === 'azure-anthropic' && !_placeholderAuth && _firstParty) {
1364
1325
  logger.debug({
1365
1326
  reqNumber: messagesRequestCount,
1366
1327
  authMode,
@@ -2087,6 +2048,12 @@ router.get("/api/tokens/stats", (req, res) => {
2087
2048
  }
2088
2049
  });
2089
2050
 
2051
+ // Claude Desktop third-party gateway: intercepts GET /v1/models for
2052
+ // Anthropic-API clients (anthropic-version header) and advertises Lynkr's
2053
+ // tiers as Claude model families. Mounted BEFORE openaiRouter so the
2054
+ // intercept wins; non-Anthropic callers fall through unchanged.
2055
+ router.use("/v1", claudeDesktopGatewayRouter);
2056
+
2090
2057
  // Mount OpenAI-compatible endpoints for Cursor IDE support
2091
2058
  router.use("/v1", openaiRouter);
2092
2059
 
package/src/auth-mode.js CHANGED
@@ -47,15 +47,34 @@
47
47
  * @module auth-mode
48
48
  */
49
49
 
50
+ // FIRST-PARTY Anthropic clients only (deliberate policy, 2026-09): the
51
+ // subscription classification — and with it the api.anthropic.com
52
+ // passthrough — is for Claude Code and Claude Desktop wrapping their OWN
53
+ // traffic. Third-party harness prefixes (codex-cli/, cursor/,
54
+ // github-copilot/, antigravity/) were removed: classifying them as
55
+ // 'subscription' handed the passthrough to exactly the harness category
56
+ // Anthropic's ToS enforcement targets. Those clients still work through
57
+ // Lynkr — they classify by token shape ('oauth'/'payg') and route through
58
+ // the orchestrator with the operator's configured providers.
59
+ //
60
+ // BREAKING CHANGE for anyone who had Cursor/Codex/Copilot riding the
61
+ // subscription path: that traffic now routes via configured tiers.
62
+ //
63
+ // If Claude Desktop's own requests carry a UA not listed here (capture one
64
+ // via the desktop gateway before assuming), add THAT — never a third-party
65
+ // harness. This is a UA check: it defines intended use, not cryptographic
66
+ // attestation.
50
67
  const SUBSCRIPTION_UA_PREFIXES = [
51
68
  'claude-cli/',
52
69
  'claude-code/',
53
- 'codex-cli/',
54
- 'cursor/',
55
70
  'claude-vscode/',
56
- 'github-copilot/',
57
71
  'anthropic-cli/',
58
- 'antigravity/',
72
+ // Claude Desktop (Electron app). Captured from a real gateway request:
73
+ // "Mozilla/5.0 (...) Claude/1.46388.1 Chrome/148... Electron/42..." —
74
+ // the app identifies as "Claude/<version>" inside a browser-style UA.
75
+ // Deliberately 'claude/' (with slash): no third-party harness UA carries
76
+ // that token, and the first-party ids above don't collide with it.
77
+ 'claude/',
59
78
  ];
60
79
 
61
80
  /**
@@ -110,7 +129,24 @@ function classifyAuthMode(headers) {
110
129
  return 'payg';
111
130
  }
112
131
 
132
+ /**
133
+ * Is this request from a first-party Anthropic client (Claude Code / Claude
134
+ * Desktop family)? Used as the explicit gate at the api.anthropic.com
135
+ * passthrough dispatch fork — the security decision lives at the point of
136
+ * dispatch, not only in classification, so the two can't silently drift
137
+ * apart. Same UA semantics as classifyAuthMode's subscription check.
138
+ *
139
+ * @param {object} headers
140
+ * @returns {boolean}
141
+ */
142
+ function isFirstPartyAnthropicClient(headers) {
143
+ const ua = getHeader(headers, 'user-agent').toLowerCase();
144
+ if (!ua) return false;
145
+ return SUBSCRIPTION_UA_PREFIXES.some((prefix) => ua.includes(prefix));
146
+ }
147
+
113
148
  module.exports = {
114
149
  classifyAuthMode,
150
+ isFirstPartyAnthropicClient,
115
151
  SUBSCRIPTION_UA_PREFIXES,
116
152
  };
@@ -75,6 +75,12 @@ class BudgetManager {
75
75
  minute_window_start INTEGER,
76
76
  hour_window_start INTEGER
77
77
  );
78
+
79
+ CREATE TABLE IF NOT EXISTS token_rate (
80
+ user_id TEXT PRIMARY KEY,
81
+ tokens_minute INTEGER NOT NULL DEFAULT 0,
82
+ minute_window_start INTEGER NOT NULL
83
+ );
78
84
  `);
79
85
 
80
86
  this.stmts = {
@@ -114,6 +120,86 @@ class BudgetManager {
114
120
  };
115
121
  }
116
122
 
123
+ /**
124
+ * Token-aware (TPM) rate limiting (ROUTING-NOTES §1 gap: request-count
125
+ * only). Off unless LYNKR_TPM_LIMIT is set — same "off unless configured"
126
+ * convention as the loop guard.
127
+ *
128
+ * Estimate → true-up pattern (the one every serious gateway converged on):
129
+ * the pre-flight check gates on the window's ACTUAL consumption plus the
130
+ * current request's cheap estimate; real usage lands in the window
131
+ * post-response via recordTokenUsage(). Soft admission control, not a hard
132
+ * ceiling — concurrent in-flight requests can overshoot by roughly one
133
+ * request's worth, by design.
134
+ *
135
+ * @param {string} userId
136
+ * @param {number} estimatedTokens — cheap pre-flight estimate for THIS request
137
+ * @returns {{allowed: boolean, reason?: string, limit?: number, current?: number, resetInMs?: number}}
138
+ */
139
+ checkTokenRate(userId, estimatedTokens = 0) {
140
+ if (!this.enabled) return { allowed: true };
141
+ const limit = Number.parseInt(process.env.LYNKR_TPM_LIMIT, 10);
142
+ if (!limit || limit <= 0 || Number.isNaN(limit)) return { allowed: true };
143
+
144
+ const now = Date.now();
145
+ const minuteWindow = 60 * 1000;
146
+ const row = this.db.prepare('SELECT * FROM token_rate WHERE user_id = ?').get(userId);
147
+ let tokensMinute = row?.tokens_minute ?? 0;
148
+ let windowStart = row?.minute_window_start ?? now;
149
+ if (now - windowStart >= minuteWindow) {
150
+ tokensMinute = 0;
151
+ windowStart = now;
152
+ this.db.prepare(`
153
+ INSERT INTO token_rate (user_id, tokens_minute, minute_window_start)
154
+ VALUES (?, 0, ?)
155
+ ON CONFLICT(user_id) DO UPDATE SET tokens_minute = 0, minute_window_start = excluded.minute_window_start
156
+ `).run(userId, windowStart);
157
+ }
158
+
159
+ if (tokensMinute + estimatedTokens > limit) {
160
+ return {
161
+ allowed: false,
162
+ reason: 'token_rate_limit_minute',
163
+ limit,
164
+ current: tokensMinute,
165
+ estimated: estimatedTokens,
166
+ resetInMs: minuteWindow - (now - windowStart),
167
+ };
168
+ }
169
+ return { allowed: true };
170
+ }
171
+
172
+ /**
173
+ * True-up: add ACTUAL token consumption to the user's TPM window after the
174
+ * response completes. No-op when TPM limiting is disabled.
175
+ * @param {string} userId
176
+ * @param {number} totalTokens — actual input+output tokens consumed
177
+ */
178
+ recordTokenUsage(userId, totalTokens) {
179
+ if (!this.enabled || !totalTokens || totalTokens <= 0) return;
180
+ const limit = Number.parseInt(process.env.LYNKR_TPM_LIMIT, 10);
181
+ if (!limit || limit <= 0 || Number.isNaN(limit)) return;
182
+ const now = Date.now();
183
+ const minuteWindow = 60 * 1000;
184
+ try {
185
+ const row = this.db.prepare('SELECT * FROM token_rate WHERE user_id = ?').get(userId);
186
+ const stale = !row || now - row.minute_window_start >= minuteWindow;
187
+ this.db.prepare(`
188
+ INSERT INTO token_rate (user_id, tokens_minute, minute_window_start)
189
+ VALUES (@user_id, @tokens, @window_start)
190
+ ON CONFLICT(user_id) DO UPDATE SET
191
+ tokens_minute = ${stale ? '@tokens' : 'tokens_minute + @tokens'},
192
+ minute_window_start = @window_start
193
+ `).run({
194
+ user_id: userId,
195
+ tokens: totalTokens,
196
+ window_start: stale ? now : row.minute_window_start,
197
+ });
198
+ } catch (err) {
199
+ logger.debug({ err: err.message }, '[Budget] token usage record failed');
200
+ }
201
+ }
202
+
117
203
  checkRateLimit(userId) {
118
204
  if (!this.enabled) return { allowed: true };
119
205
 
@@ -115,55 +115,136 @@ function simpleHash(str) {
115
115
  return hash;
116
116
  }
117
117
 
118
- // Track if embedding provider is available
118
+ // Embedding provider availability state.
119
+ //
120
+ // The hash fallback is NOT a semantic embedding — when it's active, the
121
+ // semantic cache and kNN router are effectively disabled (hash vectors are
122
+ // 384-dim vs the providers' 768-dim, so they never match real entries; the
123
+ // kNN index rejects them on dimension). Historically this degradation was a
124
+ // permanent latch (one transient provider failure disabled semantic matching
125
+ // until process restart) and logged only at debug level — i.e. invisible.
126
+ // The same failure shape elsewhere in the industry silently misrouted ~20%
127
+ // of a production cluster's spend before anyone noticed, so this state is
128
+ // now: retried on a cooldown, logged loudly on every transition, counted,
129
+ // and exposed via getEmbeddingStatus() for /metrics.
119
130
  let embeddingProviderAvailable = null;
131
+ let degradedSince = null;
132
+ let lastProviderAttempt = 0;
133
+ let fallbackCount = 0;
134
+ let lastProviderError = null;
135
+
136
+ // After a provider failure, wait this long before trying it again (instead
137
+ // of latching to the fallback forever). Env-tunable for tests/impatience.
138
+ const RETRY_COOLDOWN_MS = Number.parseInt(process.env.LYNKR_EMBEDDINGS_RETRY_COOLDOWN_MS, 10) || 60_000;
139
+
140
+ // Strict mode: throw instead of silently degrading to hash embeddings.
141
+ // For deployments that prefer fail-loud (no semantic cache is better than a
142
+ // silently fake one). Default stays fail-soft to match Lynkr's philosophy.
143
+ const STRICT = process.env.LYNKR_EMBEDDINGS_STRICT === 'true';
144
+
145
+ function _noteFallback(providerName, err) {
146
+ fallbackCount += 1;
147
+ lastProviderError = err?.message || String(err);
148
+ if (embeddingProviderAvailable !== false) {
149
+ embeddingProviderAvailable = false;
150
+ degradedSince = Date.now();
151
+ logger.warn({
152
+ provider: providerName,
153
+ error: lastProviderError,
154
+ retryCooldownMs: RETRY_COOLDOWN_MS,
155
+ }, '[Embeddings] Provider unreachable — DEGRADED to non-semantic hash embeddings. Semantic cache and kNN routing are effectively disabled until the provider recovers.');
156
+ }
157
+ }
158
+
159
+ function _noteRecovery(providerName) {
160
+ if (embeddingProviderAvailable === false) {
161
+ logger.info({
162
+ provider: providerName,
163
+ degradedForMs: degradedSince ? Date.now() - degradedSince : null,
164
+ fallbacksServed: fallbackCount,
165
+ }, '[Embeddings] Provider recovered — semantic embeddings restored');
166
+ }
167
+ embeddingProviderAvailable = true;
168
+ degradedSince = null;
169
+ }
170
+
171
+ function _wrapProvider(providerName, providerFn) {
172
+ return async (text) => {
173
+ // While degraded, only re-attempt the provider after the cooldown; serve
174
+ // the fallback in between so a dead provider doesn't add per-request
175
+ // connect timeouts to the hot path.
176
+ if (embeddingProviderAvailable === false
177
+ && Date.now() - lastProviderAttempt < RETRY_COOLDOWN_MS) {
178
+ if (STRICT) throw new Error(`Embedding provider ${providerName} degraded: ${lastProviderError}`);
179
+ fallbackCount += 1;
180
+ return generateHashEmbedding(text);
181
+ }
182
+ lastProviderAttempt = Date.now();
183
+ try {
184
+ const result = await providerFn(text);
185
+ _noteRecovery(providerName);
186
+ return result;
187
+ } catch (err) {
188
+ _noteFallback(providerName, err);
189
+ if (STRICT) throw err;
190
+ return generateHashEmbedding(text);
191
+ }
192
+ };
193
+ }
120
194
 
121
195
  /**
122
196
  * Get the appropriate embedding function based on config
123
197
  * @returns {Function} - Embedding generation function
124
198
  */
125
199
  function getEmbeddingFunction() {
126
- // If we already know embedding provider isn't available, use fallback
127
- if (embeddingProviderAvailable === false) {
128
- return (text) => Promise.resolve(generateHashEmbedding(text));
129
- }
130
-
131
200
  const provider = config.modelProvider?.type || 'databricks';
132
201
 
202
+ // In-process ONNX embedder (opt-in): no external embedding server on the
203
+ // hot path at all. Same underlying model as the Ollama default
204
+ // (nomic-embed-text, 768-dim), so existing kNN/cache vectors stay valid.
205
+ // Wrapped in the same degradation machinery — a failed model load logs
206
+ // loudly, serves the hash fallback, and retries after the cooldown.
207
+ if (process.env.LYNKR_EMBEDDINGS_PROVIDER === 'onnx') {
208
+ const { generateOnnxEmbedding, isOnnxAvailable } = require('./onnx-embedder');
209
+ if (isOnnxAvailable()) {
210
+ return _wrapProvider('onnx', generateOnnxEmbedding);
211
+ }
212
+ logger.warn('[Embeddings] LYNKR_EMBEDDINGS_PROVIDER=onnx but @huggingface/transformers is not installed (optionalDependency) — falling through to the configured network provider');
213
+ }
214
+
133
215
  // Check if we have a local embedding provider configured
134
216
  if (config.ollama?.embeddingsEndpoint || provider === 'ollama') {
135
- return async (text) => {
136
- try {
137
- const result = await generateOllamaEmbedding(text);
138
- embeddingProviderAvailable = true;
139
- return result;
140
- } catch (err) {
141
- logger.debug({ error: err.message }, 'Ollama embedding failed, using hash fallback');
142
- embeddingProviderAvailable = false;
143
- return generateHashEmbedding(text);
144
- }
145
- };
217
+ return _wrapProvider('ollama', generateOllamaEmbedding);
146
218
  }
147
219
 
148
220
  if (config.llamacpp?.embeddingsEndpoint || provider === 'llamacpp') {
149
- return async (text) => {
150
- try {
151
- const result = await generateLlamaCppEmbedding(text);
152
- embeddingProviderAvailable = true;
153
- return result;
154
- } catch (err) {
155
- logger.debug({ error: err.message }, 'LlamaCpp embedding failed, using hash fallback');
156
- embeddingProviderAvailable = false;
157
- return generateHashEmbedding(text);
158
- }
159
- };
160
- }
161
-
162
- // Fallback to hash-based embeddings
163
- logger.debug('No embedding provider configured, using hash-based fallback');
221
+ return _wrapProvider('llamacpp', generateLlamaCppEmbedding);
222
+ }
223
+
224
+ // No provider configured at all — hash fallback is the deliberate mode,
225
+ // not a degradation. Warn once so the operator knows semantic matching is
226
+ // approximate, then stay quiet.
227
+ if (embeddingProviderAvailable === null) {
228
+ embeddingProviderAvailable = false;
229
+ logger.warn('[Embeddings] No embedding provider configured — using non-semantic hash embeddings. Semantic cache matches will be approximate; configure an Ollama/llama.cpp embeddings endpoint for real semantic matching.');
230
+ }
164
231
  return (text) => Promise.resolve(generateHashEmbedding(text));
165
232
  }
166
233
 
234
+ /**
235
+ * Current embedding subsystem status, for /metrics and health surfaces.
236
+ * @returns {{ providerAvailable: boolean|null, degradedSince: number|null,
237
+ * fallbackCount: number, lastProviderError: string|null }}
238
+ */
239
+ function getEmbeddingStatus() {
240
+ return {
241
+ providerAvailable: embeddingProviderAvailable,
242
+ degradedSince,
243
+ fallbackCount,
244
+ lastProviderError,
245
+ };
246
+ }
247
+
167
248
  /**
168
249
  * Generate embedding for text
169
250
  * @param {string} text - Text to embed
@@ -178,12 +259,16 @@ async function generateEmbedding(text) {
178
259
  const maxLength = 8000;
179
260
  const truncated = text.length > maxLength ? text.substring(0, maxLength) : text;
180
261
 
262
+ const embedFn = getEmbeddingFunction();
263
+ if (STRICT) return embedFn(truncated);
181
264
  try {
182
- const embedFn = getEmbeddingFunction();
183
265
  return await embedFn(truncated);
184
266
  } catch (err) {
185
- // Final fallback to hash embeddings if everything else fails
186
- logger.debug({ error: err.message }, 'Embedding generation failed, using hash fallback');
267
+ // Final fallback to hash embeddings if everything else fails. The
268
+ // provider wrapper already logged the degradation transition loudly;
269
+ // this catch only covers unexpected non-provider errors.
270
+ logger.warn({ error: err.message }, '[Embeddings] Embedding generation failed unexpectedly, serving hash fallback');
271
+ fallbackCount += 1;
187
272
  return generateHashEmbedding(truncated);
188
273
  }
189
274
  }
@@ -193,6 +278,10 @@ async function generateEmbedding(text) {
193
278
  */
194
279
  function resetEmbeddingProvider() {
195
280
  embeddingProviderAvailable = null;
281
+ degradedSince = null;
282
+ lastProviderAttempt = 0;
283
+ fallbackCount = 0;
284
+ lastProviderError = null;
196
285
  }
197
286
 
198
287
  /**
@@ -231,5 +320,6 @@ module.exports = {
231
320
  generateHashEmbedding,
232
321
  cosineSimilarity,
233
322
  getEmbeddingFunction,
323
+ getEmbeddingStatus,
234
324
  resetEmbeddingProvider,
235
325
  };