lynkr 9.14.1 → 9.14.3

package/src/routing/shortfall.js

@@ -0,0 +1,290 @@
+/**
+ * Shortfall matching (item 1: HyDRA port, phase 1).
+ *
+ * Selects the cheapest candidate whose capabilities cover the predicted
+ * requirement vector within tolerance τ:
+ *
+ *   shortfall(m) = Σ w_k · max(0, req_k − cap_mk)
+ *
+ * All four heads share the [0,1] bandwidth, so v1 weights are uniform
+ * (band compensation is a no-op until heads diverge — noted for phase 2).
+ *
+ * Contracts:
+ *   - Catalog-decoupled: candidates arrive as [{provider, model, tier}];
+ *     capabilities resolve per MODEL FAMILY (provider never matters:
+ *     zai/baidu/ollama servings of glm-5.2 share caps) via operator
+ *     modelOverrides → seed snapshot/shipped → family heuristic → tier
+ *     profile. Adding, removing, or repricing a model re-routes via config
+ *     with zero retraining.
+ *   - Self-contained config: enabled/tau/weights also live in
+ *     config/model-capabilities.json — no env vars. The file is read once at
+ *     boot (same lifecycle as config/model-tiers.json); restart to pick up
+ *     edits.
+ *   - Cost comes from the caller (model-registry) or per-1k blended estimate;
+ *     unknown cost sorts last (conservative — never wins on cheapness alone).
+ *   - No covering candidate (all shortfalls > τ) → minimal shortfall wins,
+ *     tiebreak higher tier (correctness over cost, matches tier-fallback.js
+ *     escalate-then-demote bias). Never returns null on valid input; returns
+ *     null only on malformed input so callers fail open to legacy routing.
+ *   - Pure except for config load (cached). Never throws.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const logger = require('../logger');
+const { HEADS } = require('./capabilities');
+
+const PROFILES_PATH = path.join(__dirname, '../../config/model-capabilities.json');
+
+const DEFAULT_TAU = 0.24; // HyDRA iso-quality operating point
+const TIER_PRIORITY = { SIMPLE: 1, MEDIUM: 2, COMPLEX: 3, REASONING: 4 };
+
+let _profilesCache = null;
+
+function _defaultWeights() {
+  const w = {};
+  for (const h of HEADS) w[h] = 1 / HEADS.length;
+  return w;
+}
+
+function _sanitizeCaps(raw) {
+  const caps = {};
+  for (const h of HEADS) {
+    const v = Number(raw?.[h]);
+    caps[h] = Number.isFinite(v) ? Math.max(0, Math.min(1, v)) : 0.5;
+  }
+  return caps;
+}
+
+function loadProfiles() {
+  if (_profilesCache) return _profilesCache;
+  try {
+    const raw = JSON.parse(fs.readFileSync(PROFILES_PATH, 'utf8'));
+    const tierProfiles = {};
+    const fileTiers = raw?.tierProfiles && typeof raw.tierProfiles === 'object'
+      ? raw.tierProfiles
+      : {};
+    for (const [tier, caps] of Object.entries(fileTiers)) {
+      if (caps && typeof caps === 'object') tierProfiles[tier] = _sanitizeCaps(caps);
+    }
+    const modelOverrides = {};
+    const fileOverrides = raw?.modelOverrides && typeof raw.modelOverrides === 'object'
+      ? raw.modelOverrides
+      : {};
+    for (const [key, caps] of Object.entries(fileOverrides)) {
+      if (caps && typeof caps === 'object') modelOverrides[String(key).toLowerCase()] = _sanitizeCaps(caps);
+    }
+    const tau = Number(raw?.tau);
+    const weights = _normalizeWeights(raw?.weights);
+    _profilesCache = {
+      tierProfiles,
+      modelOverrides,
+      enabled: raw?.enabled === true,
+      tau: Number.isFinite(tau) && tau >= 0 ? tau : DEFAULT_TAU,
+      weights,
+    };
+  } catch (err) {
+    logger.debug({ err: err.message }, '[Shortfall] profiles load failed — disabled with tier fallbacks');
+    _profilesCache = { tierProfiles: {}, modelOverrides: {}, enabled: false, tau: DEFAULT_TAU, weights: _defaultWeights() };
+  }
+  return _profilesCache;
+}
+
+// Exposed for tests (cache reset / profile injection).
+function _resetProfilesCache() {
+  _profilesCache = null;
+}
+
+function _setProfilesForTests(profiles) {
+  const base = loadProfiles();
+  _profilesCache = {
+    tierProfiles: profiles?.tierProfiles ?? base.tierProfiles,
+    modelOverrides: profiles?.modelOverrides ?? base.modelOverrides,
+    enabled: profiles?.enabled ?? base.enabled,
+    tau: profiles?.tau ?? base.tau,
+    weights: profiles?.weights ?? base.weights,
+  };
+  return _profilesCache;
+}
+
+function _normalizeWeights(raw) {
+  const w = _defaultWeights();
+  if (!raw || typeof raw !== 'object') return w;
+  let touched = false;
+  for (const h of HEADS) {
+    const v = Number(raw[h]);
+    if (Number.isFinite(v) && v >= 0) {
+      w[h] = v;
+      touched = true;
+    }
+  }
+  if (!touched) return _defaultWeights();
+  const sum = Object.values(w).reduce((a, b) => a + b, 0);
+  if (sum <= 0) return _defaultWeights();
+  for (const h of HEADS) w[h] = w[h] / sum;
+  return w;
+}
+
+function getTau() {
+  return loadProfiles().tau;
+}
+
+function getWeights() {
+  return { ...loadProfiles().weights };
+}
+
+function isEnabled() {
+  return loadProfiles().enabled === true;
+}
+
+/**
+ * Resolve capabilities for one candidate. Precedence (first hit wins):
+ *   1. operator "provider:model" / "provider:*" override (manual, supreme,
+ *      returned verbatim — not even the quant haircut applies)
+ *   2. seed snapshot / shipped seeds by normalized MODEL FAMILY
+ *      (zai/baidu/ollama servings of glm-5.2 share one entry)
+ *   3. family-ladder heuristic (zero-network estimate, never frontier)
+ *   4. tier profile → tier-priority fallback (SIMPLE 0.2 … REASONING 0.9)
+ *
+ * Quantized self-hosted servings (Q4_K_M, gguf, …) take a small haircut at
+ * every level except 1: same brain, smaller body.
+ *
+ * @returns {{ caps, source }} — source is override|seed:snapshot|
+ *   seed:shipped|family|tier|tier-fallback (telemetry provenance).
+ */
+function resolveCapabilitiesWithSource({ provider, model, tier }) {
+  const { tierProfiles, modelOverrides } = loadProfiles();
+  const key = `${String(provider || '').toLowerCase()}:${String(model || '').toLowerCase()}`;
+  const wild = `${String(provider || '').toLowerCase()}:*`;
+  const pick = modelOverrides[key] || modelOverrides[wild];
+  if (pick) return { caps: { ...pick }, source: 'override' };
+
+  let family = 'unknown';
+  let quant = false;
+  try {
+    const fam = require('./capability-seeds/family');
+    ({ family, quant } = fam.normalizeFamily(provider, model));
+  } catch { /* family helpers unavailable — tier fallback below */ }
+
+  try {
+    const { resolveSeedCaps } = require('./capability-seeds/registry');
+    const seed = resolveSeedCaps(family);
+    if (seed) {
+      const caps = quant
+        ? require('./capability-seeds/family').applyQuantHaircut(seed.caps)
+        : { ...seed.caps };
+      return { caps, source: seed.source };
+    }
+  } catch { /* seed layers unavailable — keep falling through */ }
+
+  try {
+    const { heuristicCaps } = require('./capability-seeds/family-heuristics');
+    const heur = heuristicCaps(family);
+    if (heur) {
+      const caps = quant
+        ? require('./capability-seeds/family').applyQuantHaircut(heur)
+        : heur;
+      return { caps, source: 'family' };
+    }
+  } catch { /* heuristic unavailable — tier fallback below */ }
+
+  const tp = tierProfiles[tier];
+  if (tp) {
+    const caps = quant
+      ? require('./capability-seeds/family').applyQuantHaircut(tp)
+      : { ...tp };
+    return { caps, source: 'tier' };
+  }
+  const p = (TIER_PRIORITY[tier] || 1) / 4;
+  const caps = {};
+  for (const h of HEADS) caps[h] = Math.round(p * 1000) / 1000;
+  return { caps, source: 'tier-fallback' };
+}
+
+function resolveCapabilities(candidate) {
+  return resolveCapabilitiesWithSource(candidate).caps;
+}
+
+function shortfall(req, caps, weights = null) {
+  const w = weights || _defaultWeights();
+  let s = 0;
+  for (const h of HEADS) {
+    const r = Math.max(0, Math.min(1, Number(req?.[h]) || 0));
+    const c = Math.max(0, Math.min(1, Number(caps?.[h]) || 0));
+    s += (w[h] ?? 0) * Math.max(0, r - c);
+  }
+  return Math.round(s * 10000) / 10000;
+}
+
+function _costValue(c) {
+  const n = Number(c);
+  return Number.isFinite(n) && n >= 0 ? n : Number.POSITIVE_INFINITY;
+}
+
+/**
+ * @param {object} req — requirement vector {reasoning, codegen, debugging, tool_use} in [0,1]
+ * @param {Array<{provider, model, tier, cost?}>} candidates — catalog-constrained set
+ *   (callers pass getAllConfiguredModels() + model-registry costs)
+ * @param {object} [opts] — { tau, weights }
+ * @returns {null | { selected, shortfalls: Array<{provider, model, tier, shortfall, cost, source}>, tau }}
+ */
+function selectByShortfall(req, candidates, opts = {}) {
+  try {
+    if (!req || typeof req !== 'object') return null;
+    if (!Array.isArray(candidates) || candidates.length === 0) return null;
+    const tau = opts.tau ?? getTau();
+    const weights = opts.weights ?? getWeights();
+
+    const rows = candidates
+      .filter((c) => c && c.provider && c.model)
+      .map((c) => {
+        const { caps, source } = resolveCapabilitiesWithSource(c);
+        return {
+          provider: c.provider,
+          model: c.model,
+          tier: c.tier || 'MEDIUM',
+          cost: _costValue(c.cost),
+          shortfall: shortfall(req, caps, weights),
+          source,
+        };
+      });
+    if (rows.length === 0) return null;
+
+    const covering = rows.filter((r) => r.shortfall <= tau);
+    const pool = covering.length > 0 ? covering : rows;
+    pool.sort((a, b) => {
+      if (covering.length > 0) {
+        // Cheapest covering wins; cost tie (incl. all-unknown) breaks toward
+        // the LOWER tier — a covering lower tier is sufficient by definition,
+        // so prefer it over excess headroom (avoids over-provisioning).
+        if (a.cost !== b.cost) return a.cost - b.cost;
+        return (TIER_PRIORITY[a.tier] || 0) - (TIER_PRIORITY[b.tier] || 0);
+      }
+      // Nothing covers: minimal shortfall wins, tiebreak higher tier then cheaper.
+      if (a.shortfall !== b.shortfall) return a.shortfall - b.shortfall;
+      const tp = (TIER_PRIORITY[b.tier] || 0) - (TIER_PRIORITY[a.tier] || 0);
+      if (tp !== 0) return tp;
+      return a.cost - b.cost;
+    });
+
+    return { selected: pool[0], shortfalls: rows, tau };
+  } catch (err) {
+    logger.debug({ err: err.message }, '[Shortfall] select failed — failing open');
+    return null;
+  }
+}
+
+module.exports = {
+  HEADS,
+  DEFAULT_TAU,
+  loadProfiles,
+  _resetProfilesCache,
+  _setProfilesForTests,
+  getTau,
+  getWeights,
+  isEnabled,
+  resolveCapabilities,
+  resolveCapabilitiesWithSource,
+  shortfall,
+  selectByShortfall,
+};

package/src/routing/stuck-detector.js

@@ -0,0 +1,127 @@
+/**
+ * Stuck-loop / no-progress detection (ROUTING-NOTES §4.10.5).
+ *
+ * The escalation ladder reacts to score drift, context overflow, risk
+ * keywords, and vision needs — but nothing watches for "the pinned model is
+ * thrashing": re-issuing the same tool call with the same input over and
+ * over, or repeating the same assistant text verbatim. A cheap-tier model
+ * stuck in that state burns turns indefinitely, because every frame of the
+ * loop is a mid-tool-exchange pin serve (which is unconditional — tool-call
+ * IDs aren't portable across providers, so switching mid-exchange would 400).
+ *
+ * The safe intervention is the one the pin path already uses for embedded
+ * text triggers: don't switch this turn — DROP THE PIN, so the next turn
+ * boundary re-routes fresh (and full routing, seeing the whole struggling
+ * conversation, escalates on its own signals).
+ *
+ * Detection is deliberately narrow to keep false positives near zero:
+ *   - tool repetition: the last K assistant tool_use blocks are the SAME
+ *     tool with the SAME input, K >= LYNKR_STUCK_TOOL_REPEATS (default 3).
+ *     Agents legitimately re-run a tool (poll, retry-once); three identical
+ *     consecutive calls is a loop.
+ *   - text repetition: the last K assistant text blocks are identical after
+ *     whitespace normalization, K >= LYNKR_STUCK_TEXT_REPEATS (default 3).
+ *
+ * Only the tail of the conversation is scanned (SCAN_WINDOW messages), so
+ * the check is O(1)-ish per request regardless of conversation length.
+ *
+ * @module routing/stuck-detector
+ */
+
+const SCAN_WINDOW = 20;
+
+function _toolRepeats() {
+  const n = Number.parseInt(process.env.LYNKR_STUCK_TOOL_REPEATS, 10);
+  return Number.isNaN(n) ? 3 : Math.max(2, n);
+}
+
+function _textRepeats() {
+  const n = Number.parseInt(process.env.LYNKR_STUCK_TEXT_REPEATS, 10);
+  return Number.isNaN(n) ? 3 : Math.max(2, n);
+}
+
+function _enabled() {
+  return process.env.LYNKR_STUCK_DETECTOR_ENABLED !== 'false';
+}
+
+/**
+ * Extract, oldest→newest, the assistant tool_use signatures and text blocks
+ * from the tail of the conversation.
+ */
+function _assistantTail(messages) {
+  const tail = messages.slice(-SCAN_WINDOW);
+  const toolSigs = [];
+  const texts = [];
+  for (const msg of tail) {
+    if (msg?.role !== 'assistant') continue;
+    const content = Array.isArray(msg.content) ? msg.content : [];
+    for (const block of content) {
+      if (block?.type === 'tool_use') {
+        let inputSig;
+        try {
+          inputSig = JSON.stringify(block.input ?? null);
+        } catch {
+          inputSig = String(block.input);
+        }
+        toolSigs.push(`${block.name}::${inputSig}`);
+      } else if (block?.type === 'text' && typeof block.text === 'string') {
+        const normalized = block.text.replace(/\s+/g, ' ').trim();
+        if (normalized.length > 0) texts.push(normalized);
+      }
+    }
+    // String-content assistant messages count as text blocks too.
+    if (typeof msg.content === 'string') {
+      const normalized = msg.content.replace(/\s+/g, ' ').trim();
+      if (normalized.length > 0) texts.push(normalized);
+    }
+  }
+  return { toolSigs, texts };
+}
+
+function _trailingRun(items) {
+  if (items.length === 0) return 0;
+  const last = items[items.length - 1];
+  let run = 0;
+  for (let i = items.length - 1; i >= 0 && items[i] === last; i--) run++;
+  return run;
+}
+
+/**
+ * Detect a stuck loop in the conversation tail.
+ *
+ * @param {object} payload — request payload with .messages
+ * @returns {{ stuck: boolean, reason?: 'tool_repetition'|'text_repetition',
+ *             repeats?: number, signature?: string }}
+ */
+function detectStuckLoop(payload) {
+  if (!_enabled()) return { stuck: false };
+  const messages = payload?.messages;
+  if (!Array.isArray(messages) || messages.length < 4) return { stuck: false };
+
+  const { toolSigs, texts } = _assistantTail(messages);
+
+  const toolRun = _trailingRun(toolSigs);
+  if (toolRun >= _toolRepeats()) {
+    return {
+      stuck: true,
+      reason: 'tool_repetition',
+      repeats: toolRun,
+      // Truncated for logging — the full input may be huge or sensitive.
+      signature: toolSigs[toolSigs.length - 1].slice(0, 120),
+    };
+  }
+
+  const textRun = _trailingRun(texts);
+  if (textRun >= _textRepeats()) {
+    return {
+      stuck: true,
+      reason: 'text_repetition',
+      repeats: textRun,
+      signature: texts[texts.length - 1].slice(0, 120),
+    };
+  }
+
+  return { stuck: false };
+}
+
+module.exports = { detectStuckLoop };

package/src/routing/telemetry.js

@@ -175,6 +175,12 @@ function init() {
       // providers that report none) — "not measured" is distinct from 0.
       ["cache_read_tokens", "INTEGER"],
       ["cache_creation_tokens", "INTEGER"],
+      // Off-policy evaluation — the bandit's context vector (12-dim JSON
+      // array) for the row's decision. Required by the doubly-robust
+      // estimator's regression term r̂(x, a); IPS/SNIPS work without it.
+      // NULL on rows where the bandit didn't run (deterministic decisions)
+      // or recorded before this column existed.
+      ["context", "TEXT"],
     ];
     for (const [col, type] of additiveCols) {
       if (!existingCols.has(col)) {
@@ -237,7 +243,7 @@ function record(data) {
           retry_count, circuit_breaker_state, quality_score, tokens_per_second,
           cost_efficiency, request_text, response_text,
           base_tier, escalation_source, propensity, candidates, pinned, switch_reason,
-          cache_decision, cache_read_tokens, cache_creation_tokens
+          cache_decision, cache_read_tokens, cache_creation_tokens, context
         ) VALUES (
           @request_id, @session_id, @timestamp, @complexity_score, @tier,
           @agentic_type, @tool_count, @input_tokens, @message_count, @request_type,
@@ -246,7 +252,7 @@ function record(data) {
           @retry_count, @circuit_breaker_state, @quality_score, @tokens_per_second,
           @cost_efficiency, @request_text, @response_text,
           @base_tier, @escalation_source, @propensity, @candidates, @pinned, @switch_reason,
-          @cache_decision, @cache_read_tokens, @cache_creation_tokens
+          @cache_decision, @cache_read_tokens, @cache_creation_tokens, @context
         )`
       );
       if (!insert) return;
@@ -299,6 +305,9 @@ function record(data) {
             : JSON.stringify(data.cache_decision)),
         cache_read_tokens: data.cache_read_tokens ?? null,
         cache_creation_tokens: data.cache_creation_tokens ?? null,
+        context: data.context == null
+          ? null
+          : (typeof data.context === "string" ? data.context : JSON.stringify(data.context)),
       });
     } catch (err) {
       logger.debug({ err: err.message }, "Telemetry record failed");

package/src/server.js

@@ -80,7 +80,11 @@ function createApp() {
 
   // Phase 6.1 — per-tenant routing policies (LYNKR-Tenant-Id header).
   // Runs before message handling so res.locals.tenantPolicy is populated.
-  app.use('/v1/messages', tenantMiddleware);
+  // Must cover every AGENT_ENDPOINT, not just /v1/messages — mounting on a
+  // single path silently exempts OpenAI-compat clients from tenant policy
+  // (the same failure mode the comment above already warns about for other
+  // guards; see issue #100).
+  for (const p of AGENT_ENDPOINTS) app.use(p, tenantMiddleware);
 
   // Phase 6.2 — hierarchical budget enforcement (LYNKR_BUDGET_ENFORCER=false to disable).
   for (const p of AGENT_ENDPOINTS) app.use(p, budgetEnforcer);
@@ -105,7 +109,8 @@ function createApp() {
 
   app.get("/metrics/circuit-breakers", (req, res) => {
     const registry = getCircuitBreakerRegistry();
-    res.json(registry.getAll());
+    const { getHealthProber } = require("./clients/health-probe");
+    res.json({ breakers: registry.getAll(), healthProbe: getHealthProber().getStatus() });
   });
 
   app.get("/metrics/load-shedding", (req, res) => {
@@ -124,13 +129,23 @@ function createApp() {
 
   app.get("/metrics/semantic-cache", (req, res) => {
     const { getSemanticCache, isSemanticCacheEnabled } = require("./cache/semantic");
+    const { getEmbeddingStatus } = require("./cache/embeddings");
     if (!isSemanticCacheEnabled()) {
       return res.json({ enabled: false, message: "Semantic cache not enabled" });
     }
     const cache = getSemanticCache();
-    res.json({ enabled: true, ...cache.getStats() });
+    // embeddings.providerAvailable === false means matches are currently
+    // served from the non-semantic hash fallback — cache "hits" during a
+    // degraded window are approximate, not semantic.
+    res.json({ enabled: true, embeddings: getEmbeddingStatus(), ...cache.getStats() });
   });
 
+  // MCP broker — exposes configured MCP servers to other clients over HTTP.
+  // Off by default; requires LYNKR_MCP_BROKER_ENABLED + a bearer token.
+  // Mounted BEFORE the main router so /v1/mcp/* never falls through to the
+  // OpenAI-compat surface.
+  app.use(require('./api/mcp-broker'));
+
   app.use(router);
 
   app.use('/dashboard', require('./dashboard/router'));
@@ -181,6 +196,29 @@ async function start() {
 
   const app = createApp();
 
+  // Synthetic circuit-breaker health probing — recovers open circuits with a
+  // cheap background probe instead of letting the next live user request pay
+  // for testing a dead provider. Zero-cost while all breakers are closed.
+  try {
+    const { getHealthProber } = require("./clients/health-probe");
+    const prober = getHealthProber();
+    prober.start();
+    getShutdownManager().onShutdown(() => prober.stop());
+  } catch (err) {
+    logger.warn({ err: err.message }, "Health prober failed to start, circuit recovery falls back to live-request probing");
+  }
+
+  // OTel GenAI metrics export — no-op unless OTEL_EXPORTER_OTLP_ENDPOINT
+  // (or LYNKR_OTEL_ENDPOINT) is set. Zero-dependency OTLP/HTTP push.
+  try {
+    const { getOtelExporter } = require("./observability/otel");
+    const otel = getOtelExporter();
+    otel.start();
+    getShutdownManager().onShutdown(() => otel.stop());
+  } catch (err) {
+    logger.warn({ err: err.message }, "OTel exporter failed to start");
+  }
+
   // Wait for Ollama if it's the configured provider or referenced in tier config
   const provider = config.modelProvider?.type?.toLowerCase();
   if (provider === "ollama" || config.tiersReferenceOllama()) {

package/src/stores/file-store.js

@@ -89,7 +89,11 @@ async function getFileContent(id) {
 async function deleteFile(id) {
   const entry = metadata.get(id);
   if (!entry) return false;
-  try { await fsp.unlink(entry.storage_path); } catch {}
+  try {
+    await fsp.unlink(entry.storage_path);
+  } catch (err) {
+    logger.debug({ err: err.message, fileId: id }, "Best-effort unlink failed during file delete");
+  }
   metadata.delete(id);
   persistMetadata();
   return true;