lynkr 9.14.2 → 9.14.7

package/src/routing/stuck-detector.js

@@ -0,0 +1,127 @@
+/**
+ * Stuck-loop / no-progress detection (ROUTING-NOTES §4.10.5).
+ *
+ * The escalation ladder reacts to score drift, context overflow, risk
+ * keywords, and vision needs — but nothing watches for "the pinned model is
+ * thrashing": re-issuing the same tool call with the same input over and
+ * over, or repeating the same assistant text verbatim. A cheap-tier model
+ * stuck in that state burns turns indefinitely, because every frame of the
+ * loop is a mid-tool-exchange pin serve (which is unconditional — tool-call
+ * IDs aren't portable across providers, so switching mid-exchange would 400).
+ *
+ * The safe intervention is the one the pin path already uses for embedded
+ * text triggers: don't switch this turn — DROP THE PIN, so the next turn
+ * boundary re-routes fresh (and full routing, seeing the whole struggling
+ * conversation, escalates on its own signals).
+ *
+ * Detection is deliberately narrow to keep false positives near zero:
+ *   - tool repetition: the last K assistant tool_use blocks are the SAME
+ *     tool with the SAME input, K >= LYNKR_STUCK_TOOL_REPEATS (default 3).
+ *     Agents legitimately re-run a tool (poll, retry-once); three identical
+ *     consecutive calls is a loop.
+ *   - text repetition: the last K assistant text blocks are identical after
+ *     whitespace normalization, K >= LYNKR_STUCK_TEXT_REPEATS (default 3).
+ *
+ * Only the tail of the conversation is scanned (SCAN_WINDOW messages), so
+ * the check is O(1)-ish per request regardless of conversation length.
+ *
+ * @module routing/stuck-detector
+ */
+
+const SCAN_WINDOW = 20;
+
+function _toolRepeats() {
+  const n = Number.parseInt(process.env.LYNKR_STUCK_TOOL_REPEATS, 10);
+  return Number.isNaN(n) ? 3 : Math.max(2, n);
+}
+
+function _textRepeats() {
+  const n = Number.parseInt(process.env.LYNKR_STUCK_TEXT_REPEATS, 10);
+  return Number.isNaN(n) ? 3 : Math.max(2, n);
+}
+
+function _enabled() {
+  return process.env.LYNKR_STUCK_DETECTOR_ENABLED !== 'false';
+}
+
+/**
+ * Extract, oldest→newest, the assistant tool_use signatures and text blocks
+ * from the tail of the conversation.
+ */
+function _assistantTail(messages) {
+  const tail = messages.slice(-SCAN_WINDOW);
+  const toolSigs = [];
+  const texts = [];
+  for (const msg of tail) {
+    if (msg?.role !== 'assistant') continue;
+    const content = Array.isArray(msg.content) ? msg.content : [];
+    for (const block of content) {
+      if (block?.type === 'tool_use') {
+        let inputSig;
+        try {
+          inputSig = JSON.stringify(block.input ?? null);
+        } catch {
+          inputSig = String(block.input);
+        }
+        toolSigs.push(`${block.name}::${inputSig}`);
+      } else if (block?.type === 'text' && typeof block.text === 'string') {
+        const normalized = block.text.replace(/\s+/g, ' ').trim();
+        if (normalized.length > 0) texts.push(normalized);
+      }
+    }
+    // String-content assistant messages count as text blocks too.
+    if (typeof msg.content === 'string') {
+      const normalized = msg.content.replace(/\s+/g, ' ').trim();
+      if (normalized.length > 0) texts.push(normalized);
+    }
+  }
+  return { toolSigs, texts };
+}
+
+function _trailingRun(items) {
+  if (items.length === 0) return 0;
+  const last = items[items.length - 1];
+  let run = 0;
+  for (let i = items.length - 1; i >= 0 && items[i] === last; i--) run++;
+  return run;
+}
+
+/**
+ * Detect a stuck loop in the conversation tail.
+ *
+ * @param {object} payload — request payload with .messages
+ * @returns {{ stuck: boolean, reason?: 'tool_repetition'|'text_repetition',
+ *             repeats?: number, signature?: string }}
+ */
+function detectStuckLoop(payload) {
+  if (!_enabled()) return { stuck: false };
+  const messages = payload?.messages;
+  if (!Array.isArray(messages) || messages.length < 4) return { stuck: false };
+
+  const { toolSigs, texts } = _assistantTail(messages);
+
+  const toolRun = _trailingRun(toolSigs);
+  if (toolRun >= _toolRepeats()) {
+    return {
+      stuck: true,
+      reason: 'tool_repetition',
+      repeats: toolRun,
+      // Truncated for logging — the full input may be huge or sensitive.
+      signature: toolSigs[toolSigs.length - 1].slice(0, 120),
+    };
+  }
+
+  const textRun = _trailingRun(texts);
+  if (textRun >= _textRepeats()) {
+    return {
+      stuck: true,
+      reason: 'text_repetition',
+      repeats: textRun,
+      signature: texts[texts.length - 1].slice(0, 120),
+    };
+  }
+
+  return { stuck: false };
+}
+
+module.exports = { detectStuckLoop };

package/src/routing/telemetry.js

@@ -175,6 +175,12 @@ function init() {
       // providers that report none) — "not measured" is distinct from 0.
       ["cache_read_tokens", "INTEGER"],
       ["cache_creation_tokens", "INTEGER"],
+      // Off-policy evaluation — the bandit's context vector (12-dim JSON
+      // array) for the row's decision. Required by the doubly-robust
+      // estimator's regression term r̂(x, a); IPS/SNIPS work without it.
+      // NULL on rows where the bandit didn't run (deterministic decisions)
+      // or recorded before this column existed.
+      ["context", "TEXT"],
     ];
     for (const [col, type] of additiveCols) {
       if (!existingCols.has(col)) {
@@ -237,7 +243,7 @@ function record(data) {
           retry_count, circuit_breaker_state, quality_score, tokens_per_second,
           cost_efficiency, request_text, response_text,
           base_tier, escalation_source, propensity, candidates, pinned, switch_reason,
-          cache_decision, cache_read_tokens, cache_creation_tokens
+          cache_decision, cache_read_tokens, cache_creation_tokens, context
         ) VALUES (
           @request_id, @session_id, @timestamp, @complexity_score, @tier,
           @agentic_type, @tool_count, @input_tokens, @message_count, @request_type,
@@ -246,7 +252,7 @@ function record(data) {
           @retry_count, @circuit_breaker_state, @quality_score, @tokens_per_second,
           @cost_efficiency, @request_text, @response_text,
           @base_tier, @escalation_source, @propensity, @candidates, @pinned, @switch_reason,
-          @cache_decision, @cache_read_tokens, @cache_creation_tokens
+          @cache_decision, @cache_read_tokens, @cache_creation_tokens, @context
         )`
       );
       if (!insert) return;
@@ -299,6 +305,9 @@ function record(data) {
             : JSON.stringify(data.cache_decision)),
         cache_read_tokens: data.cache_read_tokens ?? null,
         cache_creation_tokens: data.cache_creation_tokens ?? null,
+        context: data.context == null
+          ? null
+          : (typeof data.context === "string" ? data.context : JSON.stringify(data.context)),
       });
     } catch (err) {
       logger.debug({ err: err.message }, "Telemetry record failed");

package/src/server.js

@@ -80,7 +80,11 @@ function createApp() {
 
   // Phase 6.1 — per-tenant routing policies (LYNKR-Tenant-Id header).
   // Runs before message handling so res.locals.tenantPolicy is populated.
-  app.use('/v1/messages', tenantMiddleware);
+  // Must cover every AGENT_ENDPOINT, not just /v1/messages — mounting on a
+  // single path silently exempts OpenAI-compat clients from tenant policy
+  // (the same failure mode the comment above already warns about for other
+  // guards; see issue #100).
+  for (const p of AGENT_ENDPOINTS) app.use(p, tenantMiddleware);
 
   // Phase 6.2 — hierarchical budget enforcement (LYNKR_BUDGET_ENFORCER=false to disable).
   for (const p of AGENT_ENDPOINTS) app.use(p, budgetEnforcer);
@@ -105,7 +109,8 @@ function createApp() {
 
   app.get("/metrics/circuit-breakers", (req, res) => {
     const registry = getCircuitBreakerRegistry();
-    res.json(registry.getAll());
+    const { getHealthProber } = require("./clients/health-probe");
+    res.json({ breakers: registry.getAll(), healthProbe: getHealthProber().getStatus() });
   });
 
   app.get("/metrics/load-shedding", (req, res) => {
@@ -124,13 +129,23 @@ function createApp() {
 
   app.get("/metrics/semantic-cache", (req, res) => {
     const { getSemanticCache, isSemanticCacheEnabled } = require("./cache/semantic");
+    const { getEmbeddingStatus } = require("./cache/embeddings");
     if (!isSemanticCacheEnabled()) {
       return res.json({ enabled: false, message: "Semantic cache not enabled" });
     }
     const cache = getSemanticCache();
-    res.json({ enabled: true, ...cache.getStats() });
+    // embeddings.providerAvailable === false means matches are currently
+    // served from the non-semantic hash fallback — cache "hits" during a
+    // degraded window are approximate, not semantic.
+    res.json({ enabled: true, embeddings: getEmbeddingStatus(), ...cache.getStats() });
   });
 
+  // MCP broker — exposes configured MCP servers to other clients over HTTP.
+  // Off by default; requires LYNKR_MCP_BROKER_ENABLED + a bearer token.
+  // Mounted BEFORE the main router so /v1/mcp/* never falls through to the
+  // OpenAI-compat surface.
+  app.use(require('./api/mcp-broker'));
+
   app.use(router);
 
   app.use('/dashboard', require('./dashboard/router'));
@@ -181,6 +196,29 @@ async function start() {
 
   const app = createApp();
 
+  // Synthetic circuit-breaker health probing — recovers open circuits with a
+  // cheap background probe instead of letting the next live user request pay
+  // for testing a dead provider. Zero-cost while all breakers are closed.
+  try {
+    const { getHealthProber } = require("./clients/health-probe");
+    const prober = getHealthProber();
+    prober.start();
+    getShutdownManager().onShutdown(() => prober.stop());
+  } catch (err) {
+    logger.warn({ err: err.message }, "Health prober failed to start, circuit recovery falls back to live-request probing");
+  }
+
+  // OTel GenAI metrics export — no-op unless OTEL_EXPORTER_OTLP_ENDPOINT
+  // (or LYNKR_OTEL_ENDPOINT) is set. Zero-dependency OTLP/HTTP push.
+  try {
+    const { getOtelExporter } = require("./observability/otel");
+    const otel = getOtelExporter();
+    otel.start();
+    getShutdownManager().onShutdown(() => otel.stop());
+  } catch (err) {
+    logger.warn({ err: err.message }, "OTel exporter failed to start");
+  }
+
   // Wait for Ollama if it's the configured provider or referenced in tier config
   const provider = config.modelProvider?.type?.toLowerCase();
   if (provider === "ollama" || config.tiersReferenceOllama()) {

package/src/tools/web-search-exec.js

@@ -0,0 +1,182 @@
+/**
+ * Server-side web_search / web_fetch execution — narrowly scoped resurrection.
+ *
+ * Context: src/tools/web.js used to implement this (SearXNG-backed search,
+ * host-allowlisted fetch) and was deleted 2026-07-22 in commit b32e988,
+ * "Remove server-mode tool execution: Lynkr always forwards tool calls to
+ * the client." That was a deliberate architecture pivot, not a bug — Lynkr
+ * stopped executing ANY tool itself.
+ *
+ * This does NOT reintroduce general server-mode tool execution. It only
+ * auto-resolves `web_search`/`web_fetch` calls, and ONLY when the calling
+ * client is unrecognized by src/routing/client-profiles.js's detectClient()
+ * — i.e. Lynkr has no signal the caller can fulfill the call itself.
+ * Known harnesses (Claude Code / Claude Desktop's shared agent-sdk — both
+ * present as `claude-cli/...`, Cursor, goose, Codex) already execute these
+ * client-side today and are never routed through this module; see the call
+ * site in src/orchestrator/index.js's runAgentLoop.
+ *
+ * @module tools/web-search-exec
+ */
+
+const config = require("../config");
+const logger = require("../logger");
+
+const AUTO_RESOLVABLE_NAMES = new Set(["web_search", "websearch", "web_fetch", "webfetch"]);
+
+/**
+ * @param {string} name
+ * @returns {boolean}
+ */
+function isAutoResolvable(name) {
+  return AUTO_RESOLVABLE_NAMES.has(String(name || "").toLowerCase());
+}
+
+/**
+ * True only when every call in this batch is one we can resolve ourselves —
+ * a mixed batch (one auto-resolvable + one arbitrary client tool) falls
+ * through to the normal "forward to client" path untouched, since we can't
+ * partially resolve a batch the model expects answered together.
+ *
+ * @param {Array} toolCalls - normalized {id, function:{name}} or {name} calls
+ * @returns {boolean}
+ */
+function canAutoResolveAll(toolCalls) {
+  return Array.isArray(toolCalls) && toolCalls.length > 0
+    && toolCalls.every((tc) => isAutoResolvable(tc?.function?.name ?? tc?.name));
+}
+
+function isHostAllowed(urlStr) {
+  if (config.webSearch?.allowAllHosts) return true;
+  try {
+    const host = new URL(urlStr).hostname.toLowerCase();
+    const allowed = config.webSearch?.allowedHosts;
+    return Array.isArray(allowed) && allowed.includes(host);
+  } catch {
+    return false;
+  }
+}
+
+async function _withTimeout(fn, timeoutMs) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    return await fn(controller.signal);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * SearXNG-backed search. Retries transient failures per config.webSearch
+ * (retryEnabled/maxRetries) — mirrors the retry policy the deleted web.js
+ * had, since a single dropped connection to a local sidecar shouldn't fail
+ * the whole tool call.
+ *
+ * @param {string} query
+ * @returns {Promise<Object>} { query, results: [{title,url,snippet}] } or { error }
+ */
+async function searchWeb(query) {
+  const endpoint = config.webSearch?.endpoint || "http://localhost:8888/search";
+  const timeoutMs = config.webSearch?.timeoutMs || 10000;
+  const maxRetries = config.webSearch?.retryEnabled ? (config.webSearch?.maxRetries ?? 2) : 0;
+  const url = `${endpoint}?q=${encodeURIComponent(query || "")}&format=json`;
+
+  let lastErr = null;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const res = await _withTimeout(
+        (signal) => fetch(url, { signal }),
+        timeoutMs,
+      );
+      if (!res.ok) {
+        lastErr = `search backend returned HTTP ${res.status}`;
+        continue;
+      }
+      const json = await res.json();
+      const results = (Array.isArray(json.results) ? json.results : [])
+        .slice(0, 8)
+        .map((r) => ({ title: r.title, url: r.url, snippet: r.content }));
+      return { query, results };
+    } catch (err) {
+      lastErr = err.message;
+    }
+  }
+  logger.warn({ query, endpoint, err: lastErr }, "[web-search-exec] search failed after retries");
+  return { query, error: `search failed: ${lastErr}` };
+}
+
+/**
+ * Generic URL fetch, host-allowlisted per config.webSearch (allowAllHosts /
+ * allowedHosts), truncated per config.webSearch.bodyPreviewMax.
+ *
+ * @param {string} targetUrl
+ * @returns {Promise<Object>}
+ */
+async function fetchUrl(targetUrl) {
+  if (!targetUrl) return { error: "no url provided" };
+  if (!isHostAllowed(targetUrl)) {
+    logger.warn({ url: targetUrl }, "[web-search-exec] fetch blocked — host not allowed");
+    return { url: targetUrl, error: "host not in allowedHosts (WEB_SEARCH_ALLOW_ALL=false)" };
+  }
+  const maxPreview = config.webSearch?.bodyPreviewMax || 10000;
+  const timeoutMs = config.webSearch?.timeoutMs || 10000;
+  try {
+    const res = await _withTimeout(
+      (signal) => fetch(targetUrl, { signal }),
+      timeoutMs,
+    );
+    const text = await res.text();
+    return {
+      url: targetUrl,
+      status: res.status,
+      content: text.slice(0, maxPreview),
+      truncated: text.length > maxPreview,
+    };
+  } catch (err) {
+    logger.warn({ url: targetUrl, err: err.message }, "[web-search-exec] fetch failed");
+    return { url: targetUrl, error: `fetch failed: ${err.message}` };
+  }
+}
+
+/**
+ * Execute every call in this (pre-checked via canAutoResolveAll) batch and
+ * append the matching assistant tool_use + user tool_result turns to
+ * `messages`, mirroring exactly what a real client round-trip would send
+ * back — so the next agent-loop iteration behaves identically to a normal
+ * client-fulfilled tool exchange, and the client never sees the exchange.
+ *
+ * @param {Array} toolCalls
+ * @param {Array} messages - mutated in place (push only, never rewritten)
+ */
+async function autoResolve(toolCalls, messages) {
+  const toolUseBlocks = [];
+  const toolResultBlocks = [];
+
+  for (const tc of toolCalls) {
+    const rawName = tc.function?.name ?? tc.name ?? "";
+    const name = String(rawName).toLowerCase();
+    let input = {};
+    try {
+      const raw = tc.function?.arguments ?? tc.input ?? {};
+      input = typeof raw === "string" ? JSON.parse(raw) : (raw || {});
+    } catch { /* leave input empty — result will just reflect an empty query/url */ }
+
+    toolUseBlocks.push({ type: "tool_use", id: tc.id, name: rawName, input });
+
+    const result = (name === "web_search" || name === "websearch")
+      ? await searchWeb(input.query || input.q || "")
+      : await fetchUrl(input.url || "");
+
+    toolResultBlocks.push({
+      type: "tool_result",
+      tool_use_id: tc.id,
+      content: JSON.stringify(result),
+    });
+  }
+
+  messages.push({ role: "assistant", content: toolUseBlocks });
+  messages.push({ role: "user", content: toolResultBlocks });
+}
+
+module.exports = { canAutoResolveAll, autoResolve, searchWeb, fetchUrl, isAutoResolvable };