lynkr 9.14.2 → 9.14.7

package/src/clients/health-probe.js

@@ -0,0 +1,157 @@
+/**
+ * Synthetic circuit-breaker health probe (ROUTING-NOTES §1 gap audit).
+ *
+ * Problem: cockatiel's halfOpenAfter is a pure timer — after it elapses, the
+ * NEXT LIVE USER REQUEST is the recovery probe. A user pays the latency/cost
+ * of testing a dead provider, and a provider with no traffic never recovers.
+ *
+ * Fix (the pattern LiteLLM ships as health-check-driven routing): a
+ * background loop that, for every breaker currently OPEN or HALF_OPEN, runs
+ * a cheap synthetic request *through the breaker* (`breaker.execute(probe)`)
+ * on a schedule. A success closes the circuit via cockatiel's normal
+ * half-open transition; a failure re-opens it. Healthy breakers are never
+ * probed — the loop is zero-cost when everything is up.
+ *
+ * Probes are registered per provider name (the breaker registry key).
+ * Built-in probes cover the local providers (ollama / llamacpp / lmstudio),
+ * whose cheap GET endpoints are well-known, free, and exactly where the
+ * dead-upstream-hang problem historically lived. Cloud providers can be
+ * added via registerHealthProbe(name, fn); without one, their recovery
+ * stays as before (next live request) — no behavior regression.
+ */
+
+const config = require('../config');
+const logger = require('../logger');
+const { getCockatielRegistry } = require('./resilience');
+
+const PROBE_INTERVAL_MS = Number.parseInt(process.env.LYNKR_HEALTH_PROBE_INTERVAL_MS, 10) || 30_000;
+const PROBE_TIMEOUT_MS = Number.parseInt(process.env.LYNKR_HEALTH_PROBE_TIMEOUT_MS, 10) || 5_000;
+const ENABLED = process.env.LYNKR_HEALTH_PROBE_ENABLED !== 'false';
+
+/** @type {Map<string, () => Promise<void>>} providerName → probe fn (throws on unhealthy) */
+const probes = new Map();
+
+async function _cheapGet(url, headers = {}) {
+  const response = await fetch(url, {
+    method: 'GET',
+    headers,
+    signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+  });
+  if (!response.ok) {
+    throw new Error(`health probe ${url} → ${response.status}`);
+  }
+  // Drain the (small) body so the socket is released cleanly.
+  await response.arrayBuffer().catch(() => {});
+}
+
+function _registerBuiltins() {
+  if (config.ollama?.endpoint) {
+    probes.set('ollama', () => _cheapGet(`${config.ollama.endpoint}/api/tags`));
+  }
+  if (config.llamacpp?.endpoint) {
+    const headers = config.llamacpp.apiKey
+      ? { Authorization: `Bearer ${config.llamacpp.apiKey}` }
+      : {};
+    probes.set('llamacpp', () => _cheapGet(`${config.llamacpp.endpoint}/v1/models`, headers));
+  }
+  if (config.lmstudio?.endpoint) {
+    const headers = config.lmstudio.apiKey
+      ? { Authorization: `Bearer ${config.lmstudio.apiKey}` }
+      : {};
+    probes.set('lmstudio', () => _cheapGet(`${config.lmstudio.endpoint}/v1/models`, headers));
+  }
+}
+
+/**
+ * Register (or override) a synthetic probe for a provider. The function must
+ * resolve when the provider is healthy and throw/reject when it is not.
+ * @param {string} providerName — must match the circuit breaker registry key
+ * @param {() => Promise<void>} probeFn
+ */
+function registerHealthProbe(providerName, probeFn) {
+  probes.set(providerName, probeFn);
+}
+
+class HealthProber {
+  constructor() {
+    this.timer = null;
+    this.stats = { sweeps: 0, probesSent: 0, recoveries: 0, failures: 0, lastSweepAt: null };
+  }
+
+  start() {
+    if (this.timer || !ENABLED) return;
+    _registerBuiltins();
+    this.timer = setInterval(() => {
+      this.sweep().catch((err) => {
+        logger.debug({ error: err.message }, '[HealthProbe] sweep error');
+      });
+    }, PROBE_INTERVAL_MS);
+    // Never keep the process alive just to probe.
+    this.timer.unref?.();
+    logger.info(
+      { intervalMs: PROBE_INTERVAL_MS, providers: [...probes.keys()] },
+      '[HealthProbe] Synthetic circuit-breaker probing started'
+    );
+  }
+
+  stop() {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+  }
+
+  /**
+   * One probing pass: probe every registered provider whose breaker is not
+   * closed. Runs probes through the breaker itself so state transitions use
+   * cockatiel's own half-open machinery.
+   */
+  async sweep() {
+    this.stats.sweeps += 1;
+    this.stats.lastSweepAt = Date.now();
+    const registry = getCockatielRegistry();
+    const work = [];
+    for (const [providerName, probeFn] of probes) {
+      const breaker = registry.breakers.get(providerName);
+      if (!breaker || breaker.state === 'CLOSED') continue;
+      work.push(
+        (async () => {
+          this.stats.probesSent += 1;
+          try {
+            await breaker.execute(probeFn);
+            this.stats.recoveries += 1;
+            logger.info({ provider: providerName }, '[HealthProbe] Provider recovered via synthetic probe — circuit closed without a live request paying for the test');
+          } catch (err) {
+            // Expected while the provider is still down (or the circuit is
+            // OPEN and not yet past halfOpenAfter, where execute rejects
+            // immediately without dialing). Stay quiet; the breaker's own
+            // state logging covers transitions.
+            this.stats.failures += 1;
+            logger.debug({ provider: providerName, error: err.message }, '[HealthProbe] probe failed');
+          }
+        })()
+      );
+    }
+    await Promise.all(work);
+  }
+
+  getStatus() {
+    return {
+      enabled: ENABLED,
+      running: !!this.timer,
+      intervalMs: PROBE_INTERVAL_MS,
+      registeredProviders: [...probes.keys()],
+      ...this.stats,
+    };
+  }
+}
+
+let prober = null;
+
+/** @returns {HealthProber} singleton */
+function getHealthProber() {
+  if (!prober) prober = new HealthProber();
+  return prober;
+}
+
+module.exports = { getHealthProber, registerHealthProbe };

package/src/clients/provider-capabilities.js

@@ -32,8 +32,35 @@ function getThinkingBehavior(providerType, model) {
   return "none";
 }
 
+/**
+ * Anthropic-shaped `thinking` request param to send upstream, honoring an
+ * explicit client request and otherwise defaulting to disabled.
+ *
+ * Confirmed live (2026-08-27) against both endpoints directly: Baidu
+ * Qianfan's `glm-5.2` and Moonshot's Kimi (`kimi-k3`) both emit verbose
+ * `reasoning_content` on EVERY call by default, sharing the same token
+ * budget as the visible answer — with no instruction at all, a baseline
+ * call to either returned ~500-600 chars of reasoning_content, an EMPTY
+ * `content`, and `finish_reason:"length"` (the whole max_tokens budget
+ * spent on reasoning, none left for the actual answer). The GLM/Qwen
+ * convention `enable_thinking:false` does NOT suppress this on either
+ * endpoint — only this Anthropic-shaped `{type:"disabled"}` param does
+ * (also confirmed live, on both). Not a caveman-specific issue — the same
+ * budget waste happens on every request to these two providers; caveman's
+ * elaborate constraint set just makes it worse, since the model has more to
+ * visibly deliberate against.
+ *
+ * @param {Object} body - incoming Anthropic-format request body
+ * @returns {Object} the `thinking` param to send upstream
+ */
+function resolveThinkingParam(body) {
+  if (body?.thinking && typeof body.thinking === "object") return body.thinking;
+  return { type: "disabled" };
+}
+
 module.exports = {
   supportsNativeThinking,
   supportsReasoningContent,
   getThinkingBehavior,
+  resolveThinkingParam,
 };

package/src/config/index.js

@@ -62,7 +62,7 @@ function resolveConfigPath(targetPath) {
   return path.resolve(normalised);
 }
 
-const SUPPORTED_MODEL_PROVIDERS = new Set(["databricks", "azure-anthropic", "ollama", "openrouter", "edenai", "azure-openai", "openai", "atlas", "llamacpp", "lmstudio", "bedrock", "zai", "vertex", "moonshot", "baidu"]);
+const SUPPORTED_MODEL_PROVIDERS = new Set(["databricks", "azure-anthropic", "ollama", "openrouter", "edenai", "azure-openai", "openai", "atlas", "llamacpp", "lmstudio", "bedrock", "zai", "vertex", "moonshot", "baidu", "fireworks"]);
 const rawModelProvider = (process.env.MODEL_PROVIDER ?? "databricks").toLowerCase();
 
 // Validate MODEL_PROVIDER early with a clear error message
@@ -153,6 +153,11 @@ const baiduApiKey = process.env.BAIDU_API_KEY?.trim() || null;
 const baiduEndpoint = process.env.BAIDU_ENDPOINT?.trim() || "https://qianfan.baidubce.com/v2/chat/completions";
 const baiduModel = process.env.BAIDU_MODEL?.trim() || "ernie-4.5-turbo-128k";
 
+// Fireworks AI configuration - OpenAI-compatible serverless inference API
+const fireworksApiKey = process.env.FIREWORKS_API_KEY?.trim() || null;
+const fireworksEndpoint = process.env.FIREWORKS_ENDPOINT?.trim() || "https://api.fireworks.ai/inference/v1/chat/completions";
+const fireworksModel = process.env.FIREWORKS_MODEL?.trim() || "accounts/fireworks/models/kimi-k2-instruct-0905";
+
 // Vertex AI (Google Gemini) configuration
 const vertexApiKey = process.env.VERTEX_API_KEY?.trim() || process.env.GOOGLE_API_KEY?.trim() || null;
 const vertexModel = process.env.VERTEX_MODEL?.trim() || "gemini-2.0-flash";
@@ -662,6 +667,11 @@ var config = {
     endpoint: baiduEndpoint,
     model: baiduModel,
   },
+  fireworks: {
+    apiKey: fireworksApiKey,
+    endpoint: fireworksEndpoint,
+    model: fireworksModel,
+  },
   codex: {
     enabled: process.env.CODEX_ENABLED !== "false",
     binaryPath: process.env.CODEX_BINARY_PATH?.trim() || null,
@@ -1125,6 +1135,8 @@ function reloadConfig() {
   config.moonshot.model = process.env.MOONSHOT_MODEL?.trim() || "kimi-k2-turbo-preview";
   config.baidu.apiKey = process.env.BAIDU_API_KEY?.trim() || null;
   config.baidu.model = process.env.BAIDU_MODEL?.trim() || "ernie-4.5-turbo-128k";
+  config.fireworks.apiKey = process.env.FIREWORKS_API_KEY?.trim() || null;
+  config.fireworks.model = process.env.FIREWORKS_MODEL?.trim() || "accounts/fireworks/models/kimi-k2-instruct-0905";
 
   // Model provider settings
   const newProvider = (process.env.MODEL_PROVIDER ?? "databricks").toLowerCase();

package/src/context/compression.js

@@ -40,14 +40,50 @@ function compressHistory(messages, options = {}) {
     keepRecentTurns: options.keepRecentTurns ?? config.historyCompression?.keepRecentTurns ?? 10,
     summarizeOlder: options.summarizeOlder ?? config.historyCompression?.summarizeOlder ?? true,
     enabled: options.enabled ?? config.historyCompression?.enabled ?? true,
+    // Model token budget (e.g. effectiveMax from the orchestrator). When set,
+    // the verbatim-recent window grows to USE the budget instead of a fixed
+    // message count. Live incident: a 935-message session against a 128k
+    // model kept only 10 recent messages (~56k tokens) while 108.8k tokens
+    // of budget were available — ~50k tokens of context thrown away, giving
+    // the model amnesia about its own recent edits.
+    budgetTokens: options.budgetTokens ?? null,
   };
 
   if (!opts.enabled) {
     return messages; // Return uncompressed if disabled
   }
 
+  // Recent-window size: fixed count by default; budget-driven when a model
+  // token budget is provided. RECENT_BUDGET_FRACTION reserves the rest of
+  // the budget for the summary block, system prompt, tools, and the
+  // response itself.
+  const RECENT_BUDGET_FRACTION = 0.6;
+  const CHARS_PER_TOKEN = 4;
+  let keepRecent = opts.keepRecentTurns;
+  if (Number.isFinite(opts.budgetTokens) && opts.budgetTokens > 0) {
+    const targetChars = opts.budgetTokens * CHARS_PER_TOKEN * RECENT_BUDGET_FRACTION;
+    let accumulated = 0;
+    let count = 0;
+    for (let i = messages.length - 1; i >= 0; i--) {
+      accumulated += estimateMessagesSize([messages[i]]);
+      if (accumulated > targetChars) break;
+      count++;
+    }
+    keepRecent = Math.max(opts.keepRecentTurns, count);
+  }
+
   // Calculate split point
-  const splitIndex = Math.max(0, messages.length - opts.keepRecentTurns);
+  let splitIndex = Math.max(0, messages.length - keepRecent);
+
+  // Tool-exchange boundary alignment: never let the recent window START with
+  // a tool_result whose paired tool_use just got summarized away — providers
+  // reject orphaned tool_result blocks ("unexpected tool_use_id"). Pull the
+  // split earlier until the boundary is clean.
+  const startsWithToolResult = (msg) =>
+    Array.isArray(msg?.content) && msg.content.some((b) => b?.type === 'tool_result');
+  while (splitIndex > 0 && startsWithToolResult(messages[splitIndex])) {
+    splitIndex--;
+  }
 
   if (splitIndex === 0) {
     // All messages are recent, no compression needed
@@ -126,31 +162,56 @@ function summarizeOldHistory(messages) {
   let hasUserInput = false;
   let hasAssistantOutput = false;
 
+  // Index tool_result blocks by tool_use_id so each tool_use below can carry
+  // a one-line recap of what it actually found. Previously this summary kept
+  // only the tool NAME ("Assistant used tools: bash") and threw away the
+  // result entirely — which meant a long tool-calling session forgot its own
+  // diagnosis every time it aged past the recent-turns window and had to
+  // re-derive it from scratch (live symptom: an agent stuck re-reading the
+  // same file and re-fetching the same issue instead of ever writing a fix).
+  const resultByToolUseId = new Map();
+  for (const msg of messages) {
+    if (!Array.isArray(msg.content)) continue;
+    for (const block of msg.content) {
+      if (block?.type === 'tool_result' && block.tool_use_id) {
+        resultByToolUseId.set(block.tool_use_id, extractResultSnippet(block));
+      }
+    }
+  }
+
   for (const msg of messages) {
     if (msg.role === 'user') {
       hasUserInput = true;
       const content = extractTextContent(msg);
-      if (content.length < 200) {
-        keyPoints.push(`User: ${content}`);
-      } else {
-        // Compress long user messages
-        keyPoints.push(`User: ${content.substring(0, 150)}...`);
+      if (content) {
+        // compressText keeps head AND tail (unlike a flat substring(0, N)),
+        // which matters here too: the actionable part of a long user
+        // message is often at the end ("...so please fix X and open a PR").
+        keyPoints.push(`User: ${compressText(content, 200)}`);
       }
     } else if (msg.role === 'assistant') {
       hasAssistantOutput = true;
       const content = extractTextContent(msg);
 
-      // Extract tool uses
-      const toolUses = extractToolUses(msg);
-      if (toolUses.length > 0) {
-        keyPoints.push(`Assistant used tools: ${toolUses.join(', ')}`);
+      // Extract tool uses WITH their outcome, not just the tool name.
+      const toolUseBlocks = extractToolUseBlocks(msg);
+      for (const use of toolUseBlocks) {
+        const argsSnippet = summarizeToolArgs(use.input);
+        const resultSnippet = resultByToolUseId.get(use.id);
+        keyPoints.push(
+          resultSnippet
+            ? `Assistant called ${use.name}(${argsSnippet}) → ${resultSnippet}`
+            : `Assistant called ${use.name}(${argsSnippet})`
+        );
       }
 
-      // Add assistant text if meaningful
-      if (content.length > 20 && content.length < 200) {
-        keyPoints.push(`Assistant: ${content}`);
-      } else if (content.length >= 200) {
-        keyPoints.push(`Assistant: ${content.substring(0, 150)}...`);
+      // Add assistant text if meaningful. compressText (head+tail) instead
+      // of a head-only substring(0, 150) — conclusions ("Cause: ...",
+      // "Findings: ...") are usually written at the END of a reasoning
+      // turn, so a head-only cut was silently discarding exactly the part
+      // that mattered for remembering what was already diagnosed.
+      if (content.length > 20) {
+        keyPoints.push(`Assistant: ${compressText(content, 300)}`);
       }
     }
   }
@@ -168,6 +229,51 @@ function summarizeOldHistory(messages) {
   };
 }
 
+/**
+ * Short, single-line recap of a tool_result's content — used so the
+ * compacted history shows what a tool call actually found, not just that it
+ * was called. Reuses compressText's head+tail truncation.
+ *
+ * @param {Object} block - tool_result block
+ * @returns {string} snippet, or '' if the result had no usable text
+ */
+function extractResultSnippet(block) {
+  const RESULT_SNIPPET_MAX = 160;
+  let text = '';
+  if (typeof block.content === 'string') {
+    text = block.content;
+  } else if (Array.isArray(block.content)) {
+    text = block.content
+      .filter(item => typeof item === 'string' || item?.type === 'text')
+      .map(item => (typeof item === 'string' ? item : item.text || ''))
+      .join(' ');
+  }
+  text = text.trim();
+  if (!text) return '';
+  const prefix = block.is_error ? 'error: ' : '';
+  return prefix + compressText(text, RESULT_SNIPPET_MAX);
+}
+
+/**
+ * Compact, single-line rendering of a tool_use's arguments for the summary
+ * (e.g. `Read(file_path=".../types.go", offset=390)`), not the full JSON.
+ *
+ * @param {Object} input - tool_use input object
+ * @returns {string}
+ */
+function summarizeToolArgs(input) {
+  const ARGS_SNIPPET_MAX = 80;
+  if (!input || typeof input !== 'object') return '';
+  try {
+    const flat = Object.entries(input)
+      .map(([k, v]) => `${k}=${typeof v === 'string' ? v : JSON.stringify(v)}`)
+      .join(', ');
+    return compressText(flat, ARGS_SNIPPET_MAX);
+  } catch {
+    return '';
+  }
+}
+
 /**
  * Compress a single message
  *
@@ -382,17 +488,19 @@ function extractTextContent(message) {
 }
 
 /**
- * Extract tool names used in message
+ * Extract tool_use blocks (id/name/input) from a message — the fuller
+ * counterpart to extractToolUses(), which only returns names. Used by
+ * summarizeOldHistory() to pair each call with its outcome.
  *
  * @param {Object} message - Message object
- * @returns {Array} Tool names
+ * @returns {Array<{id: string, name: string, input: Object}>}
  */
-function extractToolUses(message) {
+function extractToolUseBlocks(message) {
   if (!Array.isArray(message.content)) return [];
 
   return message.content
     .filter(block => block.type === 'tool_use')
-    .map(block => block.name);
+    .map(block => ({ id: block.id, name: block.name, input: block.input }));
 }
 
 /**

package/src/dashboard/api.js

@@ -17,6 +17,7 @@ function providerMeta() {
     'azure-openai':    { type: 'cloud', configured: !!(c.azureOpenAI?.endpoint && c.azureOpenAI?.apiKey) },
     vertex:            { type: 'cloud', configured: !!c.vertex?.projectId },
     moonshot:          { type: 'cloud', configured: !!c.moonshot?.apiKey },
+    fireworks:         { type: 'cloud', configured: !!c.fireworks?.apiKey },
     ollama:            { type: 'local', configured: !!c.ollama?.endpoint },
     llamacpp:          { type: 'local', configured: !!c.llamacpp?.endpoint },
     lmstudio:          { type: 'local', configured: !!c.lmstudio?.endpoint },

package/src/observability/metrics.js

@@ -29,6 +29,20 @@ class MetricsCollector {
     this.budgetBlocks = 0;
     this.rateLimitBlocks = 0;
 
+    // Loop-guard interventions (cross-request tool dedup): warnings injected
+    // into model context and force-terminated turns. Non-zero numbers here
+    // mean Lynkr actively altered agent behavior — surfaced so operators
+    // don't need log forensics to discover it.
+    this.loopGuardWarnings = 0;
+    this.loopGuardTerminations = 0;
+
+    // History compression: how often and how hard conversations are being
+    // squeezed to fit the model's context window. A high maxPercentage means
+    // models are seeing a small fraction of their sessions.
+    this.historyCompressions = 0;
+    this.historyCompressionLastPct = 0;
+    this.historyCompressionMaxPct = 0;
+
     // API metrics
     this.databricksRequests = 0;
     this.databricksErrors = 0;
@@ -120,6 +134,28 @@ class MetricsCollector {
     this.rateLimitBlocks++;
   }
 
+  /**
+   * Record a loop-guard intervention.
+   * @param {'warn'|'terminate'} kind
+   */
+  recordLoopGuard(kind) {
+    // 'observed' is the only kind emitted since the guard went observe-only;
+    // 'terminate' is kept as a counter bucket for any historical dashboards.
+    if (kind === 'terminate') this.loopGuardTerminations++;
+    else this.loopGuardWarnings++;
+  }
+
+  /**
+   * Record a history-compression event.
+   * @param {number} percentage - percent of chars removed (0-100)
+   */
+  recordHistoryCompression(percentage) {
+    this.historyCompressions++;
+    const pct = Number(percentage) || 0;
+    this.historyCompressionLastPct = pct;
+    if (pct > this.historyCompressionMaxPct) this.historyCompressionMaxPct = pct;
+  }
+
   /**
    * Record Databricks API call
    */
@@ -222,6 +258,13 @@ class MetricsCollector {
       budget_blocks_total: this.budgetBlocks,
       rate_limit_blocks_total: this.rateLimitBlocks,
 
+      // Loop guard + history compression (context interventions)
+      loop_guard_warnings_total: this.loopGuardWarnings,
+      loop_guard_terminations_total: this.loopGuardTerminations,
+      history_compressions_total: this.historyCompressions,
+      history_compression_last_pct: this.historyCompressionLastPct,
+      history_compression_max_pct: this.historyCompressionMaxPct,
+
       // API
       databricks_requests_total: this.databricksRequests,
       databricks_errors_total: this.databricksErrors,
@@ -332,6 +375,15 @@ class MetricsCollector {
     metric("http_request_duration_ms", "summary", "HTTP request latency in ms", metrics.latency_ms.p95, { quantile: "0.95" });
     metric("http_request_duration_ms", "summary", "HTTP request latency in ms", metrics.latency_ms.p99, { quantile: "0.99" });
 
+    // OTel GenAI semantic-convention aliases (dots → underscores, per
+    // Prometheus conversion rules). Same underlying counters as above,
+    // named so semconv-aware dashboards/tooling find them.
+    metric("gen_ai_client_token_usage_total", "counter", "GenAI tokens consumed (OTel semconv)", metrics.tokens_input_total, { gen_ai_token_type: "input" });
+    metric("gen_ai_client_token_usage_total", "counter", "GenAI tokens consumed (OTel semconv)", metrics.tokens_output_total, { gen_ai_token_type: "output" });
+    metric("gen_ai_client_operation_duration_ms", "summary", "GenAI operation duration (OTel semconv)", metrics.latency_ms.median, { quantile: "0.5" });
+    metric("gen_ai_client_operation_duration_ms", "summary", "GenAI operation duration (OTel semconv)", metrics.latency_ms.p95, { quantile: "0.95" });
+    metric("gen_ai_client_operation_duration_ms", "summary", "GenAI operation duration (OTel semconv)", metrics.latency_ms.p99, { quantile: "0.99" });
+
     return lines.join("\n");
   }