lynkr 9.14.1 → 9.14.3

package/src/tools/web-search-exec.js

@@ -0,0 +1,182 @@
+/**
+ * Server-side web_search / web_fetch execution — narrowly scoped resurrection.
+ *
+ * Context: src/tools/web.js used to implement this (SearXNG-backed search,
+ * host-allowlisted fetch) and was deleted 2026-07-22 in commit b32e988,
+ * "Remove server-mode tool execution: Lynkr always forwards tool calls to
+ * the client." That was a deliberate architecture pivot, not a bug — Lynkr
+ * stopped executing ANY tool itself.
+ *
+ * This does NOT reintroduce general server-mode tool execution. It only
+ * auto-resolves `web_search`/`web_fetch` calls, and ONLY when the calling
+ * client is unrecognized by src/routing/client-profiles.js's detectClient()
+ * — i.e. Lynkr has no signal the caller can fulfill the call itself.
+ * Known harnesses (Claude Code / Claude Desktop's shared agent-sdk — both
+ * present as `claude-cli/...`, Cursor, goose, Codex) already execute these
+ * client-side today and are never routed through this module; see the call
+ * site in src/orchestrator/index.js's runAgentLoop.
+ *
+ * @module tools/web-search-exec
+ */
+
+const config = require("../config");
+const logger = require("../logger");
+
+const AUTO_RESOLVABLE_NAMES = new Set(["web_search", "websearch", "web_fetch", "webfetch"]);
+
+/**
+ * @param {string} name
+ * @returns {boolean}
+ */
+function isAutoResolvable(name) {
+  return AUTO_RESOLVABLE_NAMES.has(String(name || "").toLowerCase());
+}
+
+/**
+ * True only when every call in this batch is one we can resolve ourselves —
+ * a mixed batch (one auto-resolvable + one arbitrary client tool) falls
+ * through to the normal "forward to client" path untouched, since we can't
+ * partially resolve a batch the model expects answered together.
+ *
+ * @param {Array} toolCalls - normalized {id, function:{name}} or {name} calls
+ * @returns {boolean}
+ */
+function canAutoResolveAll(toolCalls) {
+  return Array.isArray(toolCalls) && toolCalls.length > 0
+    && toolCalls.every((tc) => isAutoResolvable(tc?.function?.name ?? tc?.name));
+}
+
+function isHostAllowed(urlStr) {
+  if (config.webSearch?.allowAllHosts) return true;
+  try {
+    const host = new URL(urlStr).hostname.toLowerCase();
+    const allowed = config.webSearch?.allowedHosts;
+    return Array.isArray(allowed) && allowed.includes(host);
+  } catch {
+    return false;
+  }
+}
+
+async function _withTimeout(fn, timeoutMs) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    return await fn(controller.signal);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * SearXNG-backed search. Retries transient failures per config.webSearch
+ * (retryEnabled/maxRetries) — mirrors the retry policy the deleted web.js
+ * had, since a single dropped connection to a local sidecar shouldn't fail
+ * the whole tool call.
+ *
+ * @param {string} query
+ * @returns {Promise<Object>} { query, results: [{title,url,snippet}] } or { error }
+ */
+async function searchWeb(query) {
+  const endpoint = config.webSearch?.endpoint || "http://localhost:8888/search";
+  const timeoutMs = config.webSearch?.timeoutMs || 10000;
+  const maxRetries = config.webSearch?.retryEnabled ? (config.webSearch?.maxRetries ?? 2) : 0;
+  const url = `${endpoint}?q=${encodeURIComponent(query || "")}&format=json`;
+
+  let lastErr = null;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const res = await _withTimeout(
+        (signal) => fetch(url, { signal }),
+        timeoutMs,
+      );
+      if (!res.ok) {
+        lastErr = `search backend returned HTTP ${res.status}`;
+        continue;
+      }
+      const json = await res.json();
+      const results = (Array.isArray(json.results) ? json.results : [])
+        .slice(0, 8)
+        .map((r) => ({ title: r.title, url: r.url, snippet: r.content }));
+      return { query, results };
+    } catch (err) {
+      lastErr = err.message;
+    }
+  }
+  logger.warn({ query, endpoint, err: lastErr }, "[web-search-exec] search failed after retries");
+  return { query, error: `search failed: ${lastErr}` };
+}
+
+/**
+ * Generic URL fetch, host-allowlisted per config.webSearch (allowAllHosts /
+ * allowedHosts), truncated per config.webSearch.bodyPreviewMax.
+ *
+ * @param {string} targetUrl
+ * @returns {Promise<Object>}
+ */
+async function fetchUrl(targetUrl) {
+  if (!targetUrl) return { error: "no url provided" };
+  if (!isHostAllowed(targetUrl)) {
+    logger.warn({ url: targetUrl }, "[web-search-exec] fetch blocked — host not allowed");
+    return { url: targetUrl, error: "host not in allowedHosts (WEB_SEARCH_ALLOW_ALL=false)" };
+  }
+  const maxPreview = config.webSearch?.bodyPreviewMax || 10000;
+  const timeoutMs = config.webSearch?.timeoutMs || 10000;
+  try {
+    const res = await _withTimeout(
+      (signal) => fetch(targetUrl, { signal }),
+      timeoutMs,
+    );
+    const text = await res.text();
+    return {
+      url: targetUrl,
+      status: res.status,
+      content: text.slice(0, maxPreview),
+      truncated: text.length > maxPreview,
+    };
+  } catch (err) {
+    logger.warn({ url: targetUrl, err: err.message }, "[web-search-exec] fetch failed");
+    return { url: targetUrl, error: `fetch failed: ${err.message}` };
+  }
+}
+
+/**
+ * Execute every call in this (pre-checked via canAutoResolveAll) batch and
+ * append the matching assistant tool_use + user tool_result turns to
+ * `messages`, mirroring exactly what a real client round-trip would send
+ * back — so the next agent-loop iteration behaves identically to a normal
+ * client-fulfilled tool exchange, and the client never sees the exchange.
+ *
+ * @param {Array} toolCalls
+ * @param {Array} messages - mutated in place (push only, never rewritten)
+ */
+async function autoResolve(toolCalls, messages) {
+  const toolUseBlocks = [];
+  const toolResultBlocks = [];
+
+  for (const tc of toolCalls) {
+    const rawName = tc.function?.name ?? tc.name ?? "";
+    const name = String(rawName).toLowerCase();
+    let input = {};
+    try {
+      const raw = tc.function?.arguments ?? tc.input ?? {};
+      input = typeof raw === "string" ? JSON.parse(raw) : (raw || {});
+    } catch { /* leave input empty — result will just reflect an empty query/url */ }
+
+    toolUseBlocks.push({ type: "tool_use", id: tc.id, name: rawName, input });
+
+    const result = (name === "web_search" || name === "websearch")
+      ? await searchWeb(input.query || input.q || "")
+      : await fetchUrl(input.url || "");
+
+    toolResultBlocks.push({
+      type: "tool_result",
+      tool_use_id: tc.id,
+      content: JSON.stringify(result),
+    });
+  }
+
+  messages.push({ role: "assistant", content: toolUseBlocks });
+  messages.push({ role: "user", content: toolResultBlocks });
+}
+
+module.exports = { canAutoResolveAll, autoResolve, searchWeb, fetchUrl, isAutoResolvable };

package/src/training/trajectory-compressor.js

@@ -38,7 +38,7 @@ const telemetry = require("../routing/telemetry");
 const ANONYMIZE_PATTERNS = [
   // API keys and bearer tokens
   [/sk-[A-Za-z0-9_-]{20,}/g, "<API_KEY>"],
-  [/Bearer\s+[A-Za-z0-9._\-]+/gi, "Bearer <REDACTED>"],
+  [/Bearer\s+[A-Za-z0-9._-]+/gi, "Bearer <REDACTED>"],
   [/dapi_[A-Za-z0-9_-]+/g, "<DATABRICKS_KEY>"],
   [/eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, "<JWT>"],
   // AWS keys

package/src/workers/pool.js

@@ -310,7 +310,7 @@ class WorkerPool {
     logger.info('[WorkerPool] Shutting down...');
 
     // Reject all pending tasks
-    for (const [taskId, pending] of this.pendingTasks) {
+    for (const [_taskId, pending] of this.pendingTasks) {
       clearTimeout(pending.timeout);
       pending.reject(new Error('Worker pool shutting down'));
     }

package/src/workers/worker.js

@@ -65,7 +65,7 @@ const handlers = {
     let tokensReduced = 0;
     const seenToolResults = new Map(); // For deduplication
 
-    const transformed = messages.map((msg, idx) => {
+    const transformed = messages.map((msg) => {
       if (!msg) return msg;
 
       // Clone to avoid mutating original

package/scripts/convert-routellm.py

@@ -1,105 +0,0 @@
-#!/usr/bin/env python3
-"""
-Convert routellm/gpt4_dataset (train.jsonl) → Lynkr kNN bootstrap JSONL.
-
-Stream-downloads from HuggingFace and maps mixtral_score (1-5) to your
-TIER_* model config so the kNN index learns to recommend YOUR real models,
-not RouteLLM's pool (mixtral / gpt-4).
-
-Output format matches scripts/build-knn-index.js bootstrap input:
-    {"query": "...", "provider": "ollama", "model": "minimax-m2.5:cloud",
-     "quality": 95, "cost": 0.001, "latency": 200, "tier": "SIMPLE"}
-
-Usage:
-    .venv/bin/python scripts/convert-routellm.py [--limit 30000] -o routellm.jsonl
-"""
-import argparse
-import json
-import sys
-import urllib.request
-import ssl
-
-# Map mixtral_score → user's tier config.
-# Spread is intentionally aggressive: only score=5 (Mixtral truly nailed it)
-# stays SIMPLE; score=4 escalates to MEDIUM so the kNN index has enough MEDIUM
-# examples to actually recommend non-SIMPLE tiers (vs the previous mapping
-# where 86% of records were SIMPLE and kNN never disagreed with the heuristic).
-SCORE_TO_ROUTE = {
-    5: {"tier": "SIMPLE",    "provider": "ollama",          "model": "minimax-m2.5:cloud", "quality": 95, "cost": 0.001, "latency": 200},
-    4: {"tier": "MEDIUM",    "provider": "azure-anthropic", "model": "claude-sonnet-4-6",  "quality": 80, "cost": 0.003, "latency": 1500},
-    3: {"tier": "MEDIUM",    "provider": "azure-anthropic", "model": "claude-sonnet-4-6",  "quality": 70, "cost": 0.003, "latency": 1500},
-    2: {"tier": "COMPLEX",   "provider": "azure-anthropic", "model": "claude-sonnet-4-6",  "quality": 55, "cost": 0.003, "latency": 1500},
-    1: {"tier": "REASONING", "provider": "azure-anthropic", "model": "claude-opus-4-8",    "quality": 40, "cost": 0.015, "latency": 3500},
-}
-
-URL = "https://huggingface.co/datasets/routellm/gpt4_dataset/resolve/main/train.jsonl"
-
-
-def main():
-    p = argparse.ArgumentParser()
-    p.add_argument("--output", "-o", default="data/routellm-bootstrap.jsonl")
-    p.add_argument("--limit", type=int, default=None,
-                   help="Only convert first N rows")
-    args = p.parse_args()
-
-    print(f"Streaming {URL} …", file=sys.stderr)
-    # Some macOS Python builds have SSL issues with huggingface.co — fall back
-    # to unverified context if needed. The data is public, no auth.
-    ctx = ssl.create_default_context()
-    try:
-        with urllib.request.urlopen(URL, context=ctx, timeout=60) as resp:
-            _stream(resp, args.output, args.limit)
-    except (ssl.SSLError, OSError) as e:
-        print(f"SSL strict failed ({e}); retrying with unverified context", file=sys.stderr)
-        ctx = ssl._create_unverified_context()
-        with urllib.request.urlopen(URL, context=ctx, timeout=60) as resp:
-            _stream(resp, args.output, args.limit)
-
-
-def _stream(resp, out_path, limit):
-    written = 0
-    by_score = {s: 0 for s in SCORE_TO_ROUTE}
-    skipped = 0
-    with open(out_path, "w") as out:
-        for line in resp:
-            if limit and written >= limit:
-                break
-            try:
-                row = json.loads(line)
-            except Exception:
-                skipped += 1
-                continue
-            score = row.get("mixtral_score")
-            if score not in SCORE_TO_ROUTE:
-                skipped += 1
-                continue
-            route = SCORE_TO_ROUTE[score]
-            rec = {
-                "query": row.get("prompt", ""),
-                "provider": route["provider"],
-                "model": route["model"],
-                "quality": route["quality"],
-                "cost": route["cost"],
-                "latency": route["latency"],
-                "tier": route["tier"],
-            }
-            if not rec["query"]:
-                skipped += 1
-                continue
-            out.write(json.dumps(rec) + "\n")
-            written += 1
-            by_score[score] += 1
-            if written % 5000 == 0:
-                print(f"  {written} written…", file=sys.stderr)
-
-    print(f"\nWrote {written} records to {out_path} ({skipped} skipped)", file=sys.stderr)
-    print(f"By mixtral_score: {by_score}", file=sys.stderr)
-    print(f"\nTier distribution:", file=sys.stderr)
-    for score, route in SCORE_TO_ROUTE.items():
-        n = by_score[score]
-        if n:
-            print(f"  {route['tier']:>10}  {route['provider']}:{route['model']:30s}  n={n}", file=sys.stderr)
-
-
-if __name__ == "__main__":
-    main()

package/src/diff/comments.js

@@ -1,153 +0,0 @@
-const crypto = require("crypto");
-const db = require("../db");
-const logger = require("../logger");
-
-const insertCommentStmt = db.prepare(
-  `INSERT INTO diff_comments (
-     thread_id,
-     session_id,
-     file_path,
-     hunk,
-     line,
-     comment,
-     author,
-     created_at,
-     metadata
-   ) VALUES (
-     @thread_id,
-     @session_id,
-     @file_path,
-     @hunk,
-     @line,
-     @comment,
-     @author,
-     @created_at,
-     @metadata
-   )`,
-);
-
-const deleteCommentStmt = db.prepare("DELETE FROM diff_comments WHERE id = ?");
-
-function buildSelectCommentsQuery({ filePath, threadId }) {
-  let sql = `SELECT id,
-                    thread_id,
-                    session_id,
-                    file_path,
-                    hunk,
-                    line,
-                    comment,
-                    author,
-                    created_at,
-                    metadata
-             FROM diff_comments`;
-  const conditions = [];
-  const params = [];
-  if (filePath) {
-    conditions.push("file_path = ?");
-    params.push(filePath);
-  }
-  if (threadId) {
-    conditions.push("thread_id = ?");
-    params.push(threadId);
-  }
-  if (conditions.length) {
-    sql += ` WHERE ${conditions.join(" AND ")}`;
-  }
-  sql += " ORDER BY created_at ASC, id ASC";
-  return { sql, params };
-}
-
-function normaliseCommentRow(row) {
-  return {
-    id: row.id,
-    threadId: row.thread_id ?? null,
-    sessionId: row.session_id ?? null,
-    filePath: row.file_path,
-    hunk: row.hunk ?? null,
-    line: typeof row.line === "number" ? row.line : null,
-    comment: row.comment,
-    author: row.author ?? null,
-    createdAt: row.created_at,
-    metadata: row.metadata ? JSON.parse(row.metadata) : null,
-  };
-}
-
-function addDiffComment({
-  threadId,
-  sessionId,
-  filePath,
-  line,
-  hunk,
-  comment,
-  author,
-  metadata,
-}) {
-  if (typeof filePath !== "string" || filePath.trim().length === 0) {
-    throw new Error("Diff comment requires a file path.");
-  }
-  if (typeof comment !== "string" || comment.trim().length === 0) {
-    throw new Error("Diff comment requires non-empty comment text.");
-  }
-
-  const createdAt = Date.now();
-  const thread = threadId ?? crypto.randomUUID();
-
-  const params = {
-    thread_id: thread,
-    session_id: sessionId ?? null,
-    file_path: filePath,
-    hunk: hunk ?? null,
-    line: typeof line === "number" ? line : null,
-    comment,
-    author: author ?? null,
-    created_at: createdAt,
-    metadata: metadata ? JSON.stringify(metadata) : null,
-  };
-
-  const result = insertCommentStmt.run(params);
-
-  logger.debug(
-    {
-      id: result.lastInsertRowid,
-      thread,
-      filePath,
-      line,
-    },
-    "Recorded diff comment",
-  );
-
-  return normaliseCommentRow({
-    id: result.lastInsertRowid,
-    thread_id: thread,
-    session_id: sessionId ?? null,
-    file_path: filePath,
-    hunk: hunk ?? null,
-    line: typeof line === "number" ? line : null,
-    comment,
-    author: author ?? null,
-    created_at: createdAt,
-    metadata: metadata ? JSON.stringify(metadata) : null,
-  });
-}
-
-function listDiffComments({ filePath, threadId } = {}) {
-  const { sql, params } = buildSelectCommentsQuery({ filePath, threadId });
-  return db
-    .prepare(sql)
-    .all(...params)
-    .map(normaliseCommentRow);
-}
-
-function deleteDiffComment({ id }) {
-  if (!id) {
-    throw new Error("diff_comment_delete requires an id.");
-  }
-  const result = deleteCommentStmt.run(id);
-  return result.changes > 0;
-}
-
-module.exports = {
-  addDiffComment,
-  listDiffComments,
-  deleteDiffComment,
-};

package/src/policy/web-fallback.js

@@ -1,33 +0,0 @@
-const BROWSING_FALLBACK_PATTERNS = [
-  /i (do|don't|cannot) have (browser|browsing|internet) (capability|access)/i,
-  /cannot look up information/i,
-  /no web browsing capability/i,
-  /can'?t (access|reach) the internet/i,
-  /(do not|don't) have access to .*web (?:browsing|browser|internet)/i,
-  /(do not|don't) have .*browser/i,
-  /web(fetch|_fetch| search).*(not available|disabled|unavailable)/i,
-  /tool.*(not available|disabled|unavailable)/i,
-  /don't have access to real-time/i,
-];
-
-function needsWebFallback(text) {
-  if (typeof text !== "string") return false;
-  const trimmed = text.trim();
-  if (!trimmed) return false;
-
-  // If the response already includes concrete financial data, skip fallback.
-  if (
-    /\bclosed at \$\d[\d.,]*/i.test(trimmed) ||
-    /\bprevious close\b/i.test(trimmed) ||
-    /\bday'?s range\b/i.test(trimmed) ||
-    /\btrading volume\b/i.test(trimmed)
-  ) {
-    return false;
-  }
-
-  return BROWSING_FALLBACK_PATTERNS.some((regex) => regex.test(trimmed));
-}
-
-module.exports = {
-  needsWebFallback,
-};