lynkr 8.0.1 → 9.0.2

package/src/clients/retry.js

@@ -1,200 +1,55 @@
-const logger = require("../logger");
-
 /**
- * Retry configuration for API calls
+ * Retry logic for API calls — backed by Cockatiel
+ *
+ * This module re-exports the Cockatiel-backed retry adapter from resilience.js
+ * while preserving all original exports for consumers.
  */
+const { withCockatielRetry, DEFAULT_RETRY_CONFIG } = require("./resilience");
+
 const DEFAULT_CONFIG = {
-  maxRetries: 3,
-  initialDelay: 1000, // 1 second
-  maxDelay: 30000, // 30 seconds
-  backoffMultiplier: 2,
-  jitterFactor: 0.1, // 10% jitter
-  retryableStatuses: [429, 500, 502, 503, 504],
-  retryableErrors: ['ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'ENETUNREACH', 'ECONNREFUSED'],
+  maxRetries: DEFAULT_RETRY_CONFIG.maxRetries,
+  initialDelay: DEFAULT_RETRY_CONFIG.initialDelay,
+  maxDelay: DEFAULT_RETRY_CONFIG.maxDelay,
+  backoffMultiplier: DEFAULT_RETRY_CONFIG.backoffMultiplier,
+  jitterFactor: DEFAULT_RETRY_CONFIG.jitterFactor,
+  retryableStatuses: DEFAULT_RETRY_CONFIG.retryableStatuses,
+  retryableErrors: DEFAULT_RETRY_CONFIG.retryableErrors,
 };
 
 /**
- * Add jitter to prevent thundering herd
- */
-function addJitter(delay, jitterFactor) {
-  const jitter = delay * jitterFactor * (Math.random() * 2 - 1);
-  return Math.max(0, delay + jitter);
-}
-
-/**
- * Calculate delay with exponential backoff
+ * Calculate delay with exponential backoff (preserved for any direct callers)
  */
 function calculateDelay(attempt, config) {
   const baseDelay = config.initialDelay * Math.pow(config.backoffMultiplier, attempt);
   const cappedDelay = Math.min(baseDelay, config.maxDelay);
-  return addJitter(cappedDelay, config.jitterFactor);
+  const jitter = cappedDelay * config.jitterFactor * (Math.random() * 2 - 1);
+  return Math.max(0, cappedDelay + jitter);
 }
 
 /**
- * Check if error/response is retryable
+ * Check if error/response is retryable (preserved for any direct callers)
  */
 function isRetryable(error, response, config) {
-  // Check response status codes
   if (response && config.retryableStatuses.includes(response.status)) {
     return true;
   }
-
-  // Check error codes
   if (error && error.code && config.retryableErrors.includes(error.code)) {
     return true;
   }
-
-  // Check nested cause (Node undici wraps connection errors as TypeError)
   if (error && error.cause?.code && config.retryableErrors.includes(error.cause.code)) {
     return true;
   }
-
-  // Check for network errors
-  if (error && (error.name === 'FetchError' || error.name === 'AbortError')) {
+  if (error && (error.name === "FetchError" || error.name === "AbortError")) {
     return true;
   }
-
   return false;
 }
 
 /**
- * Detect if this is a cold start (longer than expected response time)
+ * Detect if this is a cold start
  */
 function detectColdStart(startTime, endTime, threshold = 5000) {
-  const duration = endTime - startTime;
-  return duration > threshold;
-}
-
-/**
- * Execute function with retry logic
- */
-async function withRetry(fn, options = {}) {
-  const config = { ...DEFAULT_CONFIG, ...options };
-  let lastError;
-  let lastResponse;
-
-  for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
-    const startTime = Date.now();
-
-    try {
-      const result = await fn(attempt);
-      const endTime = Date.now();
-
-      // Detect cold starts for monitoring
-      if (detectColdStart(startTime, endTime)) {
-        logger.warn({
-          attempt,
-          duration: endTime - startTime,
-        }, 'Potential cold start detected');
-      }
-
-      // Check if response indicates we should retry
-      if (result && isRetryable(null, result, config) && attempt < config.maxRetries) {
-        lastResponse = result;
-
-        // Special handling for 429 (rate limiting)
-        if (result.status === 429) {
-          // Check for Retry-After header
-          const retryAfter = result.headers?.get?.('retry-after');
-          let delay;
-
-          if (retryAfter) {
-            // Retry-After can be in seconds or a date
-            const retryAfterNum = parseInt(retryAfter, 10);
-            if (!isNaN(retryAfterNum)) {
-              delay = retryAfterNum * 1000; // Convert to ms
-            } else {
-              const retryAfterDate = new Date(retryAfter);
-              delay = retryAfterDate.getTime() - Date.now();
-            }
-          } else {
-            // Use exponential backoff with longer delays for rate limiting
-            delay = calculateDelay(attempt, {
-              ...config,
-              initialDelay: 2000, // Start at 2s for rate limits
-              maxDelay: 60000, // Up to 1 minute
-            });
-          }
-
-          logger.warn({
-            attempt,
-            delay,
-            retryAfter: retryAfter || 'not specified',
-          }, 'Rate limited (429), retrying after delay');
-
-          await sleep(delay);
-          continue;
-        }
-
-        // Regular retry with exponential backoff
-        const delay = calculateDelay(attempt, config);
-        logger.warn({
-          attempt,
-          status: result.status,
-          delay,
-        }, 'Request failed, retrying with backoff');
-
-        await sleep(delay);
-        continue;
-      }
-
-      // Success or non-retryable error
-      return result;
-
-    } catch (error) {
-      lastError = error;
-      const endTime = Date.now();
-
-      // Check if cold start
-      if (detectColdStart(startTime, endTime)) {
-        logger.warn({
-          attempt,
-          duration: endTime - startTime,
-          error: error.message,
-        }, 'Potential cold start with error detected');
-      }
-
-      // Check if we should retry
-      if (isRetryable(error, null, config) && attempt < config.maxRetries) {
-        const delay = calculateDelay(attempt, config);
-        logger.warn({
-          attempt,
-          error: error.message,
-          code: error.code,
-          delay,
-        }, 'Request error, retrying with backoff');
-
-        await sleep(delay);
-        continue;
-      }
-
-      // Not retryable or out of retries
-      throw error;
-    }
-  }
-
-  // Max retries exceeded
-  if (lastError) {
-    lastError.message = `Max retries (${config.maxRetries}) exceeded: ${lastError.message}`;
-    throw lastError;
-  }
-
-  if (lastResponse) {
-    logger.error({
-      status: lastResponse.status,
-      maxRetries: config.maxRetries,
-    }, 'Max retries exceeded');
-    return lastResponse;
-  }
-
-  throw new Error('Retry logic failed unexpectedly');
-}
-
-/**
- * Sleep helper
- */
-function sleep(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
+  return (endTime - startTime) > threshold;
 }
 
 /**
@@ -202,12 +57,12 @@ function sleep(ms) {
  */
 function createRetryWrapper(fn, defaultOptions = {}) {
   return async function (...args) {
-    return withRetry(() => fn(...args), defaultOptions);
+    return withCockatielRetry(() => fn(...args), defaultOptions);
   };
 }
 
 module.exports = {
-  withRetry,
+  withRetry: withCockatielRetry,
   createRetryWrapper,
   calculateDelay,
   isRetryable,

package/src/clients/standard-tools.js

@@ -275,7 +275,7 @@ EXAMPLE: User says "explore this project" → Call Task with subagent_type="Expl
           description: "Optional model override. Default is appropriate for each agent type."
         }
       },
-      required: ["description", "prompt", "subagent_type"]
+      required: ["prompt"]
     }
   },
   {

package/src/clients/xml-tool-extractor.js

@@ -0,0 +1,307 @@
+/**
+ * Universal Tool Call Extractor
+ *
+ * Extracts tool calls embedded as raw text (XML, JSON, custom tokens)
+ * from LLM output.  Covers Minimax, Hermes/Qwen, Qwen3-Coder, GLM,
+ * Llama, Mistral, DeepSeek, GPT-OSS, and generic formats.
+ *
+ * @module clients/xml-tool-extractor
+ */
+
+const logger = require("../logger");
+
+let callCounter = 0;
+function nextId() {
+  return `call_extracted_${Date.now()}_${callCounter++}`;
+}
+
+function tryParseJSON(str) {
+  try { return JSON.parse(str.trim()); } catch { return null; }
+}
+
+// ── Individual extractors ────────────────────────────────────────────
+
+/** 1. Minimax: <invoke name="X"><parameter name="K">V</parameter></invoke> */
+function extractMinimax(text) {
+  const calls = [];
+  const re = /<invoke\s+name="([^"]+)">([\s\S]*?)<\/invoke>/g;
+  const paramRe = /<parameter\s+name="([^"]+)">([\s\S]*?)<\/parameter>/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const name = m[1];
+    const body = m[2];
+    const args = {};
+    let pm;
+    while ((pm = paramRe.exec(body)) !== null) {
+      let val = pm[2].trim();
+      const parsed = tryParseJSON(val);
+      args[pm[1]] = parsed !== null ? parsed : val;
+    }
+    paramRe.lastIndex = 0;
+    calls.push({ name, arguments: args, _match: m[0] });
+  }
+  // Also strip wrapper tags
+  let cleaned = text;
+  if (calls.length > 0) {
+    for (const c of calls) cleaned = cleaned.replace(c._match, "");
+    cleaned = cleaned.replace(/<\/?minimax:tool_call>/g, "");
+  }
+  return { calls, cleaned };
+}
+
+/** 2. GLM: <tool_call>func_name <arg_key>k</arg_key> <arg_value>v</arg_value></tool_call> */
+function extractGLM(text) {
+  const calls = [];
+  const re = /<tool_call>([\s\S]*?)<\/tool_call>/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const body = m[1].trim();
+    // Check if it's GLM style (has <arg_key> tags)
+    if (!body.includes("<arg_key>")) continue;
+    const nameMatch = body.match(/^(\S+)/);
+    if (!nameMatch) continue;
+    const name = nameMatch[1];
+    const args = {};
+    const kvRe = /<arg_key>([\s\S]*?)<\/arg_key>\s*<arg_value>([\s\S]*?)<\/arg_value>/g;
+    let kv;
+    while ((kv = kvRe.exec(body)) !== null) {
+      let val = kv[2].trim();
+      const parsed = tryParseJSON(val);
+      args[kv[1].trim()] = parsed !== null ? parsed : val;
+    }
+    calls.push({ name, arguments: args, _match: m[0] });
+  }
+  let cleaned = text;
+  for (const c of calls) cleaned = cleaned.replace(c._match, "");
+  return { calls, cleaned };
+}
+
+/** 3. Hermes/Qwen JSON: <tool_call>{"name":"X","arguments":{...}}</tool_call> */
+function extractHermesQwen(text) {
+  const calls = [];
+  const re = /<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const body = m[1].trim();
+    // Skip GLM format (handled above)
+    if (body.includes("<arg_key>")) continue;
+    // Skip Qwen3-Coder XML format
+    if (body.includes("<tool_name>")) continue;
+    const json = tryParseJSON(body);
+    if (json && (json.name || json.function)) {
+      const name = json.name || json.function?.name || json.function || "unknown";
+      const args = json.arguments || json.parameters || json.params || {};
+      calls.push({ name, arguments: typeof args === "string" ? tryParseJSON(args) || {} : args, _match: m[0] });
+    }
+  }
+  let cleaned = text;
+  for (const c of calls) cleaned = cleaned.replace(c._match, "");
+  return { calls, cleaned };
+}
+
+/** 4. Qwen3-Coder: <tool_call><tool_name>X</tool_name><parameter name="K">V</parameter></tool_call> */
+function extractQwenCoder(text) {
+  const calls = [];
+  const re = /<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const body = m[1].trim();
+    if (!body.includes("<tool_name>")) continue;
+    const nameMatch = body.match(/<tool_name>([\s\S]*?)<\/tool_name>/);
+    if (!nameMatch) continue;
+    const name = nameMatch[1].trim();
+    const args = {};
+    const paramRe = /<parameter\s+name="([^"]+)">([\s\S]*?)<\/parameter>/g;
+    let pm;
+    while ((pm = paramRe.exec(body)) !== null) {
+      let val = pm[2].trim();
+      const parsed = tryParseJSON(val);
+      args[pm[1]] = parsed !== null ? parsed : val;
+    }
+    calls.push({ name, arguments: args, _match: m[0] });
+  }
+  let cleaned = text;
+  for (const c of calls) cleaned = cleaned.replace(c._match, "");
+  return { calls, cleaned };
+}
+
+/** 5. DeepSeek: <｜tool▁call▁begin｜>function<｜tool▁sep｜>name\n```json\n{...}\n```\n<｜tool▁call▁end｜> */
+function extractDeepSeek(text) {
+  const calls = [];
+  // Match both Unicode and ASCII approximations
+  const re = /(?:<｜tool▁call▁begin｜>|<\|tool_call_begin\|>|<\|tool_call_start\|>)\s*(?:function)?\s*(?:<｜tool▁sep｜>|<\|tool_sep\|>)?\s*(\S+)\s*```(?:json)?\s*([\s\S]*?)```\s*(?:<｜tool▁call▁end｜>|<\|tool_call_end\|>)/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const name = m[1].trim();
+    const json = tryParseJSON(m[2]);
+    if (name) {
+      calls.push({ name, arguments: json || {}, _match: m[0] });
+    }
+  }
+  let cleaned = text;
+  for (const c of calls) cleaned = cleaned.replace(c._match, "");
+  return { calls, cleaned };
+}
+
+/** 6. Mistral: [TOOL_CALLS] [{"name":"X","arguments":{...}}] */
+function extractMistral(text) {
+  const calls = [];
+  const re = /\[TOOL_CALLS\]\s*(\[[\s\S]*?\])/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const arr = tryParseJSON(m[1]);
+    if (Array.isArray(arr)) {
+      for (const item of arr) {
+        if (item.name || item.function) {
+          const name = item.name || item.function?.name || "unknown";
+          const args = item.arguments || item.parameters || {};
+          calls.push({ name, arguments: typeof args === "string" ? tryParseJSON(args) || {} : args, _match: m[0] });
+        }
+      }
+    }
+  }
+  let cleaned = text;
+  for (const c of calls) cleaned = cleaned.replace(c._match, "");
+  return { calls, cleaned };
+}
+
+/** 7. Llama python_tag: <|python_tag|>{"name":"X","arguments":{...}} */
+function extractLlamaPythonTag(text) {
+  const calls = [];
+  const re = /<\|python_tag\|>\s*(\{[\s\S]*?\})(?:<\|eom_id\|>|<\|eot_id\|>|\s*$)/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const json = tryParseJSON(m[1]);
+    if (json && (json.name || json.function)) {
+      const name = json.name || json.function || "unknown";
+      const args = json.arguments || json.parameters || {};
+      calls.push({ name, arguments: typeof args === "string" ? tryParseJSON(args) || {} : args, _match: m[0] });
+    }
+  }
+  let cleaned = text;
+  for (const c of calls) cleaned = cleaned.replace(c._match, "");
+  return { calls, cleaned };
+}
+
+/** 8. GPT-OSS Harmony: <|call|>name(key=value, ...) */
+function extractGptOss(text) {
+  const calls = [];
+  const re = /<\|call\|>\s*(\w+)\(([^)]*)\)/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const name = m[1];
+    const argsStr = m[2].trim();
+    const args = {};
+    if (argsStr) {
+      // Parse key=value pairs
+      const kvRe = /(\w+)\s*=\s*(?:"([^"]*)"|'([^']*)'|(\S+))/g;
+      let kv;
+      while ((kv = kvRe.exec(argsStr)) !== null) {
+        args[kv[1]] = kv[2] ?? kv[3] ?? kv[4];
+      }
+    }
+    calls.push({ name, arguments: args, _match: m[0] });
+  }
+  let cleaned = text;
+  for (const c of calls) cleaned = cleaned.replace(c._match, "");
+  return { calls, cleaned };
+}
+
+/** 9. Generic <function_call>{...}</function_call> */
+function extractGenericFunctionCall(text) {
+  const calls = [];
+  const re = /<function_call>\s*([\s\S]*?)\s*<\/function_call>/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    const json = tryParseJSON(m[1]);
+    if (json && (json.name || json.function)) {
+      const name = json.name || json.function?.name || json.function || "unknown";
+      const args = json.arguments || json.parameters || {};
+      calls.push({ name, arguments: typeof args === "string" ? tryParseJSON(args) || {} : args, _match: m[0] });
+    }
+  }
+  let cleaned = text;
+  for (const c of calls) cleaned = cleaned.replace(c._match, "");
+  return { calls, cleaned };
+}
+
+/** 10. Raw JSON fallback: text starts with {"name":"...","arguments":{...}} */
+function extractRawJSON(text) {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith("{")) return { calls: [], cleaned: text };
+  const json = tryParseJSON(trimmed);
+  if (!json || typeof json !== "object") return { calls: [], cleaned: text };
+  if (!json.name && !json.function) return { calls: [], cleaned: text };
+  const name = json.name || json.function?.name || json.function || "unknown";
+  const args = json.arguments || json.parameters || {};
+  return {
+    calls: [{ name, arguments: typeof args === "string" ? tryParseJSON(args) || {} : args, _match: trimmed }],
+    cleaned: "",
+  };
+}
+
+// ── Main entry point ─────────────────────────────────────────────────
+
+const EXTRACTORS = [
+  { name: "minimax", fn: extractMinimax },
+  { name: "glm", fn: extractGLM },
+  { name: "hermes_qwen", fn: extractHermesQwen },
+  { name: "qwen_coder", fn: extractQwenCoder },
+  { name: "deepseek", fn: extractDeepSeek },
+  { name: "mistral", fn: extractMistral },
+  { name: "llama_python", fn: extractLlamaPythonTag },
+  { name: "gpt_oss", fn: extractGptOss },
+  { name: "generic_function_call", fn: extractGenericFunctionCall },
+  { name: "raw_json", fn: extractRawJSON },
+];
+
+/**
+ * Extract tool calls from model text output.
+ * Tries all known patterns (most specific → most generic).
+ * Returns on first extractor that finds tool calls.
+ *
+ * @param {string} text - Raw model text content
+ * @returns {{ toolCalls: Array, cleanedText: string|null }}
+ */
+function extractToolCallsFromText(text) {
+  if (!text || typeof text !== "string") {
+    return { toolCalls: [], cleanedText: text };
+  }
+
+  for (const { name: extractorName, fn } of EXTRACTORS) {
+    try {
+      const { calls, cleaned } = fn(text);
+      if (calls.length > 0) {
+        const toolCalls = calls.map((c) => ({
+          id: nextId(),
+          type: "function",
+          function: {
+            name: c.name,
+            arguments: typeof c.arguments === "string" ? c.arguments : JSON.stringify(c.arguments),
+          },
+        }));
+
+        // Clean up stray tokens
+        let cleanedText = (cleaned || "")
+          .replace(/<\|eom_id\|>/g, "")
+          .replace(/<\|eot_id\|>/g, "")
+          .replace(/<\|end\|>/g, "")
+          .trim() || null;
+
+        logger.info({
+          extractor: extractorName,
+          toolCount: toolCalls.length,
+          tools: toolCalls.map((t) => t.function.name),
+        }, "Extracted tool calls from model text output");
+
+        return { toolCalls, cleanedText };
+      }
+    } catch (err) {
+      logger.debug({ extractor: extractorName, error: err.message }, "Tool call extractor failed, trying next");
+    }
+  }
+
+  return { toolCalls: [], cleanedText: text };
+}
+
+module.exports = { extractToolCallsFromText };

package/src/cluster.js

@@ -0,0 +1,82 @@
+/**
+ * Cluster Mode — Multi-Core Scaling
+ *
+ * Forks one worker per CPU core. Each worker runs a full Lynkr
+ * instance with its own Express server, event loop, and connection pool.
+ *
+ * Enable: CLUSTER_ENABLED=true (default: false for dev, recommended for prod)
+ * Workers: CLUSTER_WORKERS=auto (default) or a number
+ *
+ * Architecture:
+ *   Primary process → forks N workers → each worker calls start()
+ *   Primary handles: signal forwarding, worker respawning, health monitoring
+ *   Workers handle: HTTP requests, LLM proxying, tool execution
+ *
+ * Shared state considerations:
+ *   - SQLite: WAL mode supports concurrent readers across processes
+ *   - In-memory caches (prompt, circuit breaker): per-worker (not shared)
+ *   - Rate limiting: per-worker (sessions are sticky via round-robin)
+ *
+ * @module cluster
+ */
+
+const cluster = require('node:cluster');
+const os = require('node:os');
+
+const WORKER_COUNT = (() => {
+  const env = process.env.CLUSTER_WORKERS;
+  if (!env || env === 'auto') return Math.max(os.cpus().length - 1, 1);
+  const n = parseInt(env, 10);
+  return Number.isNaN(n) || n < 1 ? Math.max(os.cpus().length - 1, 1) : n;
+})();
+
+function startCluster() {
+  if (cluster.isPrimary) {
+    console.log(`[cluster] Primary ${process.pid} starting ${WORKER_COUNT} workers`);
+
+    // Fork workers
+    for (let i = 0; i < WORKER_COUNT; i++) {
+      cluster.fork();
+    }
+
+    // Respawn crashed workers
+    cluster.on('exit', (worker, code, signal) => {
+      if (signal) {
+        console.log(`[cluster] Worker ${worker.process.pid} killed by signal ${signal}`);
+      } else if (code !== 0) {
+        console.log(`[cluster] Worker ${worker.process.pid} exited with code ${code}, respawning...`);
+        cluster.fork();
+      } else {
+        console.log(`[cluster] Worker ${worker.process.pid} exited cleanly`);
+      }
+    });
+
+    // Forward SIGTERM/SIGINT to all workers for graceful shutdown
+    const shutdown = (sig) => {
+      console.log(`[cluster] Primary received ${sig}, shutting down workers...`);
+      for (const id in cluster.workers) {
+        cluster.workers[id].process.kill(sig);
+      }
+      // Give workers 10s to drain, then force exit
+      setTimeout(() => {
+        console.log('[cluster] Force exit after 10s drain timeout');
+        process.exit(0);
+      }, 10000).unref();
+    };
+
+    process.on('SIGTERM', () => shutdown('SIGTERM'));
+    process.on('SIGINT', () => shutdown('SIGINT'));
+
+    // Log worker status
+    cluster.on('online', (worker) => {
+      console.log(`[cluster] Worker ${worker.process.pid} online`);
+    });
+
+  } else {
+    // Worker process — start the normal Lynkr server
+    const { start } = require('./server');
+    start();
+  }
+}
+
+module.exports = { startCluster, WORKER_COUNT };