lynkr 8.0.1 → 9.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lynkr",
-  "version": "8.0.1",
+  "version": "9.0.1",
   "description": "Self-hosted Claude Code & Cursor proxy with Databricks,AWS BedRock,Azure  adapters, openrouter, Ollama,llamacpp,LM Studio, workspace tooling, and MCP integration.",
   "main": "index.js",
   "bin": {
@@ -8,13 +8,13 @@
     "lynkr-setup": "./scripts/setup.js"
   },
   "scripts": {
-    "prestart": "node -e \"if(process.env.HEADROOM_ENABLED==='true'&&process.env.HEADROOM_DOCKER_ENABLED!=='false'){process.exit(0)}else{process.exit(1)}\" && docker compose --profile headroom up -d headroom 2>/dev/null || echo 'Headroom skipped (disabled or Docker not running)'",
+    "prestart": "node -e \"if(process.env.HEADROOM_ENABLED==='true'&&process.env.HEADROOM_DOCKER_ENABLED!=='false'){process.exit(0)}else{process.exit(1)}\" && docker compose --profile headroom up -d --build headroom 2>/dev/null || echo 'Headroom skipped (disabled or Docker not running)'",
     "start": "node index.js 2>&1 | npx pino-pretty --sync",
     "stop": "node -e \"if(process.env.HEADROOM_ENABLED==='true'&&process.env.HEADROOM_DOCKER_ENABLED!=='false'){process.exit(0)}else{process.exit(1)}\" && docker compose --profile headroom down || echo 'Headroom skipped (disabled or Docker not running)'",
     "dev": "nodemon index.js",
     "lint": "eslint src index.js",
     "test": "npm run test:unit && npm run test:performance",
-    "test:unit": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/routing.test.js test/hybrid-routing-integration.test.js test/web-tools.test.js test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js test/azure-openai-config.test.js test/azure-openai-format-conversion.test.js test/azure-openai-routing.test.js test/azure-openai-streaming.test.js test/azure-openai-error-resilience.test.js test/azure-openai-integration.test.js test/openai-integration.test.js test/toon-compression.test.js test/llamacpp-integration.test.js test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js",
+    "test:unit": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/routing.test.js test/hybrid-routing-integration.test.js test/web-tools.test.js test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js test/azure-openai-config.test.js test/azure-openai-format-conversion.test.js test/azure-openai-routing.test.js test/azure-openai-streaming.test.js test/azure-openai-error-resilience.test.js test/azure-openai-integration.test.js test/openai-integration.test.js test/toon-compression.test.js test/llamacpp-integration.test.js test/resilience.test.js test/telemetry-routing.test.js test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js test/distill.test.js test/large-payload.test.js test/code-mode.test.js",
     "test:memory": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js",
     "test:new-features": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js",
     "test:performance": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/hybrid-routing-performance.test.js && DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/performance-tests.js",
@@ -48,6 +48,7 @@
     "@babel/parser": "^7.29.0",
     "@babel/traverse": "^7.29.0",
     "@toon-format/toon": "^2.1.0",
+    "cockatiel": "^3.2.1",
     "compression": "^1.7.4",
     "diff": "^5.2.0",
     "dotenv": "^16.4.5",

package/src/api/openai-router.js

@@ -27,6 +27,23 @@ const { IDE_SAFE_TOOLS } = require("../clients/standard-tools");
 
 const router = express.Router();
 
+/**
+ * Resolve the model name for OpenAI responses.
+ * In OpenClaw mode, returns the actual provider/model from routing metadata.
+ */
+function resolveResponseModel(resultBody, requestModel) {
+  if (config.openclaw?.enabled && resultBody?._routingMeta) {
+    const meta = resultBody._routingMeta;
+    if (meta.provider && meta.model) {
+      return `${meta.provider}/${meta.model}`;
+    }
+    if (meta.provider) {
+      return meta.provider;
+    }
+  }
+  return requestModel;
+}
+
 /**
  * Client detection - identifies which AI coding tool is making the request
  * @param {Object} headers - Request headers
@@ -499,7 +516,8 @@ router.post("/chat/completions", async (req, res) => {
         }
 
         // Convert to OpenAI format
-        const openaiResponse = convertAnthropicToOpenAI(result.body, req.body.model);
+        const streamModel = resolveResponseModel(result.body, req.body.model);
+        const openaiResponse = convertAnthropicToOpenAI(result.body, streamModel);
 
         // Debug: Log what we're about to stream
         logger.debug({
@@ -539,7 +557,7 @@ router.post("/chat/completions", async (req, res) => {
           id: openaiResponse.id,
           object: "chat.completion.chunk",
           created: openaiResponse.created,
-          model: req.body.model,
+          model: streamModel,
           system_fingerprint: "fp_lynkr",
           choices: [{
             index: 0,
@@ -561,7 +579,7 @@ router.post("/chat/completions", async (req, res) => {
             id: openaiResponse.id,
             object: "chat.completion.chunk",
             created: openaiResponse.created,
-            model: req.body.model,
+            model: streamModel,
             system_fingerprint: "fp_lynkr",
             choices: [{
               index: 0,
@@ -581,7 +599,7 @@ router.post("/chat/completions", async (req, res) => {
               id: openaiResponse.id,
               object: "chat.completion.chunk",
               created: openaiResponse.created,
-              model: req.body.model,
+              model: streamModel,
               choices: [{
                 index: 0,
                 delta: {
@@ -608,7 +626,7 @@ router.post("/chat/completions", async (req, res) => {
           id: openaiResponse.id,
           object: "chat.completion.chunk",
           created: openaiResponse.created,
-          model: req.body.model,
+          model: streamModel,
           system_fingerprint: "fp_lynkr",
           choices: [{
             index: 0,
@@ -679,7 +697,7 @@ router.post("/chat/completions", async (req, res) => {
       }, "Orchestrator result structure");
 
       // Convert Anthropic response to OpenAI format
-      const openaiResponse = convertAnthropicToOpenAI(result.body, req.body.model);
+      const openaiResponse = convertAnthropicToOpenAI(result.body, resolveResponseModel(result.body, req.body.model));
 
       // Map tool names for known IDE clients
       if (clientType !== "unknown" && openaiResponse.choices?.[0]?.message?.tool_calls?.length > 0) {
@@ -1535,7 +1553,8 @@ router.post("/responses", async (req, res) => {
         }, "=== ORCHESTRATOR RESULT FOR RESPONSES API ===");
 
         // Convert back: Anthropic → OpenAI → Responses
-        const chatResponse = convertAnthropicToOpenAI(result.body, req.body.model);
+        const responsesModel = resolveResponseModel(result.body, req.body.model);
+        const chatResponse = convertAnthropicToOpenAI(result.body, responsesModel);
 
         logger.debug({
           chatContent: chatResponse.choices?.[0]?.message?.content?.substring(0, 200),
@@ -1597,7 +1616,7 @@ router.post("/responses", async (req, res) => {
             object: "response",
             status: "in_progress",
             created_at: createdAt,
-            model: req.body.model,
+            model: responsesModel,
             output: [],
             usage: null
           },
@@ -1614,7 +1633,7 @@ router.post("/responses", async (req, res) => {
             object: "response",
             status: "in_progress",
             created_at: createdAt,
-            model: req.body.model,
+            model: responsesModel,
             output: [],
             usage: null
           },
@@ -1793,7 +1812,7 @@ router.post("/responses", async (req, res) => {
             object: "response",
             status: "completed",
             created_at: createdAt,
-            model: req.body.model,
+            model: responsesModel,
             output: outputItems,
             usage: {
               input_tokens: responsesResponse.usage?.prompt_tokens || 0,
@@ -1846,7 +1865,7 @@ router.post("/responses", async (req, res) => {
       });
 
       // Convert back: Anthropic → OpenAI → Responses
-      const chatResponse = convertAnthropicToOpenAI(result.body, req.body.model);
+      const chatResponse = convertAnthropicToOpenAI(result.body, resolveResponseModel(result.body, req.body.model));
       const responsesResponse = convertChatToResponses(chatResponse);
 
       logger.info({

package/src/api/providers-handler.js

@@ -415,8 +415,8 @@ router.get("/health/providers", (req, res) => {
     // Get circuit breaker states
     const circuitBreakerStates = {};
     const allBreakers = registry.getAll();
-    for (const [name, breaker] of Object.entries(allBreakers)) {
-      circuitBreakerStates[name] = breaker.state;
+    for (const breaker of allBreakers) {
+      circuitBreakerStates[breaker.name] = breaker.state;
     }
 
     // Get all provider health
@@ -452,7 +452,8 @@ router.get("/health/providers/:name", (req, res) => {
 
     // Get circuit breaker state for this provider
     const allBreakers = registry.getAll();
-    const circuitState = allBreakers[providerName]?.state || "CLOSED";
+    const breakerState = allBreakers.find((b) => b.name === providerName);
+    const circuitState = breakerState?.state || "CLOSED";
 
     // Get detailed metrics
     const metrics = healthTracker.getProviderMetrics(providerName);
@@ -477,4 +478,171 @@ router.get("/health/providers/:name", (req, res) => {
   }
 });
 
+// ============================================================================
+// Routing Telemetry Endpoints
+// ============================================================================
+
+const telemetry = require("../routing/telemetry");
+const { getLatencyTracker } = require("../routing/latency-tracker");
+
+/**
+ * GET /v1/routing/stats
+ *
+ * Aggregated routing telemetry statistics.
+ */
+router.get("/routing/stats", (req, res) => {
+  try {
+    const since = req.query.since ? Number(req.query.since) : undefined;
+    const until = req.query.until ? Number(req.query.until) : undefined;
+    const stats = telemetry.getStats({ since, until });
+
+    if (!stats) {
+      return res.json({ object: "routing_stats", data: null, message: "No telemetry data available" });
+    }
+
+    // Merge latency tracker percentiles
+    const latencyTracker = getLatencyTracker();
+    const latencyStats = {};
+    for (const [provider, pStats] of latencyTracker.getAllStats()) {
+      latencyStats[provider] = pStats;
+    }
+
+    res.json({
+      object: "routing_stats",
+      data: { ...stats, latencyPercentiles: latencyStats },
+      timestamp: new Date().toISOString(),
+    });
+  } catch (error) {
+    logger.error({ error: error.message }, "Error getting routing stats");
+    res.status(500).json({ error: { type: "server_error", message: error.message } });
+  }
+});
+
+/**
+ * GET /v1/routing/stats/:provider
+ *
+ * Per-provider routing telemetry.
+ */
+router.get("/routing/stats/:provider", (req, res) => {
+  try {
+    const provider = req.params.provider.toLowerCase();
+    const since = req.query.since ? Number(req.query.since) : undefined;
+    const stats = telemetry.getProviderStats(provider, { since });
+
+    if (!stats) {
+      return res.json({ object: "provider_routing_stats", data: null, message: `No data for ${provider}` });
+    }
+
+    const latencyTracker = getLatencyTracker();
+    const latency = latencyTracker.getStats(provider);
+
+    res.json({
+      object: "provider_routing_stats",
+      provider,
+      data: { ...stats, latency },
+      timestamp: new Date().toISOString(),
+    });
+  } catch (error) {
+    logger.error({ error: error.message }, "Error getting provider routing stats");
+    res.status(500).json({ error: { type: "server_error", message: error.message } });
+  }
+});
+
+/**
+ * GET /v1/routing/telemetry
+ *
+ * Raw telemetry records (most recent first).
+ */
+router.get("/routing/telemetry", (req, res) => {
+  try {
+    const filters = {
+      provider: req.query.provider,
+      tier: req.query.tier,
+      since: req.query.since ? Number(req.query.since) : undefined,
+      limit: req.query.limit ? Number(req.query.limit) : 100,
+    };
+    const records = telemetry.query(filters);
+
+    res.json({ object: "telemetry_list", data: records, count: records.length });
+  } catch (error) {
+    logger.error({ error: error.message }, "Error querying telemetry");
+    res.status(500).json({ error: { type: "server_error", message: error.message } });
+  }
+});
+
+/**
+ * GET /v1/routing/accuracy
+ *
+ * Routing accuracy analysis (over/under-provisioned percentages).
+ */
+router.get("/routing/accuracy", (req, res) => {
+  try {
+    const since = req.query.since ? Number(req.query.since) : undefined;
+    const accuracy = telemetry.getRoutingAccuracy({ since });
+
+    res.json({
+      object: "routing_accuracy",
+      data: accuracy,
+      timestamp: new Date().toISOString(),
+    });
+  } catch (error) {
+    logger.error({ error: error.message }, "Error getting routing accuracy");
+    res.status(500).json({ error: { type: "server_error", message: error.message } });
+  }
+});
+
+// ── Admin: Hot Reload Config + Reset Circuit Breakers ─────────────────
+
+router.post("/admin/reload", (req, res) => {
+  try {
+    config.reloadConfig();
+    const registry = getCircuitBreakerRegistry();
+    const states = registry.getAll();
+    res.json({
+      object: "admin_reload",
+      status: "ok",
+      reloaded: [
+        "modelTiers",
+        "apiKeys",
+        "providerSettings",
+        "circuitBreakers",
+      ],
+      circuitBreakers: states.map(s => ({ name: s.name, state: s.state })),
+      timestamp: new Date().toISOString(),
+    });
+  } catch (error) {
+    logger.error({ error: error.message }, "Admin reload failed");
+    res.status(500).json({ error: { type: "server_error", message: error.message } });
+  }
+});
+
+router.post("/admin/circuit-breakers/reset", (req, res) => {
+  try {
+    const provider = req.query.provider || req.body?.provider;
+    const registry = getCircuitBreakerRegistry();
+
+    if (provider) {
+      const breaker = registry.breakers?.get(provider);
+      if (breaker) {
+        breaker.reset();
+        res.json({ object: "circuit_breaker_reset", provider, status: "reset" });
+      } else {
+        res.status(404).json({ error: { message: `No circuit breaker for provider: ${provider}` } });
+      }
+    } else {
+      registry.resetAll();
+      const states = registry.getAll();
+      res.json({
+        object: "circuit_breaker_reset",
+        provider: "all",
+        status: "reset",
+        breakers: states.map(s => ({ name: s.name, state: s.state })),
+      });
+    }
+  } catch (error) {
+    logger.error({ error: error.message }, "Circuit breaker reset failed");
+    res.status(500).json({ error: { type: "server_error", message: error.message } });
+  }
+});
+
 module.exports = router;

package/src/api/router.js

@@ -136,7 +136,7 @@ router.post("/routing/analyze", async (req, res) => {
     const { getModelTierSelector } = require("../routing/model-tiers");
     const { getModelRegistry } = require("../routing/model-registry");
 
-    const analysis = analyzeComplexity(req.body, { weighted: req.query.weighted === "true" });
+    const analysis = await analyzeComplexity(req.body, { weighted: req.query.weighted === "true" });
     const agentic = getAgenticDetector().detect(req.body);
     const selector = getModelTierSelector();
     const tier = selector.getTier(analysis.score);
@@ -210,13 +210,17 @@ router.post("/api/event_logging/batch", (req, res) => {
 
 router.post("/v1/messages", rateLimiter, async (req, res, next) => {
   try {
+    const { createTimer } = require("../utils/perf-timer");
+    const timer = createTimer("POST /v1/messages");
     metrics.recordRequest();
     // Support both query parameter (?stream=true) and body parameter ({"stream": true})
     const wantsStream = Boolean(req.query?.stream === 'true' || req.body?.stream);
     const hasTools = Array.isArray(req.body?.tools) && req.body.tools.length > 0;
+    timer.mark("parseRequest");
 
     // Analyze complexity for routing headers (Phase 3)
-    const complexity = analyzeComplexity(req.body);
+    const complexity = await analyzeComplexity(req.body);
+    timer.mark("analyzeComplexity");
     let preRouteProvider = 'cloud';
     if (complexity.recommendation === 'local') {
       // Use tier config to determine actual provider instead of hardcoding 'ollama'
@@ -430,6 +434,7 @@ router.post("/v1/messages", rateLimiter, async (req, res, next) => {
     }
 
     // Non-streaming or tool-based requests (buffered path)
+    timer.mark("preProcessMessage");
     const result = await processMessage({
       payload: req.body,
       headers: req.headers,
@@ -440,6 +445,8 @@ router.post("/v1/messages", rateLimiter, async (req, res, next) => {
         maxDurationMs: req.body?.max_duration_ms,
       },
     });
+    timer.mark("processMessage");
+    timer.done();
 
     // Legacy streaming wrapper (for tool-based requests that requested streaming)
     if (wantsStream && hasTools) {

package/src/clients/circuit-breaker.js

@@ -1,253 +1,16 @@
-const logger = require("../logger");
-
 /**
- * Circuit Breaker Pattern
- *
- * States:
- * - CLOSED: Normal operation, requests pass through
- * - OPEN: Failures exceeded threshold, requests fail fast
- * - HALF_OPEN: Testing if service recovered
+ * Circuit Breaker Pattern — backed by Cockatiel
  *
- * Performance:
- * - Fail fast instead of waiting for timeouts
- * - Automatic recovery testing
- * - Minimal overhead in CLOSED state
- */
-
-const STATE = {
-  CLOSED: "CLOSED",
-  OPEN: "OPEN",
-  HALF_OPEN: "HALF_OPEN",
-};
-
-class CircuitBreaker {
-  constructor(name, options = {}) {
-    this.name = name;
-
-    // Configuration
-    this.failureThreshold = options.failureThreshold || 5; // failures before opening
-    this.successThreshold = options.successThreshold || 2; // successes to close from half-open
-    this.timeout = options.timeout || 60000; // time to wait before trying again (60s)
-    this.resetTimeout = options.resetTimeout || 30000; // time in half-open before resetting (30s)
-
-    // State
-    this.state = STATE.CLOSED;
-    this.failureCount = 0;
-    this.successCount = 0;
-    this.nextAttempt = Date.now();
-    this.lastStateChange = Date.now();
-
-    // Metrics
-    this.stats = {
-      totalRequests: 0,
-      totalFailures: 0,
-      totalSuccesses: 0,
-      totalRejected: 0,
-    };
-  }
-
-  /**
-   * Execute function with circuit breaker protection
-   */
-  async execute(fn) {
-    this.stats.totalRequests++;
-
-    // Check circuit state
-    if (this.state === STATE.OPEN) {
-      if (Date.now() < this.nextAttempt) {
-        this.stats.totalRejected++;
-        throw new CircuitBreakerError(
-          `Circuit breaker ${this.name} is OPEN`,
-          this.nextAttempt - Date.now()
-        );
-      }
-
-      // Try half-open
-      this.transitionTo(STATE.HALF_OPEN);
-    }
-
-    try {
-      const result = await fn();
-
-      // Success
-      this.onSuccess();
-      return result;
-    } catch (err) {
-      // Failure
-      this.onFailure();
-      throw err;
-    }
-  }
-
-  /**
-   * Handle successful request
-   */
-  onSuccess() {
-    this.stats.totalSuccesses++;
-    this.failureCount = 0;
-
-    if (this.state === STATE.HALF_OPEN) {
-      this.successCount++;
-
-      if (this.successCount >= this.successThreshold) {
-        this.transitionTo(STATE.CLOSED);
-      }
-    }
-  }
-
-  /**
-   * Handle failed request
-   */
-  onFailure() {
-    this.stats.totalFailures++;
-    this.failureCount++;
-    this.successCount = 0;
-
-    if (this.failureCount >= this.failureThreshold) {
-      this.transitionTo(STATE.OPEN);
-    }
-  }
-
-  /**
-   * Transition to new state
-   */
-  transitionTo(newState) {
-    const oldState = this.state;
-
-    if (oldState === newState) {
-      return;
-    }
-
-    this.state = newState;
-    this.lastStateChange = Date.now();
-
-    logger.info(
-      {
-        circuitBreaker: this.name,
-        oldState,
-        newState,
-        failureCount: this.failureCount,
-        successCount: this.successCount,
-      },
-      "Circuit breaker state change"
-    );
-
-    // Set next attempt time when opening
-    if (newState === STATE.OPEN) {
-      this.nextAttempt = Date.now() + this.timeout;
-      logger.warn(
-        {
-          circuitBreaker: this.name,
-          retryAfter: this.timeout,
-        },
-        "Circuit breaker opened - failing fast"
-      );
-    }
-
-    // Reset counters
-    if (newState === STATE.CLOSED) {
-      this.failureCount = 0;
-      this.successCount = 0;
-      logger.info(
-        {
-          circuitBreaker: this.name,
-        },
-        "Circuit breaker closed - normal operation resumed"
-      );
-    }
-
-    if (newState === STATE.HALF_OPEN) {
-      this.successCount = 0;
-      logger.info(
-        {
-          circuitBreaker: this.name,
-        },
-        "Circuit breaker half-open - testing service recovery"
-      );
-    }
-  }
-
-  /**
-   * Get current state
-   */
-  getState() {
-    return {
-      name: this.name,
-      state: this.state,
-      failureCount: this.failureCount,
-      successCount: this.successCount,
-      nextAttempt: this.nextAttempt,
-      lastStateChange: this.lastStateChange,
-      stats: this.stats,
-    };
-  }
-
-  /**
-   * Manually reset circuit breaker
-   */
-  reset() {
-    this.transitionTo(STATE.CLOSED);
-    this.failureCount = 0;
-    this.successCount = 0;
-  }
-}
-
-/**
- * Circuit breaker error
+ * This module re-exports Cockatiel-backed adapters from resilience.js
+ * while preserving the same API surface for all consumers.
  */
-class CircuitBreakerError extends Error {
-  constructor(message, retryAfter) {
-    super(message);
-    this.name = "CircuitBreakerError";
-    this.retryAfter = retryAfter;
-    this.code = "circuit_breaker_open";
-  }
-}
-
-/**
- * Circuit breaker registry
- */
-class CircuitBreakerRegistry {
-  constructor() {
-    this.breakers = new Map();
-  }
-
-  /**
-   * Get or create circuit breaker
-   */
-  get(name, options) {
-    if (!this.breakers.has(name)) {
-      this.breakers.set(name, new CircuitBreaker(name, options));
-    }
-    return this.breakers.get(name);
-  }
-
-  /**
-   * Get all circuit breakers
-   */
-  getAll() {
-    return Array.from(this.breakers.values()).map((breaker) => breaker.getState());
-  }
-
-  /**
-   * Reset all circuit breakers
-   */
-  resetAll() {
-    for (const breaker of this.breakers.values()) {
-      breaker.reset();
-    }
-  }
-}
-
-// Singleton registry
-let registry = null;
-
-function getCircuitBreakerRegistry() {
-  if (!registry) {
-    registry = new CircuitBreakerRegistry();
-  }
-  return registry;
-}
+const {
+  CockatielCircuitBreaker: CircuitBreaker,
+  CircuitBreakerError,
+  CockatielRegistry: CircuitBreakerRegistry,
+  getCockatielRegistry: getCircuitBreakerRegistry,
+  STATE,
+} = require("./resilience");
 
 module.exports = {
   CircuitBreaker,