lynkr 9.1.2 → 9.1.3

package/src/routing/index.js

@@ -22,16 +22,59 @@ const {
 const { getAgenticDetector, AGENT_TYPES } = require('./agentic-detector');
 const { getModelTierSelector, TIER_DEFINITIONS } = require('./model-tiers');
 const { getCostOptimizer } = require('./cost-optimizer');
-const { analyzeRisk } = require('./risk-analyzer');
+const { analyzeRisk } = require('./risk-classifier');
+
+// Phase 3-6 routing modules
+const { getKnnRouter } = require('./knn-router');
+const { getBandit } = require('./bandit');
+const { getShadowPolicy, compareAndLog: shadowCompareAndLog } = require('./shadow-mode');
+const { chooseFastest } = require('./deadline');
+const { applyTenantOverrides } = require('./tenant-policy');
 
 // Telemetry modules
 const telemetry = require('./telemetry');
 const { scoreResponseQuality } = require('./quality-scorer');
 const { getLatencyTracker } = require('./latency-tracker');
 
+// Phase 1 modules
+const contextValidator = require('./context-validator');
+const { countPayloadTokens } = require('./tokenizer');
+
 // Local providers
 const LOCAL_PROVIDERS = ['ollama', 'llamacpp', 'lmstudio'];
 
+/**
+ * Returns true when any message content block is an image.
+ * Handles both string content and structured content arrays.
+ */
+function _payloadHasImages(payload) {
+  const messages = payload?.messages;
+  if (!Array.isArray(messages)) return false;
+  return messages.some(msg => {
+    const content = msg?.content;
+    if (!Array.isArray(content)) return false;
+    return content.some(block => block?.type === 'image' || block?.type === 'image_url');
+  });
+}
+
+/**
+ * List of providers that currently have credentials configured.
+ * Used by the Phase 1.2 cost-optimizer override to scope candidates.
+ */
+function _enabledProviders() {
+  const out = [];
+  if (config.databricks?.url && config.databricks?.apiKey) out.push('databricks');
+  if (config.azureAnthropic?.endpoint && config.azureAnthropic?.apiKey) out.push('azure-anthropic');
+  if (config.bedrock?.apiKey) out.push('bedrock');
+  if (config.openrouter?.apiKey) out.push('openrouter');
+  if (config.openai?.apiKey) out.push('openai');
+  if (config.azureOpenAI?.endpoint && config.azureOpenAI?.apiKey) out.push('azure-openai');
+  if (config.ollama?.endpoint) out.push('ollama');
+  if (config.llamacpp?.endpoint) out.push('llamacpp');
+  if (config.lmstudio?.endpoint) out.push('lmstudio');
+  return out;
+}
+
 /**
  * Check if a provider is local
  */
@@ -41,15 +84,28 @@ function isLocalProvider(provider) {
 
 /**
  * Check if fallback is enabled
+ * In tier routing mode, fallback is always enabled
  */
 function isFallbackEnabled() {
+  if (config.modelTiers?.enabled) {
+    // Tier routing mode: fallback always enabled
+    return true;
+  }
+  // Static provider mode: use FALLBACK_ENABLED
   return config.modelProvider?.fallbackEnabled !== false;
 }
 
 /**
  * Get the configured fallback provider
+ * In tier routing mode, fallback = TIER_REASONING provider
  */
 function getFallbackProvider() {
+  if (config.modelTiers?.enabled && config.modelTiers?.REASONING) {
+    // Tier routing mode: extract provider from TIER_REASONING
+    const match = config.modelTiers.REASONING.match(/^([a-z-]+):/);
+    if (match) return match[1];
+  }
+  // Static provider mode: use FALLBACK_PROVIDER
   return config.modelProvider?.fallbackProvider ?? 'databricks';
 }
 
@@ -283,9 +339,11 @@ async function determineProviderSmart(payload, options = {}) {
     }
   }
 
-  // Apply routing decision based on tier config (TIER_* env vars are mandatory)
+  // Apply routing decision based on tier config (TIER_* env vars take precedence
+  // but Phase 1.2 lets the cost-optimizer pick a cheaper qualifying model when safe).
   let provider;
   let method = 'tier_config';
+  let costOptimized = false;
 
   const selector = getModelTierSelector();
   const modelSelection = selector.selectModel(tier, null);
@@ -294,8 +352,242 @@ async function determineProviderSmart(payload, options = {}) {
   selectedModel = modelSelection.model;
   logger.debug({ tier, provider, model: selectedModel }, '[Routing] Using tier config');
 
-  // TIER_* env vars are the final word — no cost optimization override.
-  // The user explicitly configured provider:model per tier; respect that.
+  // Phase 1.2 — cost-optimizer override.
+  // Only kick in when:
+  //  - feature flag enabled (default true, disable with LYNKR_COST_OPTIMIZE=false)
+  //  - risk level is not high (high-risk keeps the explicitly-configured model)
+  //  - the optimizer finds a meaningfully cheaper qualifying model
+  const costOptimizeEnabled = process.env.LYNKR_COST_OPTIMIZE !== 'false'
+    && config.routing?.costOptimize !== false;
+  if (costOptimizeEnabled && risk?.level !== 'high') {
+    try {
+      const optimizer = getCostOptimizer();
+      const availableProviders = _enabledProviders();
+      const cheapest = optimizer.findCheapestForTier(tier, availableProviders);
+      if (cheapest && cheapest.model && cheapest.model !== selectedModel) {
+        const current = optimizer.estimateCost(selectedModel, 1000);
+        const candidate = optimizer.estimateCost(cheapest.model, 1000);
+        if (candidate.totalEstimate > 0 && candidate.totalEstimate < current.totalEstimate * 0.75) {
+          logger.debug({
+            tier,
+            from: `${provider}:${selectedModel}`,
+            to: `${cheapest.provider}:${cheapest.model}`,
+            savedPerK: (current.totalEstimate - candidate.totalEstimate).toFixed(6),
+          }, '[Routing] Cost-optimizer override');
+          provider = cheapest.provider;
+          selectedModel = cheapest.model;
+          method = 'tier_config+cost_optimized';
+          costOptimized = true;
+        }
+      }
+    } catch (err) {
+      logger.debug({ err: err.message }, '[Routing] Cost-optimize failed, keeping tier_config selection');
+    }
+  }
+
+  // Phase 1.3 — context window validation. If estimated tokens exceed the
+  // selected model's context (with response headroom), escalate to a
+  // context-capable model regardless of tier.
+  try {
+    const estimatedTokens = countPayloadTokens(payload, selectedModel);
+    const ctxResult = contextValidator.validate(selectedModel, estimatedTokens);
+    if (!ctxResult.ok) {
+      const capable = selector.findContextCapable(estimatedTokens, tier);
+      if (capable) {
+        logger.info({
+          from: `${provider}:${selectedModel}`,
+          to: `${capable.provider}:${capable.model}`,
+          required: estimatedTokens,
+          oldContext: ctxResult.context,
+          newContext: capable.context,
+        }, '[Routing] Context window escalation');
+        provider = capable.provider;
+        selectedModel = capable.model;
+        if (capable.tier) tier = capable.tier;
+        method = method + '+context_escalated';
+      } else {
+        logger.warn({
+          model: selectedModel,
+          required: estimatedTokens,
+          available: ctxResult.context,
+        }, '[Routing] No context-capable fallback — request may fail upstream');
+      }
+    }
+  } catch (err) {
+    logger.debug({ err: err.message }, '[Routing] Context validation failed, proceeding without check');
+  }
+
+  // Phase 1.4 — vision capability guard.
+  // If the payload contains image content blocks but the selected model lacks
+  // vision support, silently swap to the cheapest vision-capable model at or
+  // above the current tier. Prevents silent upstream failures.
+  if (_payloadHasImages(payload)) {
+    try {
+      const { getModelRegistrySync } = require('./model-registry');
+      const registry = getModelRegistrySync();
+      const modelInfo = registry.getCost(selectedModel);
+      if (!modelInfo?.vision) {
+        const visionModel = selector.findVisionCapable(tier);
+        if (visionModel) {
+          logger.info({
+            from: `${provider}:${selectedModel}`,
+            to: `${visionModel.provider}:${visionModel.model}`,
+            tier: visionModel.tier,
+          }, '[Routing] Vision guard — upgrading to vision-capable model');
+          provider = visionModel.provider;
+          selectedModel = visionModel.model;
+          if (visionModel.tier !== tier) tier = visionModel.tier;
+          method = method + '+vision_guard';
+        } else {
+          logger.warn({ model: selectedModel }, '[Routing] Vision guard — no vision-capable model found, request may fail');
+        }
+      }
+    } catch (err) {
+      logger.debug({ err: err.message }, '[Routing] Vision guard check failed, proceeding');
+    }
+  }
+
+  // Phase 3.1 — kNN routing hint.
+  // If the index has enough entries, query it with the last user message.
+  // A high-confidence kNN suggestion overrides the heuristic selection.
+  let knnResult = null;
+  if (config.routing?.knnEnabled !== false) {
+    try {
+      const msgs = payload?.messages;
+      const lastMsg = Array.isArray(msgs) ? msgs[msgs.length - 1]?.content : null;
+      const queryText = typeof lastMsg === 'string' ? lastMsg
+        : Array.isArray(lastMsg) ? lastMsg.filter(b => b?.type === 'text').map(b => b.text || '').join(' ')
+        : null;
+      if (queryText) {
+        knnResult = await getKnnRouter().query(queryText);
+        if (knnResult && knnResult.confidence > 0.7 && knnResult.model && knnResult.model !== selectedModel) {
+          // High confidence — trust kNN's model recommendation directly.
+          logger.debug({
+            from: `${provider}:${selectedModel}`,
+            to: `${knnResult.provider}:${knnResult.model}`,
+            confidence: knnResult.confidence.toFixed(3),
+          }, '[Routing] kNN override');
+          provider = knnResult.provider;
+          selectedModel = knnResult.model;
+          method = method + '+knn';
+        } else if (knnResult && knnResult.confidence > 0.4 && knnResult.confidence <= 0.7) {
+          // Ambiguous signal — neighbors are split, we can't trust any single model
+          // recommendation. Err on quality: bump the current tier one step up so the
+          // request gets a more capable model rather than risking a bad answer from
+          // a model that was borderline for similar past requests.
+          const TIER_ORDER = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING'];
+          const currentIdx = TIER_ORDER.indexOf(tier);
+          if (currentIdx >= 0 && currentIdx < TIER_ORDER.length - 1) {
+            const upgradedTier = TIER_ORDER[currentIdx + 1];
+            try {
+              const upgraded = selector.selectModel(upgradedTier, null);
+              logger.debug({
+                from: `${tier}:${provider}:${selectedModel}`,
+                to: `${upgradedTier}:${upgraded.provider}:${upgraded.model}`,
+                confidence: knnResult.confidence.toFixed(3),
+              }, '[Routing] kNN ambiguous — escalating tier for safety');
+              provider = upgraded.provider;
+              selectedModel = upgraded.model;
+              tier = upgradedTier;
+              method = method + '+knn_ambiguous_escalate';
+            } catch (err) {
+              logger.debug({ err: err.message }, '[Routing] kNN ambiguous escalation failed, keeping current tier');
+            }
+          }
+        }
+      }
+    } catch (err) {
+      logger.debug({ err: err.message }, '[Routing] kNN query failed, ignoring');
+    }
+  }
+
+  // Phase 4.1 — LinUCB bandit intra-tier selection.
+  // When there are two candidates (heuristic vs kNN), the bandit picks the
+  // one with the highest estimated UCB score for the current context.
+  if (config.routing?.banditEnabled !== false && knnResult && knnResult.model) {
+    try {
+      // Build candidates: current selection and kNN alternative if different
+      const allCandidates = [{ provider, model: selectedModel }];
+      if (knnResult.model !== selectedModel) {
+        allCandidates.push({ provider: knnResult.provider, model: knnResult.model });
+      }
+
+      if (allCandidates.length > 1) {
+        const bandit = getBandit();
+        const TASK_TYPES = ['code_gen', 'summarization', 'reasoning', 'factoid', 'chat', 'other'];
+        const inferredTask = (analysis.breakdown?.taskType?.reason || 'other').toLowerCase();
+        const taskIdx = Math.max(0, TASK_TYPES.findIndex(t => inferredTask.includes(t)));
+        const ctx = [
+          (analysis.score || 0) / 100,
+          Math.log(Math.max(1, analysis.breakdown?.tokenCount || 0) + 1) / 15,
+          ((payload?.tools?.length ?? 0) > 0) ? 1 : 0,
+          options.streaming ? 1 : 0,
+          risk?.level === 'high' ? 1 : risk?.level === 'medium' ? 0.5 : 0,
+          agenticResult?.isAgentic ? 1 : 0,
+          ...TASK_TYPES.map((_, i) => i === taskIdx ? 1 : 0),
+        ];
+        const picked = bandit.pick(tier, allCandidates, ctx);
+        if (picked && picked.model !== selectedModel) {
+          logger.debug({
+            from: `${provider}:${selectedModel}`,
+            to: `${picked.provider}:${picked.model}`,
+            ucb: picked.ucb?.toFixed(4),
+            explored: picked.explored,
+          }, '[Routing] Bandit override');
+          provider = picked.provider;
+          selectedModel = picked.model;
+          method = method + (picked.explored ? '+bandit_explore' : '+bandit');
+        }
+      }
+    } catch (err) {
+      logger.debug({ err: err.message }, '[Routing] Bandit pick failed, ignoring');
+    }
+  }
+
+  // Phase 6.3 — deadline-aware fastest-model selection.
+  // Payload carries _deadlineMs injected by the orchestrator from the
+  // LYNKR-Deadline-Ms request header.
+  const deadlineMs = payload?._deadlineMs ?? null;
+  if (deadlineMs) {
+    try {
+      const fastest = chooseFastest([{ provider, model: selectedModel }], deadlineMs);
+      if (fastest && fastest.model !== selectedModel) {
+        logger.debug({
+          from: `${provider}:${selectedModel}`,
+          to: `${fastest.provider}:${fastest.model}`,
+          deadlineMs,
+        }, '[Routing] Deadline override');
+        provider = fastest.provider;
+        selectedModel = fastest.model;
+        method = method + '+deadline';
+      }
+    } catch (err) {
+      logger.debug({ err: err.message }, '[Routing] Deadline check failed, ignoring');
+    }
+  }
+
+  // Phase 6.1 — per-tenant policy overrides.
+  // tenantPolicy comes from options (threaded from Express res.locals via
+  // orchestrator → databricks → here).
+  if (options.tenantPolicy) {
+    try {
+      const overridden = applyTenantOverrides(
+        { provider, model: selectedModel, tier, method },
+        options.tenantPolicy,
+      );
+      if (overridden && overridden.model !== selectedModel) {
+        logger.debug({
+          from: `${provider}:${selectedModel}`,
+          to: `${overridden.provider}:${overridden.model}`,
+        }, '[Routing] Tenant override');
+        provider = overridden.provider;
+        selectedModel = overridden.model;
+        method = overridden.method;
+      }
+    } catch (err) {
+      logger.debug({ err: err.message }, '[Routing] Tenant override failed, ignoring');
+    }
+  }
 
   const decision = {
     provider,
@@ -309,10 +601,19 @@ async function determineProviderSmart(payload, options = {}) {
     analysis,
     embeddingsResult,
     agenticResult,
-    costOptimized: false,
+    costOptimized,
     risk,
+    knnResult,
   };
 
+  // Phase 4.4 — shadow-mode policy comparison (fire-and-forget).
+  const shadowFn = getShadowPolicy();
+  if (shadowFn) {
+    setImmediate(() =>
+      shadowCompareAndLog({ payload, activeDecision: decision, shadowFn }).catch(() => {})
+    );
+  }
+
   // Phase 3: Record metrics
   routingMetrics.record(decision);
 
@@ -419,6 +720,14 @@ module.exports = {
   AGENT_TYPES,
   TIER_DEFINITIONS,
 
+  // Phase 3-6 modules
+  getKnnRouter,
+  getBandit,
+  getShadowPolicy,
+  shadowCompareAndLog,
+  chooseFastest,
+  applyTenantOverrides,
+
   // Telemetry
   telemetry,
   scoreResponseQuality,

package/src/routing/knn-router.js

@@ -0,0 +1,206 @@
+/**
+ * kNN-based routing decision (Phase 3.1).
+ *
+ * Embeds the incoming query, finds the K nearest historical queries from the
+ * hnswlib-node index, and returns a confidence-weighted recommendation
+ * (model, expected quality, expected cost) based on those neighbors' actual
+ * outcomes from telemetry.
+ *
+ * Behavior:
+ *   - Empty index → returns null. Caller falls back to heuristic router.
+ *   - Sparse index (N < MIN_INDEX_SIZE) → returns null. Heuristic wins until
+ *     we have enough data to be confident.
+ *   - Embedder unavailable → returns null. Same fallback path.
+ *
+ * Bootstrap: scripts/build-knn-index.js (also accepts optional RouterBench
+ * corpus path to seed the index).
+ */
+
+const fs = require('fs');
+const path = require('path');
+const logger = require('../logger');
+const { generateEmbedding } = require('../cache/embeddings');
+const { getEmbeddingCache } = require('./embedding-cache');
+
+const INDEX_DIR = path.join(__dirname, '../../data/knn');
+const INDEX_FILE = path.join(INDEX_DIR, 'index.hnsw');
+const META_FILE = path.join(INDEX_DIR, 'meta.json');
+
+const MAX_ELEMENTS = 50000;
+const DIM = 768; // nomic-embed-text default
+const K = 10;
+const MIN_INDEX_SIZE = 1000;
+
+let _hnsw = null;
+let _hnswLoaded = false;
+function _loadHnsw() {
+  if (_hnswLoaded) return _hnsw;
+  _hnswLoaded = true;
+  try {
+    _hnsw = require('hnswlib-node');
+  } catch (err) {
+    logger.debug({ err: err.message }, '[KnnRouter] hnswlib-node not available');
+    _hnsw = null;
+  }
+  return _hnsw;
+}
+
+class KnnRouter {
+  constructor() {
+    this.index = null;
+    this.meta = []; // parallel to index: per-id outcome { query, model, quality, cost, latency, tier }
+    this.size = 0;
+    this.dim = DIM;
+    this.ready = false;
+  }
+
+  load() {
+    const hnsw = _loadHnsw();
+    if (!hnsw) return false;
+    try {
+      if (!fs.existsSync(INDEX_FILE) || !fs.existsSync(META_FILE)) {
+        // Initialize empty index (caller can add() later)
+        this.index = new hnsw.HierarchicalNSW('cosine', this.dim);
+        this.index.initIndex(MAX_ELEMENTS);
+        this.meta = [];
+        this.size = 0;
+        this.ready = true;
+        return true;
+      }
+      const metaData = JSON.parse(fs.readFileSync(META_FILE, 'utf8'));
+      this.dim = metaData.dim || DIM;
+      this.meta = metaData.entries || [];
+      this.size = this.meta.length;
+      this.index = new hnsw.HierarchicalNSW('cosine', this.dim);
+      this.index.readIndexSync(INDEX_FILE, MAX_ELEMENTS);
+      this.ready = true;
+      logger.info({ size: this.size, dim: this.dim }, '[KnnRouter] Index loaded');
+      return true;
+    } catch (err) {
+      logger.warn({ err: err.message }, '[KnnRouter] Index load failed');
+      return false;
+    }
+  }
+
+  save() {
+    if (!this.ready || !this.index) return;
+    try {
+      fs.mkdirSync(INDEX_DIR, { recursive: true });
+      this.index.writeIndexSync(INDEX_FILE);
+      fs.writeFileSync(META_FILE, JSON.stringify({ dim: this.dim, entries: this.meta }, null, 0));
+    } catch (err) {
+      logger.warn({ err: err.message }, '[KnnRouter] Index save failed');
+    }
+  }
+
+  add(embedding, outcome) {
+    if (!this.ready || !this.index || !Array.isArray(embedding)) return;
+    if (this.size >= MAX_ELEMENTS) {
+      // Simple FIFO eviction: drop the oldest meta and reuse its id
+      // hnswlib doesn't support deletion in place; we just stop adding past max
+      return;
+    }
+    this.index.addPoint(embedding, this.size);
+    this.meta.push(outcome);
+    this.size++;
+  }
+
+  async query(text) {
+    if (!this.ready) this.load();
+    if (!this.ready || !this.index || this.size < MIN_INDEX_SIZE) return null;
+    if (!text || typeof text !== 'string') return null;
+
+    const cache = getEmbeddingCache();
+    let embedding = cache.get(text);
+    if (!embedding) {
+      try {
+        embedding = await generateEmbedding(text);
+        if (!embedding || embedding.length !== this.dim) {
+          // Skip if dim mismatch (embedder produced different dimensions)
+          return null;
+        }
+        cache.set(text, embedding);
+      } catch (err) {
+        logger.debug({ err: err.message }, '[KnnRouter] Embedding failed, skipping');
+        return null;
+      }
+    }
+
+    let result;
+    try {
+      result = this.index.searchKnn(embedding, K);
+    } catch (err) {
+      logger.debug({ err: err.message }, '[KnnRouter] Search failed');
+      return null;
+    }
+
+    const neighbors = (result.neighbors || []).map((id, i) => ({
+      id,
+      distance: result.distances?.[i] ?? 1,
+      outcome: this.meta[id],
+    })).filter(n => n.outcome);
+
+    if (neighbors.length === 0) return null;
+
+    // Confidence-weighted aggregation per candidate model.
+    // weight = 1 - distance (cosine distance → similarity)
+    const byModel = new Map();
+    for (const n of neighbors) {
+      const w = Math.max(0, 1 - n.distance);
+      const m = `${n.outcome.provider}:${n.outcome.model}`;
+      if (!byModel.has(m)) {
+        byModel.set(m, { weight: 0, quality: 0, cost: 0, latency: 0, count: 0, sample: n.outcome });
+      }
+      const agg = byModel.get(m);
+      agg.weight += w;
+      agg.quality += w * (n.outcome.quality || 50);
+      agg.cost += w * (n.outcome.cost || 0);
+      agg.latency += w * (n.outcome.latency || 0);
+      agg.count++;
+    }
+
+    let best = null;
+    let bestScore = -Infinity;
+    for (const [model, agg] of byModel) {
+      const avgQ = agg.quality / agg.weight;
+      const avgC = agg.cost / agg.weight;
+      // Score = quality / log(cost+1) — reward quality, penalise cost gently
+      const score = avgQ / Math.log(avgC * 1000 + 2);
+      if (score > bestScore) {
+        bestScore = score;
+        best = {
+          provider: agg.sample.provider,
+          model: agg.sample.model,
+          tier: agg.sample.tier,
+          expectedQuality: avgQ,
+          expectedCost: avgC,
+          expectedLatency: agg.latency / agg.weight,
+          confidence: Math.min(1, agg.weight / K),
+          neighborCount: agg.count,
+        };
+      }
+    }
+
+    return best;
+  }
+
+  getStats() {
+    return {
+      size: this.size,
+      maxElements: MAX_ELEMENTS,
+      ready: this.ready,
+      dim: this.dim,
+    };
+  }
+}
+
+let _instance = null;
+function getKnnRouter() {
+  if (!_instance) {
+    _instance = new KnnRouter();
+    _instance.load();
+  }
+  return _instance;
+}
+
+module.exports = { KnnRouter, getKnnRouter };