lynkr 9.0.2 → 9.1.3

package/src/routing/knn-router.js

@@ -0,0 +1,206 @@
+/**
+ * kNN-based routing decision (Phase 3.1).
+ *
+ * Embeds the incoming query, finds the K nearest historical queries from the
+ * hnswlib-node index, and returns a confidence-weighted recommendation
+ * (model, expected quality, expected cost) based on those neighbors' actual
+ * outcomes from telemetry.
+ *
+ * Behavior:
+ *   - Empty index → returns null. Caller falls back to heuristic router.
+ *   - Sparse index (N < MIN_INDEX_SIZE) → returns null. Heuristic wins until
+ *     we have enough data to be confident.
+ *   - Embedder unavailable → returns null. Same fallback path.
+ *
+ * Bootstrap: scripts/build-knn-index.js (also accepts optional RouterBench
+ * corpus path to seed the index).
+ */
+
+const fs = require('fs');
+const path = require('path');
+const logger = require('../logger');
+const { generateEmbedding } = require('../cache/embeddings');
+const { getEmbeddingCache } = require('./embedding-cache');
+
+const INDEX_DIR = path.join(__dirname, '../../data/knn');
+const INDEX_FILE = path.join(INDEX_DIR, 'index.hnsw');
+const META_FILE = path.join(INDEX_DIR, 'meta.json');
+
+const MAX_ELEMENTS = 50000;
+const DIM = 768; // nomic-embed-text default
+const K = 10;
+const MIN_INDEX_SIZE = 1000;
+
+let _hnsw = null;
+let _hnswLoaded = false;
+function _loadHnsw() {
+  if (_hnswLoaded) return _hnsw;
+  _hnswLoaded = true;
+  try {
+    _hnsw = require('hnswlib-node');
+  } catch (err) {
+    logger.debug({ err: err.message }, '[KnnRouter] hnswlib-node not available');
+    _hnsw = null;
+  }
+  return _hnsw;
+}
+
+class KnnRouter {
+  constructor() {
+    this.index = null;
+    this.meta = []; // parallel to index: per-id outcome { query, model, quality, cost, latency, tier }
+    this.size = 0;
+    this.dim = DIM;
+    this.ready = false;
+  }
+
+  load() {
+    const hnsw = _loadHnsw();
+    if (!hnsw) return false;
+    try {
+      if (!fs.existsSync(INDEX_FILE) || !fs.existsSync(META_FILE)) {
+        // Initialize empty index (caller can add() later)
+        this.index = new hnsw.HierarchicalNSW('cosine', this.dim);
+        this.index.initIndex(MAX_ELEMENTS);
+        this.meta = [];
+        this.size = 0;
+        this.ready = true;
+        return true;
+      }
+      const metaData = JSON.parse(fs.readFileSync(META_FILE, 'utf8'));
+      this.dim = metaData.dim || DIM;
+      this.meta = metaData.entries || [];
+      this.size = this.meta.length;
+      this.index = new hnsw.HierarchicalNSW('cosine', this.dim);
+      this.index.readIndexSync(INDEX_FILE, MAX_ELEMENTS);
+      this.ready = true;
+      logger.info({ size: this.size, dim: this.dim }, '[KnnRouter] Index loaded');
+      return true;
+    } catch (err) {
+      logger.warn({ err: err.message }, '[KnnRouter] Index load failed');
+      return false;
+    }
+  }
+
+  save() {
+    if (!this.ready || !this.index) return;
+    try {
+      fs.mkdirSync(INDEX_DIR, { recursive: true });
+      this.index.writeIndexSync(INDEX_FILE);
+      fs.writeFileSync(META_FILE, JSON.stringify({ dim: this.dim, entries: this.meta }, null, 0));
+    } catch (err) {
+      logger.warn({ err: err.message }, '[KnnRouter] Index save failed');
+    }
+  }
+
+  add(embedding, outcome) {
+    if (!this.ready || !this.index || !Array.isArray(embedding)) return;
+    if (this.size >= MAX_ELEMENTS) {
+      // Simple FIFO eviction: drop the oldest meta and reuse its id
+      // hnswlib doesn't support deletion in place; we just stop adding past max
+      return;
+    }
+    this.index.addPoint(embedding, this.size);
+    this.meta.push(outcome);
+    this.size++;
+  }
+
+  async query(text) {
+    if (!this.ready) this.load();
+    if (!this.ready || !this.index || this.size < MIN_INDEX_SIZE) return null;
+    if (!text || typeof text !== 'string') return null;
+
+    const cache = getEmbeddingCache();
+    let embedding = cache.get(text);
+    if (!embedding) {
+      try {
+        embedding = await generateEmbedding(text);
+        if (!embedding || embedding.length !== this.dim) {
+          // Skip if dim mismatch (embedder produced different dimensions)
+          return null;
+        }
+        cache.set(text, embedding);
+      } catch (err) {
+        logger.debug({ err: err.message }, '[KnnRouter] Embedding failed, skipping');
+        return null;
+      }
+    }
+
+    let result;
+    try {
+      result = this.index.searchKnn(embedding, K);
+    } catch (err) {
+      logger.debug({ err: err.message }, '[KnnRouter] Search failed');
+      return null;
+    }
+
+    const neighbors = (result.neighbors || []).map((id, i) => ({
+      id,
+      distance: result.distances?.[i] ?? 1,
+      outcome: this.meta[id],
+    })).filter(n => n.outcome);
+
+    if (neighbors.length === 0) return null;
+
+    // Confidence-weighted aggregation per candidate model.
+    // weight = 1 - distance (cosine distance → similarity)
+    const byModel = new Map();
+    for (const n of neighbors) {
+      const w = Math.max(0, 1 - n.distance);
+      const m = `${n.outcome.provider}:${n.outcome.model}`;
+      if (!byModel.has(m)) {
+        byModel.set(m, { weight: 0, quality: 0, cost: 0, latency: 0, count: 0, sample: n.outcome });
+      }
+      const agg = byModel.get(m);
+      agg.weight += w;
+      agg.quality += w * (n.outcome.quality || 50);
+      agg.cost += w * (n.outcome.cost || 0);
+      agg.latency += w * (n.outcome.latency || 0);
+      agg.count++;
+    }
+
+    let best = null;
+    let bestScore = -Infinity;
+    for (const [model, agg] of byModel) {
+      const avgQ = agg.quality / agg.weight;
+      const avgC = agg.cost / agg.weight;
+      // Score = quality / log(cost+1) — reward quality, penalise cost gently
+      const score = avgQ / Math.log(avgC * 1000 + 2);
+      if (score > bestScore) {
+        bestScore = score;
+        best = {
+          provider: agg.sample.provider,
+          model: agg.sample.model,
+          tier: agg.sample.tier,
+          expectedQuality: avgQ,
+          expectedCost: avgC,
+          expectedLatency: agg.latency / agg.weight,
+          confidence: Math.min(1, agg.weight / K),
+          neighborCount: agg.count,
+        };
+      }
+    }
+
+    return best;
+  }
+
+  getStats() {
+    return {
+      size: this.size,
+      maxElements: MAX_ELEMENTS,
+      ready: this.ready,
+      dim: this.dim,
+    };
+  }
+}
+
+let _instance = null;
+function getKnnRouter() {
+  if (!_instance) {
+    _instance = new KnnRouter();
+    _instance.load();
+  }
+  return _instance;
+}
+
+module.exports = { KnnRouter, getKnnRouter };

package/src/routing/latency-tracker.js

@@ -1,80 +1,78 @@
 /**
- * Rolling Latency Tracker
+ * Rolling Latency Tracker (per provider:model)
  *
- * Tracks per-provider latency using circular buffers to provide
- * P50/P95/P99 percentile statistics for routing decisions.
+ * Tracks latency keyed by `${provider}:${model}` so models within a provider
+ * (Opus vs Haiku) get separate stats. Backward-compatible: callers that pass
+ * only a provider still work — they're tracked under `${provider}:*`.
+ *
+ * Phase 1.5 of the routing overhaul: previous version keyed by provider only.
  *
  * @module routing/latency-tracker
  */
 
 const logger = require("../logger");
 
-/** Size of the circular buffer per provider */
 const BUFFER_SIZE = 200;
-
-/** Minimum sample count before penalizeScore returns a meaningful value */
 const MIN_SAMPLES = 10;
 
-/**
- * @typedef {Object} LatencyStats
- * @property {number} p50 - 50th percentile latency (ms)
- * @property {number} p95 - 95th percentile latency (ms)
- * @property {number} p99 - 99th percentile latency (ms)
- * @property {number} avg - Average latency (ms)
- * @property {number} count - Total measurements recorded
- * @property {number} lastUpdated - Timestamp of the last recorded measurement
- */
+/** Wildcard model used when caller doesn't specify one. */
+const ANY_MODEL = '*';
+
+function _key(provider, model) {
+  return `${provider}:${model || ANY_MODEL}`;
+}
 
 class LatencyTracker {
   constructor() {
-    /** @type {Map<string, { buffer: number[], index: number, count: number, lastUpdated: number }>} */
-    this._providers = new Map();
+    /** @type {Map<string, { buffer: number[], index: number, count: number, lastUpdated: number, provider: string, model: string }>} */
+    this._entries = new Map();
   }
 
   /**
-   * Record a latency measurement for a provider.
-   * @param {string} provider - Provider name (e.g. "databricks", "ollama")
-   * @param {number} latencyMs - Measured latency in milliseconds
+   * Record a latency measurement.
+   *
+   * Signatures:
+   *   record(provider, latencyMs)              // legacy
+   *   record(provider, model, latencyMs)       // preferred
    */
-  record(provider, latencyMs) {
-    if (!provider || typeof latencyMs !== "number" || latencyMs < 0) {
-      return;
+  record(provider, modelOrLatency, maybeLatency) {
+    let model;
+    let latencyMs;
+    if (typeof modelOrLatency === 'number') {
+      model = ANY_MODEL;
+      latencyMs = modelOrLatency;
+    } else {
+      model = modelOrLatency || ANY_MODEL;
+      latencyMs = maybeLatency;
     }
 
-    let entry = this._providers.get(provider);
+    if (!provider || typeof latencyMs !== "number" || latencyMs < 0) return;
+
+    const k = _key(provider, model);
+    let entry = this._entries.get(k);
     if (!entry) {
       entry = {
         buffer: new Array(BUFFER_SIZE).fill(0),
         index: 0,
         count: 0,
         lastUpdated: 0,
+        provider,
+        model,
       };
-      this._providers.set(provider, entry);
+      this._entries.set(k, entry);
     }
-
     entry.buffer[entry.index] = latencyMs;
     entry.index = (entry.index + 1) % BUFFER_SIZE;
     entry.count += 1;
     entry.lastUpdated = Date.now();
   }
 
-  /**
-   * Get latency statistics for a specific provider.
-   * @param {string} provider - Provider name
-   * @returns {LatencyStats|null} Statistics or null if no data
-   */
-  getStats(provider) {
-    const entry = this._providers.get(provider);
-    if (!entry || entry.count === 0) {
-      return null;
-    }
-
+  _computeStats(entry) {
+    if (!entry || entry.count === 0) return null;
     const sampleCount = Math.min(entry.count, BUFFER_SIZE);
     const samples = entry.buffer.slice(0, sampleCount);
     const sorted = samples.slice().sort((a, b) => a - b);
-
     const sum = sorted.reduce((acc, v) => acc + v, 0);
-
     return {
       p50: sorted[Math.floor(sampleCount * 0.5)],
       p95: sorted[Math.floor(sampleCount * 0.95)],
@@ -82,61 +80,105 @@ class LatencyTracker {
       avg: Math.round(sum / sampleCount),
       count: entry.count,
       lastUpdated: entry.lastUpdated,
+      provider: entry.provider,
+      model: entry.model,
     };
   }
 
   /**
-   * Calculate a routing score penalty/bonus based on provider latency.
-   *
-   * Returns a value from -5 to +10 that can be added to a routing score:
-   *   +10 if P95 > 10000ms (very slow, penalise by boosting complexity toward cloud)
-   *   +5  if P95 > 5000ms
-   *   -5  if P50 < 1000ms (fast, reward)
-   *    0  otherwise or if insufficient data
-   *
-   * @param {string} provider - Provider name
-   * @returns {number} Score adjustment (-5 to +10)
+   * Get stats for a specific (provider, model) pair, or aggregated for a provider
+   * if model is omitted.
    */
-  penalizeScore(provider) {
-    const stats = this.getStats(provider);
-    if (!stats || stats.count < MIN_SAMPLES) {
-      return 0;
+  getStats(provider, model = null) {
+    if (model) {
+      return this._computeStats(this._entries.get(_key(provider, model)));
+    }
+    // Aggregate across all models for this provider
+    const provEntries = [];
+    for (const [k, entry] of this._entries) {
+      if (entry.provider === provider) provEntries.push(entry);
     }
+    if (provEntries.length === 0) return null;
+    if (provEntries.length === 1) return this._computeStats(provEntries[0]);
+
+    // Pool samples across model entries to compute combined percentiles
+    const pooled = [];
+    let total = 0;
+    let lastUpdated = 0;
+    for (const e of provEntries) {
+      const n = Math.min(e.count, BUFFER_SIZE);
+      for (let i = 0; i < n; i++) pooled.push(e.buffer[i]);
+      total += e.count;
+      if (e.lastUpdated > lastUpdated) lastUpdated = e.lastUpdated;
+    }
+    if (pooled.length === 0) return null;
+    pooled.sort((a, b) => a - b);
+    const sum = pooled.reduce((acc, v) => acc + v, 0);
+    return {
+      p50: pooled[Math.floor(pooled.length * 0.5)],
+      p95: pooled[Math.floor(pooled.length * 0.95)],
+      p99: pooled[Math.floor(pooled.length * 0.99)],
+      avg: Math.round(sum / pooled.length),
+      count: total,
+      lastUpdated,
+      provider,
+      model: ANY_MODEL,
+    };
+  }
 
+  /** Latency penalty/bonus used by complexity-analyzer. */
+  penalizeScore(provider, model = null) {
+    const stats = this.getStats(provider, model);
+    if (!stats || stats.count < MIN_SAMPLES) return 0;
     if (stats.p95 > 10000) return 10;
     if (stats.p95 > 5000) return 5;
     if (stats.p50 < 1000) return -5;
-
     return 0;
   }
 
   /**
-   * Get statistics for all tracked providers.
-   * @returns {Map<string, LatencyStats>}
+   * Phase 1.5: per-model P95 lookup for deadline-aware routing (Phase 6.3).
+   * Returns null if insufficient samples.
+   */
+  getModelP95(provider, model) {
+    const stats = this.getStats(provider, model);
+    if (!stats || stats.count < MIN_SAMPLES) return null;
+    return stats.p95;
+  }
+
+  /**
+   * Whether a model is currently degraded (P95 > 2x its historical median).
+   * Currently uses a simple absolute threshold — better signal will come in
+   * Phase 4.3 (drift detection).
+   */
+  isDegraded(provider, model) {
+    const stats = this.getStats(provider, model);
+    if (!stats || stats.count < MIN_SAMPLES) return false;
+    return stats.p95 > stats.p50 * 2 && stats.p95 > 5000;
+  }
+
+  /**
+   * Get stats for every tracked entry.
+   *
+   * Backward-compat: when an entry was recorded via the legacy 2-arg
+   * `record(provider, latency)` signature, the model is the wildcard `*`
+   * and we return it keyed by provider name only. Entries with explicit
+   * models use the `provider:model` key.
    */
   getAllStats() {
     const result = new Map();
-    for (const provider of this._providers.keys()) {
-      const stats = this.getStats(provider);
-      if (stats) {
-        result.set(provider, stats);
-      }
+    for (const [k, entry] of this._entries) {
+      const stats = this._computeStats(entry);
+      if (!stats) continue;
+      const outKey = entry.model === ANY_MODEL ? entry.provider : k;
+      result.set(outKey, stats);
     }
     return result;
   }
 }
 
-// ---------------------------------------------------------------------------
-// Singleton
-// ---------------------------------------------------------------------------
-
-/** @type {LatencyTracker|null} */
 let instance = null;
 
-/**
- * Get the singleton LatencyTracker instance.
- * @returns {LatencyTracker}
- */
 function getLatencyTracker() {
   if (!instance) {
     instance = new LatencyTracker();
@@ -145,4 +187,4 @@ function getLatencyTracker() {
   return instance;
 }
 
-module.exports = { LatencyTracker, getLatencyTracker };
+module.exports = { LatencyTracker, getLatencyTracker, ANY_MODEL };

package/src/routing/model-tiers.js

@@ -12,7 +12,10 @@ const config = require('../config');
 // Load tier config
 const TIER_CONFIG_PATH = path.join(__dirname, '../../config/model-tiers.json');
 
-// Tier definitions with complexity ranges
+// Phase 1.4: calibrated thresholds (written by scripts/calibrate-thresholds.js)
+const CALIBRATED_PATH = path.join(__dirname, '../../data/calibrated-thresholds.json');
+
+// Tier definitions with complexity ranges (defaults; may be overridden by calibration)
 const TIER_DEFINITIONS = {
   SIMPLE: {
     description: 'Greetings, simple Q&A, confirmations',
@@ -41,13 +44,30 @@ class ModelTierSelector {
     this.tierConfig = null;
     this.localProviders = {};
     this.providerAliases = {};
+    /** Per-tier ranges, possibly overridden by calibration. */
+    this.ranges = null;
     this._loadConfig();
+    this._loadCalibrated();
   }
 
   /**
    * Load tier configuration from JSON file
    */
   _loadConfig() {
+    // Check if tier routing mode is active (all 4 TIER_* env vars set)
+    const tierRoutingMode = !!(
+      config.modelTiers?.SIMPLE?.trim() &&
+      config.modelTiers?.MEDIUM?.trim() &&
+      config.modelTiers?.COMPLEX?.trim() &&
+      config.modelTiers?.REASONING?.trim()
+    );
+
+    if (tierRoutingMode) {
+      logger.debug('[ModelTiers] Tier routing mode active, building config from TIER_* env vars');
+      this._buildFromEnvVars();
+      return;
+    }
+
     try {
       if (fs.existsSync(TIER_CONFIG_PATH)) {
         const data = JSON.parse(fs.readFileSync(TIER_CONFIG_PATH, 'utf8'));
@@ -65,9 +85,86 @@ class ModelTierSelector {
     }
   }
 
+  /**
+   * Phase 1.4: load calibrated tier thresholds if the nightly job has produced them.
+   * Falls back silently to TIER_DEFINITIONS when absent or malformed.
+   */
+  _loadCalibrated() {
+    this.ranges = this._defaultRanges();
+    try {
+      if (!fs.existsSync(CALIBRATED_PATH)) return;
+      const data = JSON.parse(fs.readFileSync(CALIBRATED_PATH, 'utf8'));
+      if (!data?.ranges) return;
+      const calibrated = {};
+      for (const tier of Object.keys(TIER_DEFINITIONS)) {
+        const r = data.ranges[tier];
+        if (Array.isArray(r) && r.length === 2 && r[0] <= r[1]) {
+          calibrated[tier] = r;
+        } else {
+          calibrated[tier] = TIER_DEFINITIONS[tier].range;
+        }
+      }
+      this.ranges = calibrated;
+      logger.info({ ranges: this.ranges, calibratedAt: data.calibratedAt }, '[ModelTiers] Using calibrated thresholds');
+    } catch (err) {
+      logger.debug({ err: err.message }, '[ModelTiers] Calibrated thresholds load failed; using defaults');
+    }
+  }
+
+  _defaultRanges() {
+    const ranges = {};
+    for (const [tier, def] of Object.entries(TIER_DEFINITIONS)) {
+      ranges[tier] = def.range.slice();
+    }
+    return ranges;
+  }
+
   /**
    * Load default tier config
    */
+  /**
+   * Build tier config from TIER_* environment variables
+   * Format: TIER_SIMPLE=provider:model
+   */
+  _buildFromEnvVars() {
+    this.tierConfig = {};
+    this.localProviders = {
+      ollama: { free: true, defaultTier: 'SIMPLE' },
+      llamacpp: { free: true, defaultTier: 'SIMPLE' },
+      lmstudio: { free: true, defaultTier: 'SIMPLE' },
+      mlx: { free: true, defaultTier: 'SIMPLE' },
+    };
+
+    const tiers = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING'];
+    for (const tier of tiers) {
+      const envValue = config.modelTiers?.[tier]?.trim();
+      if (!envValue) continue;
+
+      // Parse provider:model format
+      const match = envValue.match(/^([a-z-]+):(.+)$/);
+      if (!match) {
+        logger.warn({ tier, value: envValue }, '[ModelTiers] Invalid TIER format, expected provider:model');
+        continue;
+      }
+
+      const [, provider, model] = match;
+
+      // Initialize tier config if not exists
+      if (!this.tierConfig[tier]) {
+        this.tierConfig[tier] = { preferred: {} };
+      }
+
+      // Set this as the ONLY preferred model for this tier+provider
+      this.tierConfig[tier].preferred[provider] = [model];
+
+      logger.debug({
+        tier,
+        provider,
+        model
+      }, '[ModelTiers] Tier configured from env');
+    }
+  }
+
   _loadDefaults() {
     this.tierConfig = {
       SIMPLE: { preferred: { ollama: ['llama3.2'], openai: ['gpt-4o-mini'] } },
@@ -92,20 +189,73 @@ class ModelTierSelector {
   }
 
   /**
-   * Get tier from complexity score
+   * Get tier from complexity score.
+   * Phase 1.4: honors calibrated ranges when present.
    * @param {number} complexityScore - Score from 0-100
    * @returns {string} Tier name (SIMPLE, MEDIUM, COMPLEX, REASONING)
    */
   getTier(complexityScore) {
     const score = Math.max(0, Math.min(100, complexityScore || 0));
+    const ranges = this.ranges || this._defaultRanges();
+    for (const tier of Object.keys(TIER_DEFINITIONS)) {
+      const [lo, hi] = ranges[tier];
+      if (score >= lo && score <= hi) return tier;
+    }
+    return score > 75 ? 'REASONING' : 'SIMPLE';
+  }
 
-    for (const [tier, def] of Object.entries(TIER_DEFINITIONS)) {
-      if (score >= def.range[0] && score <= def.range[1]) {
-        return tier;
+  /**
+   * Phase 1.3: find a model with at least `minContext` context window.
+   * Returns null when no qualifying model is available.
+   */
+  findContextCapable(minContext, preferredTier = null) {
+    const { getModelRegistrySync } = require('./model-registry');
+    const registry = getModelRegistrySync();
+    const tierOrder = preferredTier
+      ? [preferredTier, 'REASONING', 'COMPLEX', 'MEDIUM', 'SIMPLE']
+      : ['REASONING', 'COMPLEX', 'MEDIUM', 'SIMPLE'];
+    const seen = new Set();
+    for (const tier of tierOrder) {
+      if (seen.has(tier)) continue;
+      seen.add(tier);
+      const tierConfig = this.tierConfig[tier];
+      if (!tierConfig?.preferred) continue;
+      for (const [provider, models] of Object.entries(tierConfig.preferred)) {
+        for (const model of models) {
+          const cost = registry.getCost(model);
+          if (cost?.context && cost.context >= minContext) {
+            return { provider, model, tier, context: cost.context };
+          }
+        }
       }
     }
+    return null;
+  }
 
-    return score > 75 ? 'REASONING' : 'SIMPLE';
+  /**
+   * Find a vision-capable model at or above `preferredTier`.
+   * Walks tier order from preferred upward; returns null when none available.
+   */
+  findVisionCapable(preferredTier = null) {
+    const { getModelRegistrySync } = require('./model-registry');
+    const registry = getModelRegistrySync();
+    const tierOrder = preferredTier
+      ? [preferredTier, 'COMPLEX', 'REASONING', 'MEDIUM', 'SIMPLE']
+      : ['COMPLEX', 'REASONING', 'MEDIUM', 'SIMPLE'];
+    const seen = new Set();
+    for (const t of tierOrder) {
+      if (seen.has(t)) continue;
+      seen.add(t);
+      const tierConfig = this.tierConfig[t];
+      if (!tierConfig?.preferred) continue;
+      for (const [provider, models] of Object.entries(tierConfig.preferred)) {
+        for (const model of models) {
+          const info = registry.getCost(model);
+          if (info?.vision) return { provider, model, tier: t };
+        }
+      }
+    }
+    return null;
   }
 
   /**