lynkr 9.1.2 → 9.1.4

package/src/routing/confidence-scorer.js

@@ -0,0 +1,121 @@
+/**
+ * Confidence scoring for cascade responses (Phase 3.3).
+ *
+ * Given a response from a smaller model, estimate whether it's confident
+ * enough to return as-is or whether we should escalate to a bigger model.
+ *
+ * Three strategies, picked by task type:
+ *   - Factoid: detect refusal/uncertainty markers
+ *   - Code: parse-validity check, completeness markers
+ *   - Reasoning: optional judge-LLM (heuristic fallback when judge unavailable)
+ *
+ * Returns a [0, 1] confidence score. Caller compares against a threshold
+ * (default 0.85).
+ */
+
+const logger = require('../logger');
+
+const UNCERTAINTY_MARKERS = [
+  /\bi don't know\b/i,
+  /\bi'm not sure\b/i,
+  /\bi cannot\b/i,
+  /\bi am unable\b/i,
+  /\bunable to\b/i,
+  /\bnot certain\b/i,
+  /\bunclear\b/i,
+  /\bambiguous\b/i,
+  /\b(?:no|insufficient) (?:information|context|details)\b/i,
+];
+
+const REFUSAL_MARKERS = [
+  /\bi can't help\b/i,
+  /\bi won't\b/i,
+  /\bagainst (?:my )?(?:guidelines|policy)\b/i,
+];
+
+const CODE_INCOMPLETE_MARKERS = [
+  /\/\/\s*TODO\b/,
+  /\/\*\s*TODO\b/,
+  /#\s*TODO\b/i,
+  /\.\.\.\s*$/m,
+  /\bimplement (?:this|here|me)\b/i,
+  /<replace[_ -]?this>/i,
+  /<your[_ -]?code>/i,
+];
+
+function _extractText(content) {
+  if (!content) return '';
+  if (typeof content === 'string') return content;
+  if (Array.isArray(content)) {
+    return content
+      .filter(b => b?.type === 'text')
+      .map(b => b.text || '')
+      .join(' ');
+  }
+  return '';
+}
+
+function _hasMarkers(text, patterns) {
+  for (const re of patterns) {
+    if (re.test(text)) return true;
+  }
+  return false;
+}
+
+function scoreFactoid(response) {
+  const text = _extractText(response?.content);
+  if (!text) return 0;
+  if (_hasMarkers(text, REFUSAL_MARKERS)) return 0.2;
+  if (_hasMarkers(text, UNCERTAINTY_MARKERS)) return 0.5;
+  // Short answers to factoid questions are usually fine; long hedged answers less so.
+  if (text.length < 200) return 0.9;
+  if (text.length < 500) return 0.85;
+  return 0.8;
+}
+
+function scoreCode(response) {
+  const text = _extractText(response?.content);
+  if (!text) return 0;
+  if (_hasMarkers(text, CODE_INCOMPLETE_MARKERS)) return 0.4;
+  if (_hasMarkers(text, UNCERTAINTY_MARKERS)) return 0.55;
+  // Look for code blocks
+  const fenced = (text.match(/```[\s\S]*?```/g) || []).join('\n');
+  if (!fenced) return 0.6; // Code-gen request without code is suspicious
+  // Very basic balance check
+  const opens = (fenced.match(/[\{\[\(]/g) || []).length;
+  const closes = (fenced.match(/[\}\]\)]/g) || []).length;
+  if (Math.abs(opens - closes) > 2) return 0.5;
+  return 0.9;
+}
+
+async function scoreReasoning(response, opts = {}) {
+  const text = _extractText(response?.content);
+  if (!text) return 0;
+  if (_hasMarkers(text, REFUSAL_MARKERS)) return 0.2;
+  if (_hasMarkers(text, UNCERTAINTY_MARKERS)) return 0.5;
+  // Optional judge LLM via opts.judge({ question, answer }) → [0, 1]
+  if (typeof opts.judge === 'function') {
+    try {
+      const judged = await opts.judge({ question: opts.question, answer: text });
+      if (typeof judged === 'number') return Math.max(0, Math.min(1, judged));
+    } catch (err) {
+      logger.debug({ err: err.message }, '[ConfidenceScorer] Judge LLM failed, using heuristic');
+    }
+  }
+  // Heuristic: well-structured responses (paragraphs + concrete claims) score higher
+  const sentenceCount = (text.match(/[.!?]+\s/g) || []).length;
+  if (sentenceCount < 2) return 0.6;
+  if (sentenceCount > 30) return 0.7; // very long answers are often padding
+  return 0.85;
+}
+
+async function score(response, opts = {}) {
+  const taskType = (opts.taskType || 'reasoning').toLowerCase();
+  if (taskType.includes('code')) return scoreCode(response);
+  if (taskType.includes('factoid') || taskType.includes('qa') || taskType.includes('simple_qa')) {
+    return scoreFactoid(response);
+  }
+  return scoreReasoning(response, opts);
+}
+
+module.exports = { score, scoreFactoid, scoreCode, scoreReasoning };

package/src/routing/context-validator.js

@@ -0,0 +1,71 @@
+/**
+ * Context-window validation for routed models.
+ *
+ * After tier selection, verifies the chosen model can hold the estimated
+ * input tokens (plus response headroom). When it can't, callers should
+ * escalate to a context-capable model.
+ *
+ * Phase 1.3 of the routing overhaul.
+ *
+ * @module routing/context-validator
+ */
+
+const logger = require('../logger');
+const { getModelRegistrySync } = require('./model-registry');
+
+/** Fraction of the context window reserved for the prompt (rest left for response). */
+const HEADROOM_FRACTION = 0.85;
+
+function getContextLimit(model) {
+  if (!model) return null;
+  try {
+    const registry = getModelRegistrySync();
+    const cost = registry.getCost(model);
+    return cost?.context || null;
+  } catch (err) {
+    return null;
+  }
+}
+
+/**
+ * Quick yes/no fit check.
+ * Unknown context windows return true (assume fits — we don't have data to reject).
+ */
+function fits(model, estimatedTokens, fraction = HEADROOM_FRACTION) {
+  const ctx = getContextLimit(model);
+  if (!ctx) return true;
+  return estimatedTokens <= ctx * fraction;
+}
+
+/**
+ * Detailed validation result.
+ * @returns {{ ok: boolean, context: number|null, required: number, limit: number|null, reason?: string }}
+ */
+function validate(model, estimatedTokens) {
+  const ctx = getContextLimit(model);
+  if (!ctx) {
+    return {
+      ok: true,
+      reason: 'unknown_context',
+      context: null,
+      required: estimatedTokens,
+      limit: null,
+    };
+  }
+  const limit = Math.floor(ctx * HEADROOM_FRACTION);
+  if (estimatedTokens <= limit) {
+    return { ok: true, context: ctx, required: estimatedTokens, limit };
+  }
+  logger.debug(
+    { model, context: ctx, required: estimatedTokens, limit },
+    '[ContextValidator] Estimated tokens exceed model context'
+  );
+  return { ok: false, context: ctx, required: estimatedTokens, limit };
+}
+
+module.exports = {
+  validate,
+  fits,
+  getContextLimit,
+  HEADROOM_FRACTION,
+};

package/src/routing/cost-optimizer.js

@@ -8,6 +8,7 @@ const logger = require('../logger');
 const config = require('../config');
 const { getModelRegistry, getModelRegistrySync } = require('./model-registry');
 const { getModelTierSelector, TIER_DEFINITIONS } = require('./model-tiers');
+const { ratioFor } = require('./output-ratios');
 
 // Session cost tracking (in-memory)
 const sessionCosts = new Map(); // sessionId -> { total, requests, byModel, byProvider }
@@ -62,12 +63,14 @@ class CostOptimizer {
    * @param {number} outputTokens - Estimated output tokens (optional)
    * @returns {Object} Cost estimate
    */
-  estimateCost(model, inputTokens, outputTokens = null) {
+  estimateCost(model, inputTokens, outputTokens = null, taskType = null) {
     const registry = this._getRegistry();
     const costs = registry.getCost(model);
 
     const inputCost = (inputTokens / 1_000_000) * costs.input;
-    const estimatedOutputTokens = outputTokens || Math.min(inputTokens * 0.5, 4096);
+    // Phase 2.3: per-task-type output ratio learned from telemetry
+    const ratio = taskType ? ratioFor(taskType) : 0.5;
+    const estimatedOutputTokens = outputTokens || Math.min(inputTokens * ratio, costs.maxOutput || 4096);
     const outputCost = (estimatedOutputTokens / 1_000_000) * costs.output;
 
     return {

package/src/routing/deadline.js

@@ -0,0 +1,52 @@
+/**
+ * Deadline-aware routing (Phase 6.3).
+ *
+ * Reads LYNKR-Deadline-Ms from the request, filters out candidate models
+ * whose P95 latency exceeds the deadline. If the originally-routed model
+ * is too slow, find a faster qualifying alternative.
+ */
+
+const { getLatencyTracker } = require('./latency-tracker');
+
+const SAFETY_FACTOR = 1.2; // leave 20% safety margin against P95 estimates
+
+function getDeadlineMs(req) {
+  if (!req) return null;
+  const h = req.headers || req;
+  const raw = h['lynkr-deadline-ms'] || h['LYNKR-Deadline-Ms'];
+  if (!raw) return null;
+  const num = Number(raw);
+  return Number.isFinite(num) && num > 0 ? num : null;
+}
+
+/**
+ * Check whether a routed model is fast enough for the deadline.
+ */
+function fits(provider, model, deadlineMs) {
+  if (!deadlineMs) return true;
+  const tracker = getLatencyTracker();
+  const p95 = tracker.getModelP95(provider, model);
+  if (p95 === null) return true; // unknown — assume yes
+  return p95 * SAFETY_FACTOR <= deadlineMs;
+}
+
+/**
+ * Pick the fastest model among candidates that meets the deadline.
+ */
+function chooseFastest(candidates, deadlineMs) {
+  if (!Array.isArray(candidates) || candidates.length === 0) return null;
+  const tracker = getLatencyTracker();
+  let bestP95 = Infinity;
+  let best = null;
+  for (const c of candidates) {
+    const p95 = tracker.getModelP95(c.provider, c.model) ?? 5000;
+    const eligible = !deadlineMs || p95 * SAFETY_FACTOR <= deadlineMs;
+    if (eligible && p95 < bestP95) {
+      bestP95 = p95;
+      best = { ...c, p95 };
+    }
+  }
+  return best;
+}
+
+module.exports = { getDeadlineMs, fits, chooseFastest, SAFETY_FACTOR };

package/src/routing/drift-monitor.js

@@ -0,0 +1,113 @@
+/**
+ * Drift monitor (Phase 4.3).
+ *
+ * Tracks two kinds of drift:
+ *   - Input drift: distribution of query embeddings week-over-week via PSI
+ *     (Population Stability Index) over coarse bucket assignments.
+ *   - Output drift: refusal rate, average response length, latency
+ *     distribution per model.
+ *
+ * Computes a PSI per metric; alerts when PSI > 0.2 (warning) or > 0.3
+ * (full retrain recommended). Writes alerts to data/drift-alerts.json.
+ *
+ * Auto-retrain is gated on LYNKR_AUTO_RETRAIN=true and not implemented here —
+ * the consumer (a cron job or the dashboard) decides what to do.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const logger = require('../logger');
+
+const ALERTS_PATH = path.join(__dirname, '../../data/drift-alerts.json');
+const WARN_THRESHOLD = 0.2;
+const RETRAIN_THRESHOLD = 0.3;
+
+function _bucketize(values, bucketCount = 10, min, max) {
+  if (values.length === 0) return new Array(bucketCount).fill(0);
+  const lo = min ?? Math.min(...values);
+  const hi = max ?? Math.max(...values);
+  const range = Math.max(1e-9, hi - lo);
+  const counts = new Array(bucketCount).fill(0);
+  for (const v of values) {
+    const idx = Math.max(0, Math.min(bucketCount - 1, Math.floor(((v - lo) / range) * bucketCount)));
+    counts[idx]++;
+  }
+  return counts;
+}
+
+/**
+ * Population Stability Index between two distributions.
+ * PSI = Σ (p_new - p_old) · ln(p_new / p_old)
+ */
+function psi(oldCounts, newCounts) {
+  const oldTotal = oldCounts.reduce((s, c) => s + c, 0);
+  const newTotal = newCounts.reduce((s, c) => s + c, 0);
+  if (oldTotal === 0 || newTotal === 0) return 0;
+  let sum = 0;
+  for (let i = 0; i < oldCounts.length; i++) {
+    const p = (oldCounts[i] + 0.5) / (oldTotal + oldCounts.length * 0.5);
+    const q = (newCounts[i] + 0.5) / (newTotal + newCounts.length * 0.5);
+    sum += (q - p) * Math.log(q / p);
+  }
+  return sum;
+}
+
+function _writeAlert(alert) {
+  try {
+    fs.mkdirSync(path.dirname(ALERTS_PATH), { recursive: true });
+    let existing = [];
+    if (fs.existsSync(ALERTS_PATH)) {
+      try { existing = JSON.parse(fs.readFileSync(ALERTS_PATH, 'utf8')); } catch {}
+    }
+    const out = Array.isArray(existing) ? existing : [];
+    out.push({ ...alert, timestamp: Date.now() });
+    fs.writeFileSync(ALERTS_PATH, JSON.stringify(out.slice(-200), null, 2));
+  } catch (err) {
+    logger.warn({ err: err.message }, '[DriftMonitor] Alert write failed');
+  }
+}
+
+/**
+ * Detect drift between two value series.
+ * @param {string} metric - name for logging
+ * @param {number[]} oldValues - reference window (e.g. last week)
+ * @param {number[]} newValues - current window (e.g. last 24h)
+ * @returns {{ psi, level, metric }}
+ */
+function detect(metric, oldValues, newValues) {
+  if (oldValues.length < 50 || newValues.length < 20) {
+    return { psi: 0, level: 'insufficient_data', metric };
+  }
+  const combinedMin = Math.min(...oldValues, ...newValues);
+  const combinedMax = Math.max(...oldValues, ...newValues);
+  const oldB = _bucketize(oldValues, 10, combinedMin, combinedMax);
+  const newB = _bucketize(newValues, 10, combinedMin, combinedMax);
+  const p = psi(oldB, newB);
+  let level = 'ok';
+  if (p >= RETRAIN_THRESHOLD) level = 'retrain';
+  else if (p >= WARN_THRESHOLD) level = 'warn';
+
+  if (level !== 'ok') {
+    _writeAlert({ metric, psi: p, level, oldSize: oldValues.length, newSize: newValues.length });
+  }
+  return { psi: p, level, metric };
+}
+
+/**
+ * Detect refusal-rate drift by counting the share of responses containing
+ * refusal markers in two windows.
+ */
+function refusalRateShift(oldResponses, newResponses) {
+  const markers = [/i can't help/i, /i won't/i, /against (?:my )?guidelines/i, /i cannot/i];
+  const _rate = (arr) => arr.filter(t => markers.some(m => m.test(t))).length / Math.max(1, arr.length);
+  return { old: _rate(oldResponses), new: _rate(newResponses) };
+}
+
+module.exports = {
+  psi,
+  detect,
+  refusalRateShift,
+  _bucketize,
+  WARN_THRESHOLD,
+  RETRAIN_THRESHOLD,
+};

package/src/routing/embedding-cache.js

@@ -0,0 +1,77 @@
+/**
+ * In-memory LRU cache for query embeddings.
+ *
+ * Used by Phase 3.1 (kNN router) and Phase 4.3 (drift detector) to avoid
+ * repeated embedding calls for queries we've already seen.
+ */
+
+const crypto = require('crypto');
+const logger = require('../logger');
+
+const DEFAULT_MAX = 5000;
+
+class EmbeddingCache {
+  constructor(maxSize = DEFAULT_MAX) {
+    this.maxSize = maxSize;
+    this.cache = new Map(); // hash -> { embedding, lastAccess }
+    this.hits = 0;
+    this.misses = 0;
+  }
+
+  _hash(text) {
+    return crypto.createHash('sha1').update(text).digest('hex');
+  }
+
+  get(text) {
+    if (!text || typeof text !== 'string') return null;
+    const key = this._hash(text);
+    const entry = this.cache.get(key);
+    if (!entry) {
+      this.misses++;
+      return null;
+    }
+    // LRU touch
+    this.cache.delete(key);
+    entry.lastAccess = Date.now();
+    this.cache.set(key, entry);
+    this.hits++;
+    return entry.embedding;
+  }
+
+  set(text, embedding) {
+    if (!text || !embedding) return;
+    const key = this._hash(text);
+    if (this.cache.has(key)) this.cache.delete(key);
+    this.cache.set(key, { embedding, lastAccess: Date.now() });
+    if (this.cache.size > this.maxSize) {
+      // Evict least-recently-used (Map keeps insertion/access order)
+      const oldest = this.cache.keys().next().value;
+      this.cache.delete(oldest);
+    }
+  }
+
+  getStats() {
+    const total = this.hits + this.misses;
+    return {
+      size: this.cache.size,
+      maxSize: this.maxSize,
+      hits: this.hits,
+      misses: this.misses,
+      hitRate: total > 0 ? (this.hits / total).toFixed(3) : '0',
+    };
+  }
+
+  clear() {
+    this.cache.clear();
+    this.hits = 0;
+    this.misses = 0;
+  }
+}
+
+let _instance = null;
+function getEmbeddingCache() {
+  if (!_instance) _instance = new EmbeddingCache();
+  return _instance;
+}
+
+module.exports = { EmbeddingCache, getEmbeddingCache };