lynkr 9.0.1 → 9.1.2

package/src/dashboard/api.js

@@ -0,0 +1,170 @@
+const config = require('../config');
+const telemetry = require('../routing/telemetry');
+const { getUsage } = require('../usage/aggregator');
+const metrics = require('../metrics');
+const { getMetricsCollector } = require('../observability/metrics');
+const { TIER_DEFINITIONS } = require('../routing/model-tiers');
+
+function getConfiguredProviders() {
+  const c = config;
+  const providers = [];
+  const add = (name, type, ok) => ok && providers.push({ name, type });
+
+  add('databricks',     'cloud', c.databricks?.url && c.databricks?.apiKey);
+  add('azure-anthropic','cloud', c.azureAnthropic?.endpoint && c.azureAnthropic?.apiKey);
+  add('bedrock',        'cloud', c.bedrock?.apiKey);
+  add('openrouter',     'cloud', c.openrouter?.apiKey);
+  add('openai',         'cloud', c.openai?.apiKey);
+  add('azure-openai',   'cloud', c.azureOpenAI?.endpoint && c.azureOpenAI?.apiKey);
+  add('vertex',         'cloud', c.vertex?.projectId);
+  add('moonshot',       'cloud', c.moonshot?.apiKey);
+  add('ollama',         'local', c.ollama?.endpoint);
+  add('llamacpp',       'local', c.llamacpp?.endpoint);
+  add('lmstudio',       'local', c.lmstudio?.endpoint);
+
+  return providers;
+}
+
+// Noise provider names injected by unit tests — filter them out of UI
+const TEST_PROVIDER_RE = /^(accuracy-|stats-|provider-stats-|roundtrip-|latency-)/;
+
+// Find the widest window that has at least one row, so the UI never shows
+// empty panels just because there were no requests in the last 24 hours.
+function findActiveWindow() {
+  const newest = telemetry.query({ limit: 1 });
+  if (!newest.length) return { since: Date.now() - 86400000, label: '24h' };
+
+  const ageMs = Date.now() - newest[0].timestamp;
+  if (ageMs <= 86400000)    return { since: Date.now() - 86400000,        label: '24h'      };
+  if (ageMs <= 7*86400000)  return { since: Date.now() - 7*86400000,      label: '7d'       };
+  if (ageMs <= 30*86400000) return { since: Date.now() - 30*86400000,     label: '30d'      };
+  return                           { since: 0,                             label: 'all time' };
+}
+
+function getCircuitBreakerStates() {
+  try {
+    const { getCircuitBreakerRegistry } = require('../clients/circuit-breaker');
+    const reg = getCircuitBreakerRegistry();
+    return reg.getAll();
+  } catch {
+    return {};
+  }
+}
+
+// Group telemetry rows by calendar day (UTC), returning last `days` buckets
+function dailyBreakdown(rows, days = 7) {
+  const now = Date.now();
+  const DAY = 86400000;
+  const result = [];
+
+  for (let i = days - 1; i >= 0; i--) {
+    const start = now - (i + 1) * DAY;
+    const end   = now - i * DAY;
+    const bucket = rows.filter(r => r.timestamp >= start && r.timestamp < end);
+
+    const byTier = {};
+    let cost = 0;
+    for (const r of bucket) {
+      const t = r.tier || 'UNKNOWN';
+      byTier[t] = (byTier[t] || 0) + 1;
+      cost += Number(r.cost_usd) || 0;
+    }
+
+    result.push({
+      label: new Date(start).toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
+      total: bucket.length,
+      byTier,
+      cost: Math.round(cost * 10000) / 10000,
+    });
+  }
+  return result;
+}
+
+function overview(req, res) {
+  const win         = findActiveWindow();
+  const todayUsage  = getUsage({ window: win.label === '24h' ? '1d' : win.label === 'all time' ? 'all' : win.label });
+  const recentRows  = telemetry.query({ limit: 10 });
+  const todayStats  = telemetry.getStats({ since: win.since });
+  const snap        = metrics.snapshot();
+
+  res.json({
+    uptime:        Math.floor(process.uptime()),
+    port:          config.port,
+    version:       process.env.npm_package_version || '9.0.2',
+    modelProvider: config.modelProvider?.type || 'unknown',
+    providers:     getConfiguredProviders(),
+    statsWindow:   win.label,
+    metrics: {
+      requestsTotal:    snap.requestsTotal,
+      responsesSuccess: snap.responses?.success || 0,
+      responsesError:   snap.responses?.error   || 0,
+    },
+    today: {
+      requests:      todayUsage.totals?.requests      || 0,
+      totalTokens:   todayUsage.totals?.totalTokens   || 0,
+      cost:          todayUsage.totals?.actualCost     || 0,
+      saved:         todayUsage.totals?.saved          || 0,
+      savedPercent:  todayUsage.totals?.savedPercent   || 0,
+    },
+    stats:          todayStats,
+    recentRequests: recentRows,
+  });
+}
+
+function usage(req, res) {
+  const window   = req.query.window   || '7d';
+  const provider = req.query.provider || undefined;
+  const model    = req.query.model    || undefined;
+
+  const data = getUsage({ window, provider, model });
+
+  // Add daily breakdown for chart (last 7 or 30 days depending on window)
+  const days = window === '1d' ? 1 : window === '30d' ? 30 : 7;
+  const since = window === 'all' ? 0 : Date.now() - days * 86400000;
+  const rawRows = since > 0
+    ? telemetry.query({ since, limit: 50000 })
+    : telemetry.query({ limit: 50000 });
+
+  data.daily = dailyBreakdown(rawRows, Math.min(days, 30));
+
+  res.json(data);
+}
+
+function routing(req, res) {
+  const win      = findActiveWindow();
+  const { since } = win;
+
+  const accuracy  = telemetry.getRoutingAccuracy({ since });
+  const stats     = telemetry.getStats({ since });
+  const cbStates  = getCircuitBreakerStates();
+
+  // Derive providers from actual DB rows — never miss a provider not in config
+  const dbRows = telemetry.query({ limit: 100000, since });
+  const dbProviders = [...new Set(
+    dbRows.map(r => r.provider).filter(p => p && !TEST_PROVIDER_RE.test(p))
+  )];
+
+  const providerStats = {};
+  for (const p of dbProviders) {
+    const s = telemetry.getProviderStats(p, { since });
+    if (s) providerStats[p] = s;
+  }
+
+  res.json({ tierDefinitions: TIER_DEFINITIONS, accuracy, stats, providerStats, circuitBreakers: cbStates, window: win.label });
+}
+
+function logs(req, res) {
+  const limit   = Math.min(parseInt(req.query.limit || '100', 10), 500);
+  const filters = { limit };
+
+  if (req.query.provider) filters.provider = req.query.provider;
+  if (req.query.tier)     filters.tier     = req.query.tier;
+  if (req.query.since)    filters.since    = parseInt(req.query.since, 10);
+
+  let rows = telemetry.query(filters);
+  if (req.query.error === 'true') rows = rows.filter(r => r.error_type);
+
+  res.json(rows);
+}
+
+module.exports = { overview, usage, routing, logs };

package/src/dashboard/router.js

@@ -0,0 +1,13 @@
+const express = require('express');
+const path    = require('path');
+const api     = require('./api');
+
+const router = express.Router();
+
+router.get('/',            (_req, res) => res.sendFile(path.join(__dirname, '../../public/dashboard.html')));
+router.get('/api/overview', api.overview);
+router.get('/api/usage',    api.usage);
+router.get('/api/routing',  api.routing);
+router.get('/api/logs',     api.logs);
+
+module.exports = router;

package/src/headroom/client.js

@@ -58,6 +58,7 @@ async function checkHealth() {
       return {
         available: data.headroom_loaded === true,
         status: data.status,
+        version: data.headroom_version,
         ccrEnabled: data.ccr_enabled,
         llmlinguaEnabled: data.llmlingua_enabled,
         entriesCached: data.entries_cached,
@@ -154,8 +155,10 @@ async function compressMessages(messages, tools = [], options = {}) {
           tokensBefore: result.stats?.tokens_before,
           tokensAfter: result.stats?.tokens_after,
           savingsPercent: result.stats?.savings_percent,
+          compressionRatio: result.stats?.compression_ratio,
           latencyMs: result.stats?.latency_ms,
           transforms: result.stats?.transforms_applied,
+          headroomVersion: result.stats?.headroom_version,
         },
         "Headroom compression applied"
       );
@@ -245,112 +248,6 @@ async function ccrRetrieve(hash, query = null, maxResults = 20) {
   }
 }
 
-/**
- * Track compression for proactive CCR expansion
- */
-async function ccrTrack(hashKey, turnNumber, toolName, sample) {
-  const headroomConfig = getConfig();
-
-  if (!isEnabled()) {
-    return { tracked: false };
-  }
-
-  try {
-    const params = new URLSearchParams({
-      hash_key: hashKey,
-      turn_number: String(turnNumber),
-      tool_name: toolName,
-      sample: sample.substring(0, 500),
-    });
-
-    const response = await fetch(`${headroomConfig.endpoint}/ccr/track?${params}`, {
-      method: "POST",
-      signal: AbortSignal.timeout(2000),
-    });
-
-    if (response.ok) {
-      return await response.json();
-    }
-    return { tracked: false };
-  } catch (err) {
-    logger.debug({ error: err.message }, "CCR tracking failed");
-    return { tracked: false };
-  }
-}
-
-/**
- * Analyze query for proactive CCR expansion
- */
-async function ccrAnalyze(query, turnNumber) {
-  const headroomConfig = getConfig();
-
-  if (!isEnabled()) {
-    return { expansions: [] };
-  }
-
-  try {
-    const response = await fetch(`${headroomConfig.endpoint}/ccr/analyze`, {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ query, turn_number: turnNumber }),
-      signal: AbortSignal.timeout(2000),
-    });
-
-    if (response.ok) {
-      return await response.json();
-    }
-    return { expansions: [] };
-  } catch (err) {
-    logger.debug({ error: err.message }, "CCR analysis failed");
-    return { expansions: [] };
-  }
-}
-
-/**
- * Compress text using LLMLingua-2 ML compression
- * (Optional - requires LLMLingua enabled in sidecar)
- */
-async function llmlinguaCompress(text, targetRatio = 0.5, forceTokens = null) {
-  const headroomConfig = getConfig();
-
-  if (!isEnabled()) {
-    return { success: false, error: "Headroom disabled" };
-  }
-
-  try {
-    const params = new URLSearchParams({
-      text,
-      target_ratio: String(targetRatio),
-    });
-
-    if (forceTokens && Array.isArray(forceTokens)) {
-      params.append("force_tokens", JSON.stringify(forceTokens));
-    }
-
-    const response = await fetch(`${headroomConfig.endpoint}/compress/llmlingua?${params}`, {
-      method: "POST",
-      signal: AbortSignal.timeout(30000), // LLMLingua can be slow
-    });
-
-    if (!response.ok) {
-      const error = await response.text();
-      return { success: false, error };
-    }
-
-    const result = await response.json();
-    return {
-      success: true,
-      compressed: result.compressed,
-      originalTokens: result.original_tokens,
-      compressedTokens: result.compressed_tokens,
-      ratio: result.ratio,
-    };
-  } catch (err) {
-    logger.error({ error: err.message }, "LLMLingua compression failed");
-    return { success: false, error: err.message };
-  }
-}
-
 /**
  * Get client-side metrics
  */
@@ -424,9 +321,6 @@ module.exports = {
   checkHealth,
   compressMessages,
   ccrRetrieve,
-  ccrTrack,
-  ccrAnalyze,
-  llmlinguaCompress,
   getMetrics,
   getServerMetrics,
   getCombinedMetrics,

package/src/headroom/index.js

@@ -125,20 +125,6 @@ class HeadroomManager {
     return client.ccrRetrieve(hash, query, maxResults);
   }
 
-  /**
-   * Track compression for proactive expansion
-   */
-  async ccrTrack(hashKey, turnNumber, toolName, sample) {
-    return client.ccrTrack(hashKey, turnNumber, toolName, sample);
-  }
-
-  /**
-   * Analyze query for proactive CCR expansion
-   */
-  async ccrAnalyze(query, turnNumber) {
-    return client.ccrAnalyze(query, turnNumber);
-  }
-
   /**
    * Check if Headroom is enabled
    */

package/src/memory/extractor.js

@@ -312,6 +312,28 @@ async function createMemoryWithSurprise(options) {
 
     return memory;
   } catch (err) {
+    // FK constraint fails when session is ephemeral (passthrough mode) —
+    // retry without session link so the memory still gets saved
+    if (err.code === 'SQLITE_CONSTRAINT_FOREIGNKEY' && sessionId) {
+      try {
+        return store.createMemory({
+          sessionId: null,
+          content,
+          type,
+          category,
+          importance,
+          surpriseScore,
+          metadata: {
+            ...metadata,
+            extractedAt: Date.now(),
+            originalSessionId: sessionId,
+          },
+        });
+      } catch (retryErr) {
+        logger.warn({ err: retryErr, content }, 'Failed to store memory (retry without session)');
+        return null;
+      }
+    }
     logger.warn({ err, content }, 'Failed to store memory');
     return null;
   }

package/src/memory/search.js

@@ -258,58 +258,8 @@ function searchMemories(options) {
   }
 }
 
-/**
-
- * Search with keyword expansion (UPDATED - now uses sanitized keywords)
-=======
- * Prepare FTS5 query - handle special characters and phrases
- */
-function prepareFTS5Query(query) {
-  // FTS5 special characters: " * ( ) < > - : AND OR NOT
-  // Strategy: Strip XML/HTML tags, then sanitize remaining text
-  let cleaned = query.trim();
-
-  // Step 1: Remove XML/HTML tags (common in error messages)
-  // Matches: <tag>, </tag>, <tag attr="value">
-  cleaned = cleaned.replace(/<[^>]+>/g, ' ');
-
-  // Step 2: Remove excess whitespace from tag removal
-  cleaned = cleaned.replace(/\s+/g, ' ').trim();
-
-  if (!cleaned) {
-    // Query was all tags, return safe fallback
-    return '"empty query"';
-  }
-
-  // Step 3: Check if query contains FTS5 operators (AND, OR, NOT)
-  const hasFTS5Operators = /\b(AND|OR|NOT)\b/i.test(cleaned);
-
-  // Step 4: ENHANCED - Remove ALL special characters that could break FTS5
-  // Keep only: letters, numbers, spaces
-  // Remove: * ( ) < > - : [ ] | , + = ? ! ; / \ @ # $ % ^ & { }
-  cleaned = cleaned.replace(/[*()<>\-:\[\]|,+=?!;\/\\@#$%^&{}]/g, ' ');
-  cleaned = cleaned.replace(/\s+/g, ' ').trim();
-
-  // Step 5: Escape double quotes (FTS5 uses "" for literal quote)
-  cleaned = cleaned.replace(/"/g, '""');
-
-  // Step 6: Additional safety - remove any remaining non-alphanumeric except spaces
-  cleaned = cleaned.replace(/[^\w\s""]/g, ' ');
-  cleaned = cleaned.replace(/\s+/g, ' ').trim();
-
-  // Step 7: Wrap in quotes for phrase search (safest approach)
-  if (!hasFTS5Operators) {
-    // Treat as literal phrase search
-    cleaned = `"${cleaned}"`;
-  }
-
-  // If query has FTS5 operators, let FTS5 parse them (advanced users)
-  return cleaned;
-}
-
 /**
  * Search with keyword expansion (extract key terms)
-
  */
 function searchWithExpansion(options) {
   const { query, limit = 10 } = options;