lynkr 9.0.2 → 9.1.2

package/src/routing/telemetry.js

@@ -105,6 +105,9 @@ function init() {
 
       CREATE INDEX IF NOT EXISTS idx_telemetry_timestamp
         ON routing_telemetry(timestamp);
+
+      CREATE INDEX IF NOT EXISTS idx_telemetry_session_id
+        ON routing_telemetry(session_id, timestamp);
     `);
 
     logger.info({ dbPath }, "Routing telemetry database initialised");
@@ -233,6 +236,10 @@ function query(filters = {}) {
     clauses.push("timestamp >= @since");
     params.since = filters.since;
   }
+  if (filters.session_id) {
+    clauses.push("session_id = @session_id");
+    params.session_id = filters.session_id;
+  }
 
   const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
   const limit = filters.limit ?? 100;

package/src/server.js

@@ -147,6 +147,9 @@ function createApp() {
 
   app.use(router);
 
+  // Dashboard UI
+  app.use('/dashboard', require('./dashboard/router'));
+
   // Files API
   const filesRouter = require("./api/files-router");
   app.use("/v1", filesRouter);

package/src/stores/file-store.js

@@ -1,9 +1,11 @@
 const fs = require("fs");
+const fsp = require("fs").promises;
 const path = require("path");
 const crypto = require("crypto");
 const logger = require("../logger");
 
 const STORAGE_DIR = path.resolve(process.env.FILES_STORAGE_PATH || "./data/files");
+const METADATA_FILE = path.join(STORAGE_DIR, "_metadata.json");
 const MAX_FILES = parseInt(process.env.FILES_MAX_COUNT || "1000", 10);
 
 const metadata = new Map();
@@ -14,15 +16,46 @@ function ensureStorageDir() {
   }
 }
 
-function storeFile(buffer, { filename, purpose, mimeType }) {
+function persistMetadata() {
+  try {
+    const entries = Array.from(metadata.values());
+    fs.writeFileSync(METADATA_FILE, JSON.stringify(entries), "utf8");
+  } catch (err) {
+    logger.warn({ err: err.message }, "Failed to persist file metadata");
+  }
+}
+
+function loadMetadata() {
+  ensureStorageDir();
+  try {
+    if (!fs.existsSync(METADATA_FILE)) return;
+    const entries = JSON.parse(fs.readFileSync(METADATA_FILE, "utf8"));
+    for (const entry of entries) {
+      // Only restore entries whose backing file still exists on disk
+      if (fs.existsSync(entry.storage_path)) {
+        metadata.set(entry.id, entry);
+      } else {
+        logger.debug({ fileId: entry.id }, "Dropping orphaned metadata entry (file missing)");
+      }
+    }
+    logger.info({ count: metadata.size }, "File metadata restored from disk");
+  } catch (err) {
+    logger.warn({ err: err.message }, "Could not load file metadata; starting fresh");
+  }
+}
+
+// Restore metadata at module load so restarts don't orphan files
+loadMetadata();
+
+async function storeFile(buffer, { filename, purpose, mimeType }) {
   ensureStorageDir();
   if (metadata.size >= MAX_FILES) {
     const oldest = metadata.keys().next().value;
-    deleteFile(oldest);
+    await deleteFile(oldest);
   }
   const id = `file-${crypto.randomUUID()}`;
   const storagePath = path.join(STORAGE_DIR, id);
-  fs.writeFileSync(storagePath, buffer);
+  await fsp.writeFile(storagePath, buffer);
   const entry = {
     id,
     object: "file",
@@ -34,6 +67,7 @@ function storeFile(buffer, { filename, purpose, mimeType }) {
     storage_path: storagePath,
   };
   metadata.set(id, entry);
+  persistMetadata();
   logger.info({ fileId: id, bytes: buffer.length, filename }, "File stored");
   return entry;
 }
@@ -42,21 +76,22 @@ function getFile(id) {
   return metadata.get(id) || null;
 }
 
-function getFileContent(id) {
+async function getFileContent(id) {
   const entry = metadata.get(id);
   if (!entry) return null;
   try {
-    return fs.readFileSync(entry.storage_path);
+    return await fsp.readFile(entry.storage_path);
   } catch {
     return null;
   }
 }
 
-function deleteFile(id) {
+async function deleteFile(id) {
   const entry = metadata.get(id);
   if (!entry) return false;
-  try { fs.unlinkSync(entry.storage_path); } catch {}
+  try { await fsp.unlink(entry.storage_path); } catch {}
   metadata.delete(id);
+  persistMetadata();
   return true;
 }
 

package/src/tools/smart-selection.js

@@ -280,9 +280,18 @@ function selectToolsSmartly(tools, classification, options = {}) {
 
   // Get relevant tool names for this request type
   const relevantToolNames = TOOL_SELECTION_MAP[requestType] || TOOL_SELECTION_MAP.coding;
+  const relevantLower = new Set(relevantToolNames.map(n => n.toLowerCase()));
 
-  // Filter to relevant tools only
-  let selectedTools = tools.filter(tool => relevantToolNames.includes(tool.name));
+  // Filter to relevant tools only (case-insensitive match so external clients
+  // using lowercase names like Pi's `bash`/`read` aren't filtered out)
+  let selectedTools = tools.filter(tool => relevantLower.has(String(tool.name || '').toLowerCase()));
+
+  // If nothing matched, the caller is using a tool ecosystem we don't recognize
+  // (e.g. Pi's read/write/edit/bash). Pass tools through untouched rather than
+  // deleting them — otherwise the LLM gets no schema and hallucinates defaults.
+  if (selectedTools.length === 0) {
+    return tools;
+  }
 
   // Mode-specific adjustments
   if (config.mode === 'aggressive') {

package/src/training/trajectory-compressor.js

@@ -0,0 +1,266 @@
+/**
+ * Trajectory Compressor
+ *
+ * Reads completed agent sessions out of Lynkr's session DB,
+ * joins with routing telemetry to pick up tier / score / outcome
+ * metadata, and emits JSONL training samples for fine-tuning small
+ * models on tool-call generation and tier-routing decisions.
+ *
+ * Each line of the output JSONL is a self-contained sample:
+ *
+ *   {
+ *     "session_id":     "...",
+ *     "messages":       [{"role": "...", "content": ...}, ...],
+ *     "tool_calls":     [...],
+ *     "outcome":        "success" | "error",
+ *     "tier":           "SIMPLE" | "MEDIUM" | "COMPLEX" | "REASONING",
+ *     "complexity_score": 38,
+ *     "model_used":     "gpt-4o",
+ *     "provider_used":  "azure-openai",
+ *     "tokens_in":      1234,
+ *     "tokens_out":     456,
+ *     "latency_ms":     2400,
+ *     "started_at":     "2026-05-03T10:11:12Z",
+ *     "ended_at":       "2026-05-03T10:11:14Z"
+ *   }
+ *
+ * The compressor is read-only — it never modifies the source DBs.
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const db = require("../db");
+const telemetry = require("../routing/telemetry");
+
+// Patterns for the optional --anonymize pass. Order matters: more
+// specific patterns first so they don't get clobbered by generic ones.
+const ANONYMIZE_PATTERNS = [
+  // API keys and bearer tokens
+  [/sk-[A-Za-z0-9_-]{20,}/g, "<API_KEY>"],
+  [/Bearer\s+[A-Za-z0-9._\-]+/gi, "Bearer <REDACTED>"],
+  [/dapi_[A-Za-z0-9_-]+/g, "<DATABRICKS_KEY>"],
+  [/eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, "<JWT>"],
+  // AWS keys
+  [/AKIA[0-9A-Z]{16}/g, "<AWS_ACCESS_KEY>"],
+  // Email addresses
+  [/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, "<EMAIL>"],
+  // Absolute filesystem paths under /Users/<name>/ or /home/<name>/
+  [/\/Users\/[^/\s]+/g, "/Users/<USER>"],
+  [/\/home\/[^/\s]+/g, "/home/<USER>"],
+  // IPs
+  [/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, "<IP>"],
+  // Hostnames containing service-now / corporate domains (configurable)
+  [/[A-Za-z0-9-]+\.service-now\.com/gi, "<SERVICENOW_HOST>"],
+];
+
+function anonymize(value) {
+  if (typeof value === "string") {
+    let out = value;
+    for (const [re, replacement] of ANONYMIZE_PATTERNS) {
+      out = out.replace(re, replacement);
+    }
+    return out;
+  }
+  if (Array.isArray(value)) return value.map(anonymize);
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) out[k] = anonymize(v);
+    return out;
+  }
+  return value;
+}
+
+function parseJsonSafe(text, fallback = null) {
+  if (text == null) return fallback;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return fallback;
+  }
+}
+
+/**
+ * List session ids in a window, optionally filtered by tier.
+ */
+function listSessions({ since = null, tier = null } = {}) {
+  if (!db) return [];
+
+  const rows = since
+    ? db
+        .prepare(
+          "SELECT id, created_at, updated_at, metadata FROM sessions WHERE updated_at >= ? ORDER BY updated_at DESC"
+        )
+        .all(since)
+    : db.prepare("SELECT id, created_at, updated_at, metadata FROM sessions ORDER BY updated_at DESC").all();
+
+  if (!tier) return rows;
+
+  // Tier filter requires joining against routing telemetry — we do that
+  // per-session lazily so we don't pre-load the whole telemetry table.
+  return rows.filter((s) => sessionTier(s.id) === tier);
+}
+
+/**
+ * Find the dominant tier picked across a session's telemetry rows.
+ */
+function sessionTier(sessionId) {
+  try {
+    const rows = telemetry.query({ session_id: sessionId, limit: 1000 });
+    if (rows.length === 0) return null;
+    const counts = {};
+    for (const r of rows) counts[r.tier || "UNKNOWN"] = (counts[r.tier || "UNKNOWN"] || 0) + 1;
+    return Object.entries(counts).sort((a, b) => b[1] - a[1])[0][0];
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Build one trajectory record for a single session.
+ */
+function buildTrajectory(session, options = {}) {
+  if (!db) return null;
+
+  const historyStmt = db.prepare(
+    "SELECT role, type, status, content, metadata, timestamp FROM session_history WHERE session_id = ? ORDER BY timestamp ASC"
+  );
+  const history = historyStmt.all(session.id);
+
+  // Convert each session_history row into a chat message, preserving
+  // tool-call structure when present in metadata.
+  const messages = [];
+  const toolCalls = [];
+
+  for (const row of history) {
+    const meta = parseJsonSafe(row.metadata) || {};
+    const content = parseJsonSafe(row.content, row.content);
+
+    if (row.role === "tool" || row.type === "tool_use" || row.type === "tool_result") {
+      // Capture tool calls as a separate stream alongside the chat
+      toolCalls.push({
+        type: row.type,
+        timestamp: row.timestamp,
+        content,
+        metadata: meta,
+      });
+    }
+
+    if (row.role === "user" || row.role === "assistant" || row.role === "system") {
+      messages.push({
+        role: row.role,
+        content,
+      });
+    }
+  }
+
+  // Pull telemetry records associated with this session to enrich.
+  const teleRows = telemetry.query({ session_id: session.id, limit: 1000 });
+
+  const totals = teleRows.reduce(
+    (acc, r) => {
+      acc.tokens_in += r.input_tokens || 0;
+      acc.tokens_out += r.output_tokens || 0;
+      acc.latency_ms += r.latency_ms || 0;
+      return acc;
+    },
+    { tokens_in: 0, tokens_out: 0, latency_ms: 0 }
+  );
+
+  // Pick the modal tier (most-used) and the most-recent model/provider.
+  const tier = sessionTier(session.id);
+  const last = teleRows[0]; // telemetry.query orders DESC
+  const errorRow = teleRows.find((r) => r.error_type);
+  const outcome = errorRow ? "error" : "success";
+  const complexityAvg =
+    teleRows.length > 0
+      ? Math.round(
+          teleRows.reduce((sum, r) => sum + (r.complexity_score || 0), 0) /
+            teleRows.length
+        )
+      : null;
+
+  let trajectory = {
+    session_id: session.id,
+    messages,
+    tool_calls: toolCalls,
+    outcome,
+    tier,
+    complexity_score: complexityAvg,
+    model_used: last?.model || null,
+    provider_used: last?.provider || null,
+    tokens_in: totals.tokens_in,
+    tokens_out: totals.tokens_out,
+    latency_ms: totals.latency_ms,
+    started_at: new Date(session.created_at).toISOString(),
+    ended_at: new Date(session.updated_at).toISOString(),
+  };
+
+  if (options.anonymize) {
+    trajectory = anonymize(trajectory);
+  }
+  return trajectory;
+}
+
+/**
+ * Stream trajectories to a writable target (a path or a stream).
+ *
+ * @param {Object} options
+ * @param {string|number|Date} [options.since]   Window start (ms / Date / "Nd")
+ * @param {string} [options.tier]                Filter to one tier
+ * @param {boolean} [options.anonymize=false]    Strip PII / paths / secrets
+ * @param {string|stream.Writable} [options.output="-"]   File path, "-" for stdout, or a stream
+ * @param {function} [options.onProgress]        Optional progress callback (count) => void
+ * @returns {{ count: number, output: string }}
+ */
+function exportJsonl(options = {}) {
+  const since = resolveSince(options.since);
+  const sessions = listSessions({ since, tier: options.tier });
+
+  let stream;
+  let outputPath = "-";
+  let closeStream = false;
+
+  if (!options.output || options.output === "-") {
+    stream = process.stdout;
+  } else if (typeof options.output === "string") {
+    outputPath = options.output;
+    fs.mkdirSync(path.dirname(path.resolve(outputPath)), { recursive: true });
+    stream = fs.createWriteStream(outputPath);
+    closeStream = true;
+  } else {
+    stream = options.output;
+  }
+
+  let count = 0;
+  for (const session of sessions) {
+    const trajectory = buildTrajectory(session, options);
+    if (!trajectory || trajectory.messages.length === 0) continue;
+    stream.write(JSON.stringify(trajectory) + "\n");
+    count++;
+    if (options.onProgress) options.onProgress(count);
+  }
+
+  if (closeStream) stream.end();
+  return { count, output: outputPath };
+}
+
+function resolveSince(value) {
+  if (value == null) return null;
+  if (value instanceof Date) return value.getTime();
+  if (typeof value === "number") return value;
+  if (typeof value === "string") {
+    const m = value.match(/^(\d+)d$/);
+    if (m) return Date.now() - parseInt(m[1], 10) * 24 * 60 * 60 * 1000;
+    const parsed = Date.parse(value);
+    if (!Number.isNaN(parsed)) return parsed;
+  }
+  return null;
+}
+
+module.exports = {
+  exportJsonl,
+  buildTrajectory,
+  listSessions,
+  anonymize,
+};

package/src/usage/aggregator.js

@@ -0,0 +1,206 @@
+/**
+ * Usage Aggregator
+ *
+ * Reads routing telemetry from .lynkr/telemetry.db and produces
+ * actionable spend / savings reports.
+ *
+ * The "savings" calculation answers the question:
+ *   "How much would this same workload have cost if every request
+ *    had hit the most expensive flagship model?"
+ *
+ * That's the number Lynkr's tier router exists to make small.
+ */
+
+const telemetry = require("../routing/telemetry");
+const { getCostOptimizer } = require("../routing/cost-optimizer");
+
+// What we treat as the "flagship comparison" — the model a developer
+// would otherwise run every request against if they didn't have Lynkr.
+// Picked to match Claude Code / Cursor defaults.
+const DEFAULT_FLAGSHIP_MODEL = "claude-sonnet-4-5-20250929";
+
+const WINDOW_PRESETS = {
+  "1d": 1 * 24 * 60 * 60 * 1000,
+  "7d": 7 * 24 * 60 * 60 * 1000,
+  "30d": 30 * 24 * 60 * 60 * 1000,
+  all: null,
+};
+
+/**
+ * Resolve a window string ("7d", "30d", "all") or a Date / ISO string
+ * into a `since` timestamp in ms. Returns null for "all".
+ */
+function resolveSince(window) {
+  if (!window || window === "all") return null;
+  if (window instanceof Date) return window.getTime();
+  if (typeof window === "string") {
+    if (WINDOW_PRESETS[window] !== undefined) {
+      return WINDOW_PRESETS[window] === null ? null : Date.now() - WINDOW_PRESETS[window];
+    }
+    if (/^\d+d$/.test(window)) {
+      const days = parseInt(window, 10);
+      return Date.now() - days * 24 * 60 * 60 * 1000;
+    }
+    const parsed = Date.parse(window);
+    if (!Number.isNaN(parsed)) return parsed;
+  }
+  if (typeof window === "number") return window;
+  return null;
+}
+
+/**
+ * Compute usage stats for a time window.
+ *
+ * @param {Object} options
+ * @param {string|Date|number} [options.window="30d"]   "1d", "7d", "30d", "all", ISO string, or epoch ms
+ * @param {string}             [options.flagship]      Model id used for the "what if I'd run flagship-only" comparison
+ * @param {string}             [options.model]         Filter to a single model
+ * @param {string}             [options.provider]      Filter to a single provider
+ * @returns {Object} Aggregated usage report
+ */
+function getUsage(options = {}) {
+  const window = options.window || "30d";
+  const since = resolveSince(window);
+  const flagship = options.flagship || DEFAULT_FLAGSHIP_MODEL;
+
+  const filters = { limit: 100000 };
+  if (since !== null) filters.since = since;
+  if (options.provider) filters.provider = options.provider;
+
+  let rows;
+  try {
+    rows = telemetry.query(filters);
+  } catch (err) {
+    return {
+      window,
+      since: since ? new Date(since).toISOString() : null,
+      flagship,
+      totals: emptyTotals(),
+      byTier: {},
+      byProvider: {},
+      byModel: {},
+      error: err.message,
+    };
+  }
+
+  // Optional model filter (telemetry.query doesn't support it natively)
+  if (options.model) {
+    rows = rows.filter((r) => r.model === options.model);
+  }
+
+  const optimizer = (() => {
+    try {
+      return getCostOptimizer();
+    } catch {
+      return null;
+    }
+  })();
+
+  const totals = emptyTotals();
+  const byTier = {};
+  const byProvider = {};
+  const byModel = {};
+
+  for (const row of rows) {
+    const inputTokens = row.input_tokens || 0;
+    const outputTokens = row.output_tokens || 0;
+    const totalTokens = inputTokens + outputTokens;
+    const actualCost = Number(row.cost_usd) || 0;
+
+    // Hypothetical cost if this same request had hit the flagship model.
+    let flagshipCost = 0;
+    if (optimizer && totalTokens > 0) {
+      try {
+        const est = optimizer.estimateCost(flagship, inputTokens, outputTokens);
+        flagshipCost = (est.inputCost || 0) + (est.outputCost || 0);
+      } catch {
+        flagshipCost = 0;
+      }
+    }
+    const saved = Math.max(0, flagshipCost - actualCost);
+
+    totals.requests += 1;
+    totals.inputTokens += inputTokens;
+    totals.outputTokens += outputTokens;
+    totals.totalTokens += totalTokens;
+    totals.actualCost += actualCost;
+    totals.flagshipCost += flagshipCost;
+    totals.saved += saved;
+    if (row.was_fallback) totals.fallbacks += 1;
+    if (row.error_type) totals.errors += 1;
+
+    bumpBucket(byTier, row.tier || "UNKNOWN", inputTokens, outputTokens, actualCost, flagshipCost);
+    bumpBucket(byProvider, row.provider || "unknown", inputTokens, outputTokens, actualCost, flagshipCost);
+    bumpBucket(byModel, row.model || "unknown", inputTokens, outputTokens, actualCost, flagshipCost);
+  }
+
+  return {
+    window,
+    since: since ? new Date(since).toISOString() : null,
+    flagship,
+    totals,
+    byTier,
+    byProvider,
+    byModel,
+  };
+}
+
+function emptyTotals() {
+  return {
+    requests: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    totalTokens: 0,
+    actualCost: 0,
+    flagshipCost: 0,
+    saved: 0,
+    savedPercent: 0,
+    fallbacks: 0,
+    errors: 0,
+  };
+}
+
+function bumpBucket(bucket, key, inputTokens, outputTokens, actualCost, flagshipCost) {
+  if (!bucket[key]) {
+    bucket[key] = {
+      requests: 0,
+      inputTokens: 0,
+      outputTokens: 0,
+      totalTokens: 0,
+      actualCost: 0,
+      flagshipCost: 0,
+      saved: 0,
+    };
+  }
+  const b = bucket[key];
+  b.requests += 1;
+  b.inputTokens += inputTokens;
+  b.outputTokens += outputTokens;
+  b.totalTokens += inputTokens + outputTokens;
+  b.actualCost += actualCost;
+  b.flagshipCost += flagshipCost;
+  b.saved += Math.max(0, flagshipCost - actualCost);
+}
+
+/**
+ * Compute and finalise totals (savedPercent etc.) on a usage object.
+ * Mutates and returns the object — convenient for chaining.
+ */
+function finalise(usage) {
+  const t = usage.totals;
+  t.savedPercent = t.flagshipCost > 0 ? Math.round((t.saved / t.flagshipCost) * 1000) / 10 : 0;
+  for (const bucket of [usage.byTier, usage.byProvider, usage.byModel]) {
+    for (const key of Object.keys(bucket)) {
+      const b = bucket[key];
+      b.savedPercent = b.flagshipCost > 0 ? Math.round((b.saved / b.flagshipCost) * 1000) / 10 : 0;
+    }
+  }
+  return usage;
+}
+
+module.exports = {
+  getUsage: (options) => finalise(getUsage(options)),
+  resolveSince,
+  DEFAULT_FLAGSHIP_MODEL,
+  WINDOW_PRESETS,
+};