lynkr 9.0.1 → 9.1.2

package/src/training/trajectory-compressor.js

@@ -0,0 +1,266 @@
+/**
+ * Trajectory Compressor
+ *
+ * Reads completed agent sessions out of Lynkr's session DB,
+ * joins with routing telemetry to pick up tier / score / outcome
+ * metadata, and emits JSONL training samples for fine-tuning small
+ * models on tool-call generation and tier-routing decisions.
+ *
+ * Each line of the output JSONL is a self-contained sample:
+ *
+ *   {
+ *     "session_id":     "...",
+ *     "messages":       [{"role": "...", "content": ...}, ...],
+ *     "tool_calls":     [...],
+ *     "outcome":        "success" | "error",
+ *     "tier":           "SIMPLE" | "MEDIUM" | "COMPLEX" | "REASONING",
+ *     "complexity_score": 38,
+ *     "model_used":     "gpt-4o",
+ *     "provider_used":  "azure-openai",
+ *     "tokens_in":      1234,
+ *     "tokens_out":     456,
+ *     "latency_ms":     2400,
+ *     "started_at":     "2026-05-03T10:11:12Z",
+ *     "ended_at":       "2026-05-03T10:11:14Z"
+ *   }
+ *
+ * The compressor is read-only — it never modifies the source DBs.
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const db = require("../db");
+const telemetry = require("../routing/telemetry");
+
+// Patterns for the optional --anonymize pass. Order matters: more
+// specific patterns first so they don't get clobbered by generic ones.
+const ANONYMIZE_PATTERNS = [
+  // API keys and bearer tokens
+  [/sk-[A-Za-z0-9_-]{20,}/g, "<API_KEY>"],
+  [/Bearer\s+[A-Za-z0-9._\-]+/gi, "Bearer <REDACTED>"],
+  [/dapi_[A-Za-z0-9_-]+/g, "<DATABRICKS_KEY>"],
+  [/eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, "<JWT>"],
+  // AWS keys
+  [/AKIA[0-9A-Z]{16}/g, "<AWS_ACCESS_KEY>"],
+  // Email addresses
+  [/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, "<EMAIL>"],
+  // Absolute filesystem paths under /Users/<name>/ or /home/<name>/
+  [/\/Users\/[^/\s]+/g, "/Users/<USER>"],
+  [/\/home\/[^/\s]+/g, "/home/<USER>"],
+  // IPs
+  [/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, "<IP>"],
+  // Hostnames containing service-now / corporate domains (configurable)
+  [/[A-Za-z0-9-]+\.service-now\.com/gi, "<SERVICENOW_HOST>"],
+];
+
+function anonymize(value) {
+  if (typeof value === "string") {
+    let out = value;
+    for (const [re, replacement] of ANONYMIZE_PATTERNS) {
+      out = out.replace(re, replacement);
+    }
+    return out;
+  }
+  if (Array.isArray(value)) return value.map(anonymize);
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) out[k] = anonymize(v);
+    return out;
+  }
+  return value;
+}
+
+function parseJsonSafe(text, fallback = null) {
+  if (text == null) return fallback;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return fallback;
+  }
+}
+
+/**
+ * List session ids in a window, optionally filtered by tier.
+ */
+function listSessions({ since = null, tier = null } = {}) {
+  if (!db) return [];
+
+  const rows = since
+    ? db
+        .prepare(
+          "SELECT id, created_at, updated_at, metadata FROM sessions WHERE updated_at >= ? ORDER BY updated_at DESC"
+        )
+        .all(since)
+    : db.prepare("SELECT id, created_at, updated_at, metadata FROM sessions ORDER BY updated_at DESC").all();
+
+  if (!tier) return rows;
+
+  // Tier filter requires joining against routing telemetry — we do that
+  // per-session lazily so we don't pre-load the whole telemetry table.
+  return rows.filter((s) => sessionTier(s.id) === tier);
+}
+
+/**
+ * Find the dominant tier picked across a session's telemetry rows.
+ */
+function sessionTier(sessionId) {
+  try {
+    const rows = telemetry.query({ session_id: sessionId, limit: 1000 });
+    if (rows.length === 0) return null;
+    const counts = {};
+    for (const r of rows) counts[r.tier || "UNKNOWN"] = (counts[r.tier || "UNKNOWN"] || 0) + 1;
+    return Object.entries(counts).sort((a, b) => b[1] - a[1])[0][0];
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Build one trajectory record for a single session.
+ */
+function buildTrajectory(session, options = {}) {
+  if (!db) return null;
+
+  const historyStmt = db.prepare(
+    "SELECT role, type, status, content, metadata, timestamp FROM session_history WHERE session_id = ? ORDER BY timestamp ASC"
+  );
+  const history = historyStmt.all(session.id);
+
+  // Convert each session_history row into a chat message, preserving
+  // tool-call structure when present in metadata.
+  const messages = [];
+  const toolCalls = [];
+
+  for (const row of history) {
+    const meta = parseJsonSafe(row.metadata) || {};
+    const content = parseJsonSafe(row.content, row.content);
+
+    if (row.role === "tool" || row.type === "tool_use" || row.type === "tool_result") {
+      // Capture tool calls as a separate stream alongside the chat
+      toolCalls.push({
+        type: row.type,
+        timestamp: row.timestamp,
+        content,
+        metadata: meta,
+      });
+    }
+
+    if (row.role === "user" || row.role === "assistant" || row.role === "system") {
+      messages.push({
+        role: row.role,
+        content,
+      });
+    }
+  }
+
+  // Pull telemetry records associated with this session to enrich.
+  const teleRows = telemetry.query({ session_id: session.id, limit: 1000 });
+
+  const totals = teleRows.reduce(
+    (acc, r) => {
+      acc.tokens_in += r.input_tokens || 0;
+      acc.tokens_out += r.output_tokens || 0;
+      acc.latency_ms += r.latency_ms || 0;
+      return acc;
+    },
+    { tokens_in: 0, tokens_out: 0, latency_ms: 0 }
+  );
+
+  // Pick the modal tier (most-used) and the most-recent model/provider.
+  const tier = sessionTier(session.id);
+  const last = teleRows[0]; // telemetry.query orders DESC
+  const errorRow = teleRows.find((r) => r.error_type);
+  const outcome = errorRow ? "error" : "success";
+  const complexityAvg =
+    teleRows.length > 0
+      ? Math.round(
+          teleRows.reduce((sum, r) => sum + (r.complexity_score || 0), 0) /
+            teleRows.length
+        )
+      : null;
+
+  let trajectory = {
+    session_id: session.id,
+    messages,
+    tool_calls: toolCalls,
+    outcome,
+    tier,
+    complexity_score: complexityAvg,
+    model_used: last?.model || null,
+    provider_used: last?.provider || null,
+    tokens_in: totals.tokens_in,
+    tokens_out: totals.tokens_out,
+    latency_ms: totals.latency_ms,
+    started_at: new Date(session.created_at).toISOString(),
+    ended_at: new Date(session.updated_at).toISOString(),
+  };
+
+  if (options.anonymize) {
+    trajectory = anonymize(trajectory);
+  }
+  return trajectory;
+}
+
+/**
+ * Stream trajectories to a writable target (a path or a stream).
+ *
+ * @param {Object} options
+ * @param {string|number|Date} [options.since]   Window start (ms / Date / "Nd")
+ * @param {string} [options.tier]                Filter to one tier
+ * @param {boolean} [options.anonymize=false]    Strip PII / paths / secrets
+ * @param {string|stream.Writable} [options.output="-"]   File path, "-" for stdout, or a stream
+ * @param {function} [options.onProgress]        Optional progress callback (count) => void
+ * @returns {{ count: number, output: string }}
+ */
+function exportJsonl(options = {}) {
+  const since = resolveSince(options.since);
+  const sessions = listSessions({ since, tier: options.tier });
+
+  let stream;
+  let outputPath = "-";
+  let closeStream = false;
+
+  if (!options.output || options.output === "-") {
+    stream = process.stdout;
+  } else if (typeof options.output === "string") {
+    outputPath = options.output;
+    fs.mkdirSync(path.dirname(path.resolve(outputPath)), { recursive: true });
+    stream = fs.createWriteStream(outputPath);
+    closeStream = true;
+  } else {
+    stream = options.output;
+  }
+
+  let count = 0;
+  for (const session of sessions) {
+    const trajectory = buildTrajectory(session, options);
+    if (!trajectory || trajectory.messages.length === 0) continue;
+    stream.write(JSON.stringify(trajectory) + "\n");
+    count++;
+    if (options.onProgress) options.onProgress(count);
+  }
+
+  if (closeStream) stream.end();
+  return { count, output: outputPath };
+}
+
+function resolveSince(value) {
+  if (value == null) return null;
+  if (value instanceof Date) return value.getTime();
+  if (typeof value === "number") return value;
+  if (typeof value === "string") {
+    const m = value.match(/^(\d+)d$/);
+    if (m) return Date.now() - parseInt(m[1], 10) * 24 * 60 * 60 * 1000;
+    const parsed = Date.parse(value);
+    if (!Number.isNaN(parsed)) return parsed;
+  }
+  return null;
+}
+
+module.exports = {
+  exportJsonl,
+  buildTrajectory,
+  listSessions,
+  anonymize,
+};

package/src/usage/aggregator.js

@@ -0,0 +1,206 @@
+/**
+ * Usage Aggregator
+ *
+ * Reads routing telemetry from .lynkr/telemetry.db and produces
+ * actionable spend / savings reports.
+ *
+ * The "savings" calculation answers the question:
+ *   "How much would this same workload have cost if every request
+ *    had hit the most expensive flagship model?"
+ *
+ * That's the number Lynkr's tier router exists to make small.
+ */
+
+const telemetry = require("../routing/telemetry");
+const { getCostOptimizer } = require("../routing/cost-optimizer");
+
+// What we treat as the "flagship comparison" — the model a developer
+// would otherwise run every request against if they didn't have Lynkr.
+// Picked to match Claude Code / Cursor defaults.
+const DEFAULT_FLAGSHIP_MODEL = "claude-sonnet-4-5-20250929";
+
+const WINDOW_PRESETS = {
+  "1d": 1 * 24 * 60 * 60 * 1000,
+  "7d": 7 * 24 * 60 * 60 * 1000,
+  "30d": 30 * 24 * 60 * 60 * 1000,
+  all: null,
+};
+
+/**
+ * Resolve a window string ("7d", "30d", "all") or a Date / ISO string
+ * into a `since` timestamp in ms. Returns null for "all".
+ */
+function resolveSince(window) {
+  if (!window || window === "all") return null;
+  if (window instanceof Date) return window.getTime();
+  if (typeof window === "string") {
+    if (WINDOW_PRESETS[window] !== undefined) {
+      return WINDOW_PRESETS[window] === null ? null : Date.now() - WINDOW_PRESETS[window];
+    }
+    if (/^\d+d$/.test(window)) {
+      const days = parseInt(window, 10);
+      return Date.now() - days * 24 * 60 * 60 * 1000;
+    }
+    const parsed = Date.parse(window);
+    if (!Number.isNaN(parsed)) return parsed;
+  }
+  if (typeof window === "number") return window;
+  return null;
+}
+
+/**
+ * Compute usage stats for a time window.
+ *
+ * @param {Object} options
+ * @param {string|Date|number} [options.window="30d"]   "1d", "7d", "30d", "all", ISO string, or epoch ms
+ * @param {string}             [options.flagship]      Model id used for the "what if I'd run flagship-only" comparison
+ * @param {string}             [options.model]         Filter to a single model
+ * @param {string}             [options.provider]      Filter to a single provider
+ * @returns {Object} Aggregated usage report
+ */
+function getUsage(options = {}) {
+  const window = options.window || "30d";
+  const since = resolveSince(window);
+  const flagship = options.flagship || DEFAULT_FLAGSHIP_MODEL;
+
+  const filters = { limit: 100000 };
+  if (since !== null) filters.since = since;
+  if (options.provider) filters.provider = options.provider;
+
+  let rows;
+  try {
+    rows = telemetry.query(filters);
+  } catch (err) {
+    return {
+      window,
+      since: since ? new Date(since).toISOString() : null,
+      flagship,
+      totals: emptyTotals(),
+      byTier: {},
+      byProvider: {},
+      byModel: {},
+      error: err.message,
+    };
+  }
+
+  // Optional model filter (telemetry.query doesn't support it natively)
+  if (options.model) {
+    rows = rows.filter((r) => r.model === options.model);
+  }
+
+  const optimizer = (() => {
+    try {
+      return getCostOptimizer();
+    } catch {
+      return null;
+    }
+  })();
+
+  const totals = emptyTotals();
+  const byTier = {};
+  const byProvider = {};
+  const byModel = {};
+
+  for (const row of rows) {
+    const inputTokens = row.input_tokens || 0;
+    const outputTokens = row.output_tokens || 0;
+    const totalTokens = inputTokens + outputTokens;
+    const actualCost = Number(row.cost_usd) || 0;
+
+    // Hypothetical cost if this same request had hit the flagship model.
+    let flagshipCost = 0;
+    if (optimizer && totalTokens > 0) {
+      try {
+        const est = optimizer.estimateCost(flagship, inputTokens, outputTokens);
+        flagshipCost = (est.inputCost || 0) + (est.outputCost || 0);
+      } catch {
+        flagshipCost = 0;
+      }
+    }
+    const saved = Math.max(0, flagshipCost - actualCost);
+
+    totals.requests += 1;
+    totals.inputTokens += inputTokens;
+    totals.outputTokens += outputTokens;
+    totals.totalTokens += totalTokens;
+    totals.actualCost += actualCost;
+    totals.flagshipCost += flagshipCost;
+    totals.saved += saved;
+    if (row.was_fallback) totals.fallbacks += 1;
+    if (row.error_type) totals.errors += 1;
+
+    bumpBucket(byTier, row.tier || "UNKNOWN", inputTokens, outputTokens, actualCost, flagshipCost);
+    bumpBucket(byProvider, row.provider || "unknown", inputTokens, outputTokens, actualCost, flagshipCost);
+    bumpBucket(byModel, row.model || "unknown", inputTokens, outputTokens, actualCost, flagshipCost);
+  }
+
+  return {
+    window,
+    since: since ? new Date(since).toISOString() : null,
+    flagship,
+    totals,
+    byTier,
+    byProvider,
+    byModel,
+  };
+}
+
+function emptyTotals() {
+  return {
+    requests: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    totalTokens: 0,
+    actualCost: 0,
+    flagshipCost: 0,
+    saved: 0,
+    savedPercent: 0,
+    fallbacks: 0,
+    errors: 0,
+  };
+}
+
+function bumpBucket(bucket, key, inputTokens, outputTokens, actualCost, flagshipCost) {
+  if (!bucket[key]) {
+    bucket[key] = {
+      requests: 0,
+      inputTokens: 0,
+      outputTokens: 0,
+      totalTokens: 0,
+      actualCost: 0,
+      flagshipCost: 0,
+      saved: 0,
+    };
+  }
+  const b = bucket[key];
+  b.requests += 1;
+  b.inputTokens += inputTokens;
+  b.outputTokens += outputTokens;
+  b.totalTokens += inputTokens + outputTokens;
+  b.actualCost += actualCost;
+  b.flagshipCost += flagshipCost;
+  b.saved += Math.max(0, flagshipCost - actualCost);
+}
+
+/**
+ * Compute and finalise totals (savedPercent etc.) on a usage object.
+ * Mutates and returns the object — convenient for chaining.
+ */
+function finalise(usage) {
+  const t = usage.totals;
+  t.savedPercent = t.flagshipCost > 0 ? Math.round((t.saved / t.flagshipCost) * 1000) / 10 : 0;
+  for (const bucket of [usage.byTier, usage.byProvider, usage.byModel]) {
+    for (const key of Object.keys(bucket)) {
+      const b = bucket[key];
+      b.savedPercent = b.flagshipCost > 0 ? Math.round((b.saved / b.flagshipCost) * 1000) / 10 : 0;
+    }
+  }
+  return usage;
+}
+
+module.exports = {
+  getUsage: (options) => finalise(getUsage(options)),
+  resolveSince,
+  DEFAULT_FLAGSHIP_MODEL,
+  WINDOW_PRESETS,
+};

package/src/utils/markdown-ansi.js

@@ -0,0 +1,146 @@
+/**
+ * Markdown → ANSI escape code renderer.
+ *
+ * Activated by MARKDOWN_RENDER_ANSI=true in the environment.
+ * Applied to text blocks in the SSE emission path so clients like claw
+ * receive pre-formatted output without needing their own markdown renderer.
+ *
+ * Deliberately avoids external dependencies — pure regex + string ops.
+ */
+
+// ---------------------------------------------------------------------------
+// ANSI primitives
+// ---------------------------------------------------------------------------
+const R  = '\x1b[0m';          // reset all
+const B  = '\x1b[1m';          // bold on
+const B_ = '\x1b[22m';         // bold off
+const I  = '\x1b[3m';          // italic on
+const I_ = '\x1b[23m';         // italic off
+const S  = '\x1b[9m';          // strikethrough on
+const S_ = '\x1b[29m';         // strikethrough off
+const DIM = '\x1b[2m';         // dim
+
+const CYAN    = '\x1b[1;96m';  // bold bright-cyan  — H1
+const BLUE    = '\x1b[1;94m';  // bold bright-blue  — H2
+const MAGENTA = '\x1b[1;95m';  // bold bright-magenta — H3
+const WHITE_B = '\x1b[1;97m';  // bold white         — H4-H6
+const YELLOW  = '\x1b[33m';    // yellow             — inline code
+const GREEN   = '\x1b[92m';    // bright green       — code block body
+const GRAY    = '\x1b[90m';    // dark gray          — HR / code fence border
+const ORANGE  = '\x1b[38;5;214m'; // orange          — code fence lang tag
+
+// ---------------------------------------------------------------------------
+// Inline formatting (applied to single lines outside code fences)
+// ---------------------------------------------------------------------------
+function inlineFmt(line) {
+  // Bold + italic: ***text***
+  line = line.replace(/\*\*\*(.+?)\*\*\*/g, `${B}${I}$1${I_}${B_}`);
+  // Bold: **text** or __text__
+  line = line.replace(/\*\*(.+?)\*\*/g,  `${B}$1${B_}`);
+  line = line.replace(/__(.+?)__/g,       `${B}$1${B_}`);
+  // Italic: *text* or _text_ (single, not preceded/followed by same char)
+  line = line.replace(/(?<!\*)\*(?!\*)(.+?)(?<!\*)\*(?!\*)/g, `${I}$1${I_}`);
+  line = line.replace(/(?<!_)_(?!_)(.+?)(?<!_)_(?!_)/g,       `${I}$1${I_}`);
+  // Strikethrough: ~~text~~
+  line = line.replace(/~~(.+?)~~/g, `${S}$1${S_}`);
+  // Inline code: `code`  (done last so ANSI inside code isn't re-processed)
+  line = line.replace(/`([^`]+)`/g, `${YELLOW}$1${R}`);
+  return line;
+}
+
+// ---------------------------------------------------------------------------
+// Block-level rendering (processes the whole text at once)
+// ---------------------------------------------------------------------------
+function markdownToAnsi(text) {
+  if (!text) return text;
+
+  const lines  = text.split('\n');
+  const out    = [];
+  let inCode   = false;
+  let codeLang = '';
+
+  for (let i = 0; i < lines.length; i++) {
+    const raw = lines[i];
+
+    // ── Code fence open/close ──────────────────────────────────────────────
+    const fenceMatch = raw.match(/^(`{3,})(.*)/);
+    if (fenceMatch) {
+      if (!inCode) {
+        inCode   = true;
+        codeLang = fenceMatch[2].trim();
+        const tag = codeLang ? ` ${codeLang} ` : '';
+        out.push(`${GRAY}┌─${ORANGE}${tag}${GRAY}${'─'.repeat(Math.max(0, 46 - tag.length))}${R}`);
+      } else {
+        inCode = false;
+        out.push(`${GRAY}└${'─'.repeat(48)}${R}`);
+      }
+      continue;
+    }
+
+    // ── Inside a code block ───────────────────────────────────────────────
+    if (inCode) {
+      out.push(`${GRAY}│ ${GREEN}${raw}${R}`);
+      continue;
+    }
+
+    // ── Horizontal rule ───────────────────────────────────────────────────
+    if (/^[-*_]{3,}\s*$/.test(raw.trim())) {
+      out.push(`${GRAY}${'─'.repeat(50)}${R}`);
+      continue;
+    }
+
+    // ── Headings ──────────────────────────────────────────────────────────
+    const h6 = raw.match(/^(#{1,6})\s+(.*)/);
+    if (h6) {
+      const level = h6[1].length;
+      const title = inlineFmt(h6[2]);
+      const colors = [CYAN, BLUE, MAGENTA, WHITE_B, WHITE_B, WHITE_B];
+      const prefix = ['━━ ', '── ', '   ', '   ', '   ', '   '][level - 1];
+      out.push(`${colors[level - 1]}${prefix}${title}${R}`);
+      continue;
+    }
+
+    // ── Blockquote ────────────────────────────────────────────────────────
+    if (raw.startsWith('> ')) {
+      out.push(`${DIM}│ ${inlineFmt(raw.slice(2))}${R}`);
+      continue;
+    }
+
+    // ── Unordered list ────────────────────────────────────────────────────
+    const ulMatch = raw.match(/^(\s*)[*\-+] (.*)/);
+    if (ulMatch) {
+      const indent = ulMatch[1];
+      const depth  = Math.floor(indent.length / 2);
+      const bullet = ['•', '◦', '▸'][Math.min(depth, 2)];
+      out.push(`${indent}${YELLOW}${bullet}${R} ${inlineFmt(ulMatch[2])}`);
+      continue;
+    }
+
+    // ── Ordered list ──────────────────────────────────────────────────────
+    const olMatch = raw.match(/^(\s*)(\d+)\. (.*)/);
+    if (olMatch) {
+      out.push(`${olMatch[1]}${YELLOW}${olMatch[2]}.${R} ${inlineFmt(olMatch[3])}`);
+      continue;
+    }
+
+    // ── Normal line (apply inline formatting) ─────────────────────────────
+    out.push(inlineFmt(raw));
+  }
+
+  // Close an unclosed code fence gracefully
+  if (inCode) out.push(`${GRAY}└${'─'.repeat(48)}${R}`);
+
+  return out.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+const enabled = process.env.MARKDOWN_RENDER_ANSI === 'true';
+
+function renderText(text) {
+  if (!enabled || !text) return text;
+  return markdownToAnsi(text);
+}
+
+module.exports = { renderText, markdownToAnsi, enabled };