lynkr 9.7.2 → 9.9.0

package/src/context/gcf.js

@@ -0,0 +1,275 @@
+const logger = require("../logger");
+const { countTokens } = require("../routing/tokenizer");
+
+// GCF (Graph Compact Format) context compression. Drop-in alternative to the
+// TOON adapter (src/context/toon.js): same encode-only, fail-open,
+// read-only-context contract; only the encoder differs. Round-trips losslessly.
+//
+// @blackwell-systems/gcf ships a CommonJS build, so the encoder is resolved with
+// a synchronous require and cached.
+
+let cachedEncode;
+let cachedDecode;
+let cachedLoadError;
+let warnedMissingDependency = false;
+
+// A clear byte reduction reliably implies a token reduction, so blobs that shrink by
+// at least this margin convert without paying to tokenize both strings. Blobs in the
+// ambiguous zone (or larger) fall through to the exact token comparison.
+const BYTE_FASTPATH_RATIO = 0.9;
+
+function normaliseSettings(settings = {}) {
+  const minBytesRaw =
+    typeof settings.minBytes === "number" ? settings.minBytes : Number.parseInt(settings.minBytes ?? "4096", 10);
+  return {
+    enabled: settings.enabled === true,
+    minBytes: Number.isFinite(minBytesRaw) && minBytesRaw > 0 ? minBytesRaw : 4096,
+    failOpen: settings.failOpen !== false,
+    logStats: settings.logStats !== false,
+    verify: settings.verify !== false,
+  };
+}
+
+// Order-insensitive for object keys, order-sensitive for arrays: the round-trip
+// check treats two JSON values as equal when they carry the same data.
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (a === null || b === null || typeof a !== typeof b) return false;
+  if (Array.isArray(a)) {
+    if (!Array.isArray(b) || a.length !== b.length) return false;
+    return a.every((v, i) => deepEqual(v, b[i]));
+  }
+  if (typeof a === "object") {
+    const ka = Object.keys(a);
+    const kb = Object.keys(b);
+    if (ka.length !== kb.length) return false;
+    return ka.every((k) => Object.prototype.hasOwnProperty.call(b, k) && deepEqual(a[k], b[k]));
+  }
+  return false;
+}
+
+function resolveEncodeFn(overrideEncode) {
+  if (typeof overrideEncode === "function") return overrideEncode;
+  if (cachedEncode !== undefined) return cachedEncode;
+  try {
+    const mod = require("@blackwell-systems/gcf");
+    const fn = mod?.encodeGeneric ?? mod?.default?.encodeGeneric ?? null;
+    cachedEncode = typeof fn === "function" ? fn : null;
+    cachedLoadError = cachedEncode
+      ? null
+      : new Error("Missing encodeGeneric() export from @blackwell-systems/gcf");
+  } catch (err) {
+    cachedEncode = null;
+    cachedLoadError = err;
+  }
+  return cachedEncode;
+}
+
+function resolveDecodeFn(overrideDecode) {
+  if (typeof overrideDecode === "function") return overrideDecode;
+  if (cachedDecode !== undefined) return cachedDecode;
+  try {
+    const mod = require("@blackwell-systems/gcf");
+    const fn = mod?.decodeGeneric ?? mod?.default?.decodeGeneric ?? null;
+    cachedDecode = typeof fn === "function" ? fn : null;
+  } catch {
+    cachedDecode = null;
+  }
+  return cachedDecode;
+}
+
+function looksLikeJsonObjectOrArray(text) {
+  if (typeof text !== "string") return false;
+  const trimmed = text.trim();
+  if (trimmed.length < 2) return false;
+  return (
+    (trimmed.startsWith("{") && trimmed.endsWith("}")) ||
+    (trimmed.startsWith("[") && trimmed.endsWith("]"))
+  );
+}
+
+function safeJsonParse(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return null;
+  }
+}
+
+function toGcfString(encodeFn, value) {
+  const encoded = encodeFn(value);
+  if (typeof encoded === "string") return encoded;
+  if (encoded && typeof encoded[Symbol.iterator] === "function") {
+    return Array.from(encoded).join("\n");
+  }
+  return "";
+}
+
+function compressStringContent(content, cfg, encodeFn, decodeFn, stats, model) {
+  if (typeof content !== "string") return content;
+
+  const originalBytes = Buffer.byteLength(content, "utf8");
+  if (originalBytes < cfg.minBytes) {
+    stats.skippedBySize += 1;
+    return content;
+  }
+
+  stats.candidateCount += 1;
+  if (!looksLikeJsonObjectOrArray(content)) {
+    stats.skippedByShape += 1;
+    return content;
+  }
+
+  const parsed = safeJsonParse(content);
+  if (!parsed || typeof parsed !== "object") {
+    stats.skippedByParse += 1;
+    return content;
+  }
+
+  const gcfText = toGcfString(encodeFn, parsed);
+  if (typeof gcfText !== "string" || gcfText.trim().length === 0) {
+    return content;
+  }
+
+  const gcfBytes = Buffer.byteLength(gcfText, "utf8");
+
+  // Fast path: a clear byte reduction reliably means a token reduction too, so skip
+  // the (per-blob, hot-path) token comparison.
+  if (gcfBytes <= originalBytes * BYTE_FASTPATH_RATIO) {
+    if (!verifiesLossless(parsed, gcfText, cfg, decodeFn, stats)) return content;
+    stats.convertedCount += 1;
+    stats.originalBytes += originalBytes;
+    stats.compressedBytes += gcfBytes;
+    return gcfText;
+  }
+
+  // Ambiguous zone (or larger in bytes): compare token counts with the target model's
+  // encoding, and keep the original JSON if GCF does not reduce the token count.
+  const originalTokens = countTokens(content, model);
+  const gcfTokens = countTokens(gcfText, model);
+  if (gcfTokens >= originalTokens) {
+    stats.skippedByGrowth += 1;
+    return content;
+  }
+
+  if (!verifiesLossless(parsed, gcfText, cfg, decodeFn, stats)) return content;
+  stats.convertedCount += 1;
+  stats.originalBytes += originalBytes;
+  stats.compressedBytes += gcfBytes;
+  stats.originalTokens += originalTokens;
+  stats.compressedTokens += gcfTokens;
+  return gcfText;
+}
+
+// Round-trip check on a payload we have otherwise decided to convert: decode the
+// encoding and require that it reproduces the input exactly. GCF is lossless by
+// design, so this is insurance rather than an expected path; it makes the
+// compression provably lossless per payload. Returns true (safe to convert) when
+// verification is off or the decoder is unavailable.
+function verifiesLossless(parsed, gcfText, cfg, decodeFn, stats) {
+  if (!cfg.verify || typeof decodeFn !== "function") return true;
+  let decoded;
+  try {
+    decoded = decodeFn(gcfText);
+  } catch {
+    decoded = undefined;
+  }
+  if (deepEqual(parsed, decoded)) return true;
+  stats.skippedByVerify += 1;
+  return false;
+}
+
+function applyGcfCompression(payload, settings = {}, options = {}) {
+  const cfg = normaliseSettings(settings);
+  const stats = {
+    enabled: cfg.enabled,
+    available: true,
+    convertedCount: 0,
+    candidateCount: 0,
+    skippedBySize: 0,
+    skippedByShape: 0,
+    skippedByParse: 0,
+    skippedByGrowth: 0,
+    skippedByVerify: 0,
+    failureCount: 0,
+    originalBytes: 0,
+    compressedBytes: 0,
+    originalTokens: 0,
+    compressedTokens: 0,
+  };
+
+  if (!cfg.enabled) return { payload, stats };
+  if (!payload || !Array.isArray(payload.messages) || payload.messages.length === 0) {
+    return { payload, stats };
+  }
+
+  // Token counts for the never-grow guard use the target model's encoding when known.
+  const model = typeof payload.model === "string" ? payload.model : null;
+
+  const encodeFn = resolveEncodeFn(options.encode);
+  if (typeof encodeFn !== "function") {
+    stats.available = false;
+    const err = cachedLoadError ?? new Error("GCF encoder unavailable");
+    if (!cfg.failOpen) throw err;
+    if (!warnedMissingDependency) {
+      logger.warn(
+        { error: err.message },
+        "GCF enabled but encoder dependency is unavailable; falling back to JSON",
+      );
+      warnedMissingDependency = true;
+    }
+    return { payload, stats };
+  }
+
+  const decodeFn = cfg.verify ? resolveDecodeFn(options.decode) : null;
+
+  for (const message of payload.messages) {
+    if (!message || typeof message !== "object") continue;
+    if (message.role === "tool") continue; // Never mutate machine-executed protocol payloads
+    try {
+      if (typeof message.content === "string") {
+        message.content = compressStringContent(message.content, cfg, encodeFn, decodeFn, stats, model);
+        continue;
+      }
+
+      if (!Array.isArray(message.content)) continue;
+      for (const block of message.content) {
+        if (!block || typeof block !== "object") continue;
+
+        // Keep protocol blocks untouched. Only compress user-language text fields.
+        if (block.type === "text" && typeof block.text === "string") {
+          block.text = compressStringContent(block.text, cfg, encodeFn, decodeFn, stats, model);
+          continue;
+        }
+
+        if (block.type === "input_text" && typeof block.input_text === "string") {
+          block.input_text = compressStringContent(block.input_text, cfg, encodeFn, decodeFn, stats, model);
+        }
+      }
+    } catch (err) {
+      stats.failureCount += 1;
+      if (!cfg.failOpen) throw err;
+    }
+  }
+
+  if (cfg.logStats && (stats.convertedCount > 0 || stats.skippedByVerify > 0)) {
+    logger.info(
+      {
+        convertedCount: stats.convertedCount,
+        candidateCount: stats.candidateCount,
+        skippedByVerify: stats.skippedByVerify,
+        originalBytes: stats.originalBytes,
+        compressedBytes: stats.compressedBytes,
+        originalTokens: stats.originalTokens,
+        compressedTokens: stats.compressedTokens,
+      },
+      "GCF compression applied to message context",
+    );
+  }
+
+  return { payload, stats };
+}
+
+module.exports = {
+  applyGcfCompression,
+};

package/src/context/tool-result-compressor.js

@@ -56,7 +56,11 @@ function recordMetric(pattern, originalLen, compressedLen) {
     metrics.patterns[pattern] = { count: 0, tokensSaved: 0 };
   }
   metrics.patterns[pattern].count++;
-  metrics.patterns[pattern].tokensSaved += Math.ceil((originalLen - compressedLen) / 4);
+  const saved = Math.ceil((originalLen - compressedLen) / 4);
+  metrics.patterns[pattern].tokensSaved += saved;
+  try {
+    require('../routing/telemetry').recordSavings('compression', saved);
+  } catch { /* telemetry is best-effort */ }
 }
 
 function getMetrics() {
@@ -120,6 +124,17 @@ function compressTestOutput(text) {
 
   if (summary.length === 0 && failures.length === 0) return null;
 
+  // Corroboration: one summary-looking line alone is not a test run — a
+  // README quoting "1041 passing" would otherwise compress the whole doc
+  // down to that quote. Require failures, a second summary line, or at
+  // least three per-test result markers before treating it as a test run.
+  if (failures.length === 0 && summary.length < 2) {
+    const perTestMarkers = lines.filter(l =>
+      /^\s*(?:✓|✔|✗|✘|ok \d+|not ok \d+|PASS\b|FAIL\b)/.test(l.trim())
+    ).length;
+    if (perTestMarkers < 3) return null;
+  }
+
   const parts = [];
   if (summary.length > 0) parts.push(summary.join("\n"));
   if (failures.length > 0) {
@@ -172,9 +187,12 @@ function compressGitDiff(text) {
 
 // 3. Git status
 function compressGitStatus(text) {
+  // Anchor on git-status STRUCTURE (branch line or section headers), not on
+  // bare "modified:" / "new file:" substrings — source code or prose that
+  // merely contains those strings must not be compressed into a fake status.
   if (!text.includes("Changes not staged") && !text.includes("Changes to be committed") &&
-      !text.includes("Untracked files") && !text.includes("On branch") &&
-      !text.includes("modified:") && !text.includes("new file:")) return null;
+      !text.includes("Untracked files") && !/^On branch \S+/m.test(text) &&
+      !/^HEAD detached/m.test(text)) return null;
 
   const staged = [];
   const modified = [];
@@ -438,18 +456,35 @@ function extractJSONStructure(obj, depth, maxDepth) {
   return typeof obj;
 }
 
-// 10. Docker/kubectl output
+// 10. Docker/kubectl table output (docker ps, docker images, kubectl get).
+// Detection anchors on the TABLE HEADER, never on a keyword appearing
+// anywhere in the text: the old /docker/i containment trigger fired on any
+// `ls` output that contained "Dockerfile", truncated the listing to 10
+// lines, and the model hallucinated the dropped file names.
+const CONTAINER_HEADER_COLUMNS = [
+  "CONTAINER ID", "IMAGE ID", "IMAGE", "COMMAND", "CREATED", "STATUS",
+  "PORTS", "NAMES", "REPOSITORY", "TAG", "SIZE",
+  "NAMESPACE", "NAME", "READY", "RESTARTS", "AGE", "CLUSTER-IP",
+  "EXTERNAL-IP", "TYPE", "DESIRED", "CURRENT", "AVAILABLE", "UP-TO-DATE",
+];
 function compressContainerOutput(text) {
-  const isDocker = /(?:CONTAINER ID|IMAGE|PORTS|STATUS|docker|NAMESPACE|READY|RESTARTS|AGE|kubectl|pod\/)/i.test(text);
-  if (!isDocker) return null;
-
   const lines = text.split("\n").filter(l => l.trim());
   if (lines.length < 3) return null;
 
-  // Keep header + data rows, strip verbose columns
+  // The first line must be a real docker/kubectl column header: at least
+  // three known column tokens and nothing but known tokens and whitespace.
   const header = lines[0];
-  const dataLines = lines.slice(1).filter(l => l.trim());
+  let rest = header;
+  let columns = 0;
+  for (const col of CONTAINER_HEADER_COLUMNS) {
+    if (rest.includes(col)) {
+      columns++;
+      rest = rest.split(col).join(" ");
+    }
+  }
+  if (columns < 3 || rest.trim() !== "") return null;
 
+  const dataLines = lines.slice(1);
   if (dataLines.length <= 10) return null; // Not enough to compress
 
   return `${header}\n${dataLines.slice(0, 10).join("\n")}\n... +${dataLines.length - 10} more (${dataLines.length} total)`;
@@ -557,6 +592,13 @@ function compressSmartTruncate(text) {
 }
 
 // ── Compression Pipeline ─────────────────────────────────────────────
+//
+// DETECTION RULE: a compressor's trigger must anchor on the STRUCTURE of
+// its target format (header line anatomy, per-line shape, section markers)
+// — never on a keyword appearing anywhere in the text. A misfire doesn't
+// just waste tokens: it silently drops content the model then hallucinates
+// back (live incident: /docker/i matched "Dockerfile" in an ls listing,
+// truncated it to 10 lines, and the model invented the rest).
 
 const COMPRESSORS = [
   { name: "test_output", fn: compressTestOutput },

package/src/dashboard/api.js

@@ -13,6 +13,7 @@ function providerMeta() {
     'azure-anthropic': { type: 'cloud', configured: !!(c.azureAnthropic?.endpoint && c.azureAnthropic?.apiKey) },
     bedrock:           { type: 'cloud', configured: !!c.bedrock?.apiKey },
     openrouter:        { type: 'cloud', configured: !!c.openrouter?.apiKey },
+    edenai:            { type: 'cloud', configured: !!c.edenai?.apiKey },
     openai:            { type: 'cloud', configured: !!c.openai?.apiKey },
     'azure-openai':    { type: 'cloud', configured: !!(c.azureOpenAI?.endpoint && c.azureOpenAI?.apiKey) },
     vertex:            { type: 'cloud', configured: !!c.vertex?.projectId },

package/src/logger/index.js

@@ -95,10 +95,23 @@ if (config.oversizedErrorLogging?.enabled) {
 	});
 }
 
+// Root level must be the most verbose of any enabled stream, or quieter
+// streams starve louder ones: LOG_LEVEL=silent (the recommended wrap
+// setting) used to make LOG_FILE_ENABLED capture nothing — the exact
+// sessions that most need a log file. Per-stream levels still apply, so
+// the console stays at LOG_LEVEL.
+const LEVEL_ORDER = ["trace", "debug", "info", "warn", "error", "fatal", "silent"];
+const rootLevel = streams
+	.map((s) => s.level)
+	.reduce(
+		(min, l) => (LEVEL_ORDER.indexOf(l) < LEVEL_ORDER.indexOf(min) ? l : min),
+		config.logger.level,
+	);
+
 // Create logger with multistream
 const logger = pino(
 	{
-		level: config.logger.level,
+		level: rootLevel,
 		name: "claude-backend",
 		base: {
 			env: config.env,

package/src/memory/search.js

@@ -10,7 +10,7 @@ const MAX_QUERY_LENGTH = 1000;
 const MAX_OR_TERMS = 50;
 
 // ============================================================================
-// KEYWORD SANITIZATION (NEW - Critical for FTS5 Safety)
+// KEYWORD SANITIZATION
 // ============================================================================
 
 /**
@@ -52,13 +52,13 @@ function sanitizeKeywords(keywords) {
 }
 
 // ============================================================================
-// FTS5 QUERY PREPARATION (UPDATED - Hardened)
+// FTS5 QUERY PREPARATION
 // ============================================================================
 
 /**
  * Prepare FTS5 query - handle special characters and phrases
  * 
- * CRITICAL FIX for better-sqlite3 v12+ (SQLite 3.46+):
+ * For better-sqlite3 v12+ (SQLite 3.46+):
  * - Commas and periods inside quoted strings cause "fts5: syntax error near ,"
  * - Solution: Extract keywords and search for them individually with OR
  * - This is more robust than attempting to quote complex phrases
@@ -66,7 +66,6 @@ function sanitizeKeywords(keywords) {
 function prepareFTS5Query(query) {
   let cleaned = query.trim();
 
-  // Length validation
   if (cleaned.length > MAX_QUERY_LENGTH) {
     logger.warn({ 
       queryLength: cleaned.length, 
@@ -145,7 +144,7 @@ function prepareFTS5Query(query) {
 }
 
 // ============================================================================
-// SEARCH FUNCTIONS (UPDATED)
+// SEARCH FUNCTIONS
 // ============================================================================
 
 /**
@@ -175,7 +174,6 @@ function searchMemories(options) {
       ftsQuery: ftsQuery.substring(0, 100) 
     }, 'FTS5 query prepared');
 
-    // Build SQL with filters
     let sql = `
       SELECT m.id, m.session_id, m.content, m.type, m.category,
              m.importance, m.surprise_score, m.access_count, m.decay_factor,
@@ -188,7 +186,6 @@ function searchMemories(options) {
 
     const params = [ftsQuery];
 
-    // Add filters
     if (sessionId) {
       sql += ` AND (m.session_id = ? OR m.session_id IS NULL)`;
       params.push(sessionId);
@@ -211,7 +208,6 @@ function searchMemories(options) {
       params.push(minImportance);
     }
 
-    // Order by FTS5 rank and importance
     sql += ` ORDER BY memories_fts.rank, m.importance DESC LIMIT ?`;
     params.push(limit);
 
@@ -264,9 +260,8 @@ function searchMemories(options) {
 function searchWithExpansion(options) {
   const { query, limit = 10 } = options;
 
-  // Extract keywords from query
   const keywords = extractKeywords(query);
-  const sanitizedKeywords = sanitizeKeywords(keywords);  // ✅ ADDED
+  const sanitizedKeywords = sanitizeKeywords(keywords);
 
   // Search with original query (already sanitized by prepareFTS5Query)
   const results = searchMemories({ ...options, limit: limit * 2 });
@@ -275,7 +270,7 @@ function searchWithExpansion(options) {
   if (results.length < limit && sanitizedKeywords.length > 1) {
     const seen = new Set(results.map((r) => r.id));
 
-    for (const keyword of sanitizedKeywords) {  // ✅ CHANGED - use sanitized
+    for (const keyword of sanitizedKeywords) {
       if (results.length >= limit) break;
 
       const kwResults = searchMemories({
@@ -315,7 +310,7 @@ function extractKeywords(text) {
 }
 
 /**
- * Find similar memories by keyword overlap (UPDATED - sanitized)
+ * Find similar memories by keyword overlap
  */
 function findSimilar(memoryId, limit = 5) {
   const memory = store.getMemory(memoryId);
@@ -324,12 +319,12 @@ function findSimilar(memoryId, limit = 5) {
   }
 
   const keywords = extractKeywords(memory.content);
-  const sanitizedKeywords = sanitizeKeywords(keywords);  // ✅ ADDED
-  
+  const sanitizedKeywords = sanitizeKeywords(keywords);
+
   if (sanitizedKeywords.length === 0) return [];
 
   // Build OR query with SANITIZED keywords
-  const query = sanitizedKeywords.join(" OR ");  // ✅ CHANGED - use sanitized
+  const query = sanitizedKeywords.join(" OR ");
 
   const results = searchMemories({
     query,
@@ -339,27 +334,6 @@ function findSimilar(memoryId, limit = 5) {
   return results.filter((r) => r.id !== memoryId).slice(0, limit);
 }
 
-/**
- * Search by content similarity (UPDATED - sanitized)
- */
-function searchByContent(content, options = {}) {
-  const keywords = extractKeywords(content);
-  const sanitizedKeywords = sanitizeKeywords(keywords);  // ✅ ADDED
-  
-  if (sanitizedKeywords.length === 0) return [];
-
-  const query = sanitizedKeywords.slice(0, 5).join(" OR ");  // ✅ CHANGED
-  return searchMemories({ ...options, query });
-}
-
-/**
- * Count search results without fetching them
- */
-function countSearchResults(options) {
-  const results = searchMemories({ ...options, limit: 1000 });
-  return results.length;
-}
-
 // ============================================================================
 // EXPORTS
 // ============================================================================
@@ -369,9 +343,7 @@ module.exports = {
   searchWithExpansion,
   extractKeywords,
   findSimilar,
-  searchByContent,
-  countSearchResults,
   prepareFTS5Query,
-  sanitizeKeyword,      // ✅ NEW - exported for testing
-  sanitizeKeywords,     // ✅ NEW - exported for testing
+  sanitizeKeyword,      // exported for testing
+  sanitizeKeywords,     // exported for testing
 };