karajan-code 1.58.2 → 1.59.1

package/src/proxy/compression/deterministic/bash-pkg.js

@@ -0,0 +1,49 @@
+/**
+ * Deterministic compression for package manager output.
+ * Handles: npm/yarn/pnpm install, npm list, npm outdated.
+ */
+import { stripAnsi, collapseWhitespace, truncateLines } from "./utils.js";
+
+const PKG_PATTERNS = [
+  /added \d+ packages?/i,
+  /npm warn|npm error/i,
+  /packages? in \d/,
+  /├──|└──|│/,
+  /\d+ vulnerabilities/i,
+  /npm outdated/i,
+  /Current\s+Wanted\s+Latest/i
+];
+
+export function looksLike(text) {
+  const clean = stripAnsi(text);
+  return PKG_PATTERNS.some((p) => p.test(clean));
+}
+
+export function compact(text) {
+  const clean = stripAnsi(text);
+
+  // npm install summary
+  if (/added \d+ packages?/i.test(clean)) {
+    const lines = clean.split("\n");
+    const kept = [];
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (/added \d+ packages?/i.test(trimmed) || /vulnerabilit/i.test(trimmed)) {
+        kept.push(trimmed);
+      }
+    }
+    return kept.join("\n") || clean;
+  }
+
+  // npm list — truncate tree
+  if (/├──|└──/.test(clean)) {
+    return truncateLines(clean, 25);
+  }
+
+  // npm outdated — compact table
+  if (/Current\s+Wanted\s+Latest/i.test(clean)) {
+    return truncateLines(collapseWhitespace(clean), 20);
+  }
+
+  return truncateLines(collapseWhitespace(clean), 30);
+}

package/src/proxy/compression/deterministic/bash-test.js

@@ -0,0 +1,61 @@
+/**
+ * Deterministic compression for test runner output.
+ * Handles: vitest, jest, mocha, playwright, pytest, cargo test, go test.
+ */
+import { stripAnsi, collapseWhitespace, truncateLines } from "./utils.js";
+
+const TEST_PATTERNS = [
+  /Tests?\s+\d+\s+(passed|failed)/i,
+  /✓|✗|✘|PASS|FAIL/,
+  /Test Suites?:/,
+  /test result:/i,
+  /FAILED|PASSED|ERROR/,
+  /vitest|jest|mocha|playwright|pytest|cargo test|go test/i,
+  /\d+ passing/,
+  /\d+ failing/
+];
+
+export function looksLike(text) {
+  const clean = stripAnsi(text);
+  return TEST_PATTERNS.some((p) => p.test(clean));
+}
+
+export function compact(text) {
+  const clean = stripAnsi(text);
+
+  // Extract summary line if present
+  const summaryPatterns = [
+    /Tests?\s+\d+\s+(passed|failed).*/gi,
+    /Test Suites?:.*$/gm,
+    /\d+ passing.*$/gm,
+    /\d+ failing.*$/gm,
+    /test result:.*$/gim,
+    /Tests:.*\d+ total$/gm
+  ];
+
+  const summaries = [];
+  for (const p of summaryPatterns) {
+    const matches = clean.match(p);
+    if (matches) summaries.push(...matches.map((m) => m.trim()));
+  }
+
+  // Extract failed test details (name + error)
+  const failures = [];
+  const failBlocks = clean.split(/(?=(?:FAIL|✗|✘|×)\s)/);
+  for (const block of failBlocks) {
+    if (/^(?:FAIL|✗|✘|×)\s/.test(block)) {
+      failures.push(truncateLines(block.trim(), 8));
+    }
+  }
+
+  const parts = [];
+  if (summaries.length) parts.push(summaries.join("\n"));
+  if (failures.length) {
+    parts.push(`Failed tests:\n${failures.join("\n---\n")}`);
+  }
+
+  if (parts.length) return parts.join("\n\n");
+
+  // Fallback: collapse and truncate
+  return truncateLines(collapseWhitespace(clean), 50);
+}

package/src/proxy/compression/deterministic/glob.js

@@ -0,0 +1,26 @@
+/**
+ * Deterministic compression for glob/find/ls output.
+ * Truncates long file listings while keeping count.
+ */
+import { stripAnsi, truncateLines } from "./utils.js";
+
+const MAX_FILES_SHOWN = 40;
+
+export function looksLike(text) {
+  const clean = stripAnsi(text);
+  const lines = clean.split("\n").filter((l) => l.trim());
+  if (lines.length < 5) return false;
+  // Most lines look like file paths
+  const pathLike = lines.filter((l) => /^[\w./-]+\.\w+$/.test(l.trim()) || /^\//.test(l.trim()));
+  return pathLike.length >= lines.length * 0.6;
+}
+
+export function compact(text) {
+  const clean = stripAnsi(text);
+  const lines = clean.split("\n").filter((l) => l.trim());
+  if (lines.length <= MAX_FILES_SHOWN) return clean;
+
+  const shown = lines.slice(0, MAX_FILES_SHOWN);
+  shown.push(`\n... (${lines.length} total files, showing first ${MAX_FILES_SHOWN})`);
+  return shown.join("\n");
+}

package/src/proxy/compression/deterministic/grep.js

@@ -0,0 +1,53 @@
+/**
+ * Deterministic compression for grep/ripgrep output.
+ * Collapses repetitive results, keeps first N per file + total count.
+ */
+import { stripAnsi, truncateLines } from "./utils.js";
+
+const MAX_MATCHES_PER_FILE = 3;
+const MAX_FILES = 15;
+
+export function looksLike(text) {
+  const clean = stripAnsi(text);
+  // grep-style output: file:line:content or file:content
+  const lines = clean.split("\n").filter((l) => l.trim());
+  if (lines.length < 2) return false;
+  const matchingLines = lines.filter((l) => /^[^\s:]+:\d+:/.test(l) || /^[^\s:]+:/.test(l));
+  return matchingLines.length >= lines.length * 0.5;
+}
+
+export function compact(text) {
+  const clean = stripAnsi(text);
+  const lines = clean.split("\n").filter((l) => l.trim());
+
+  // Group by file
+  const byFile = new Map();
+  for (const line of lines) {
+    const match = line.match(/^([^\s:]+):/);
+    const file = match ? match[1] : "__other__";
+    if (!byFile.has(file)) byFile.set(file, []);
+    byFile.get(file).push(line);
+  }
+
+  const totalFiles = byFile.size;
+  const totalMatches = lines.length;
+  const result = [];
+  let fileCount = 0;
+
+  for (const [file, matches] of byFile) {
+    if (fileCount >= MAX_FILES) break;
+    fileCount++;
+    const shown = matches.slice(0, MAX_MATCHES_PER_FILE);
+    result.push(...shown);
+    if (matches.length > MAX_MATCHES_PER_FILE) {
+      result.push(`  ... (${matches.length - MAX_MATCHES_PER_FILE} more matches in ${file})`);
+    }
+  }
+
+  if (totalFiles > MAX_FILES) {
+    result.push(`\n... (${totalFiles - MAX_FILES} more files)`);
+  }
+  result.push(`\nTotal: ${totalMatches} matches in ${totalFiles} files`);
+
+  return result.join("\n");
+}

package/src/proxy/compression/deterministic/index.js

@@ -0,0 +1,58 @@
+/**
+ * Deterministic compression dispatcher.
+ * Tries known patterns in order of specificity; returns original text if nothing matches.
+ */
+import * as bashGit from "./bash-git.js";
+import * as bashTest from "./bash-test.js";
+import * as bashBuild from "./bash-build.js";
+import * as bashInfra from "./bash-infra.js";
+import * as bashPkg from "./bash-pkg.js";
+import * as bashMisc from "./bash-misc.js";
+import * as grepMod from "./grep.js";
+import * as readMod from "./read.js";
+import * as globMod from "./glob.js";
+
+/** Compressors keyed by tool name hint. */
+const TOOL_COMPRESSORS = {
+  Grep: grepMod,
+  Read: readMod,
+  Glob: globMod
+};
+
+/** Bash compressors tried in order of specificity. */
+const BASH_COMPRESSORS = [
+  bashGit,
+  bashTest,
+  bashBuild,
+  bashInfra,
+  bashPkg,
+  bashMisc
+];
+
+/**
+ * Compress tool output deterministically.
+ * @param {string} text - raw tool output
+ * @param {string} [toolName] - tool name hint (e.g. "Bash", "Grep", "Read", "Glob")
+ * @returns {{ text: string, compressed: boolean }}
+ */
+export function compressDeterministic(text, toolName = "") {
+  if (!text || typeof text !== "string") return { text: text ?? "", compressed: false };
+
+  // Direct tool match
+  const directCompressor = TOOL_COMPRESSORS[toolName];
+  if (directCompressor && directCompressor.looksLike(text)) {
+    return { text: directCompressor.compact(text), compressed: true };
+  }
+
+  // For Bash or unknown tools, try all bash compressors in order
+  if (!toolName || toolName === "Bash") {
+    for (const compressor of BASH_COMPRESSORS) {
+      if (compressor.looksLike(text)) {
+        return { text: compressor.compact(text), compressed: true };
+      }
+    }
+  }
+
+  // No pattern matched
+  return { text, compressed: false };
+}

package/src/proxy/compression/deterministic/read.js

@@ -0,0 +1,22 @@
+/**
+ * Deterministic compression for file read output.
+ * Truncates very long file contents while preserving structure.
+ */
+import { truncateLines, countTokens } from "./utils.js";
+
+const MAX_LINES = 200;
+const TOKEN_THRESHOLD = 2000;
+
+export function looksLike(text) {
+  // Read output is generic text; we use this as a fallback for very long content
+  // that looks like source code or structured data.
+  if (countTokens(text) < TOKEN_THRESHOLD) return false;
+  const lines = text.split("\n");
+  if (lines.length < 50) return false;
+  // Heuristic: numbered lines (cat -n style) or code-like content
+  return /^\s*\d+\t/.test(lines[0]) || /^(import |export |const |function |class |def |package )/.test(lines[0]);
+}
+
+export function compact(text) {
+  return truncateLines(text, MAX_LINES);
+}

package/src/proxy/compression/deterministic/utils.js

@@ -0,0 +1,33 @@
+/**
+ * Shared utilities for deterministic compression.
+ */
+
+/** Strip ANSI escape sequences (colors, bold, etc.) */
+export function stripAnsi(text) {
+  // eslint-disable-next-line no-control-regex
+  return text.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, "");
+}
+
+/** Collapse consecutive whitespace (spaces/tabs) into a single space per line. */
+export function collapseWhitespace(text) {
+  return text
+    .split("\n")
+    .map((line) => line.replace(/[ \t]+/g, " ").trim())
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n");
+}
+
+/** Keep at most `max` lines, appending a truncation note. */
+export function truncateLines(text, max = 100) {
+  const lines = text.split("\n");
+  if (lines.length <= max) return text;
+  const kept = lines.slice(0, max);
+  kept.push(`... (${lines.length - max} more lines truncated)`);
+  return kept.join("\n");
+}
+
+/** Rough token estimate: chars / 4. */
+export function countTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}

package/src/proxy/compression/pipeline.js

@@ -0,0 +1,270 @@
+/**
+ * Compression pipeline orchestrator.
+ *
+ * 5-step pipeline: extract → dedup → deterministic → ai-compress → rebuild.
+ *
+ * @module
+ */
+
+import { createHash } from "node:crypto";
+import { compressDeterministic } from "./deterministic/index.js";
+import { countTokens } from "./deterministic/utils.js";
+
+// ---------------------------------------------------------------------------
+// Hashing
+// ---------------------------------------------------------------------------
+
+/** SHA-256 hex digest of a string. */
+function sha256(text) {
+  return createHash("sha256").update(text).digest("hex");
+}
+
+// ---------------------------------------------------------------------------
+// Adaptive pressure
+// ---------------------------------------------------------------------------
+
+/**
+ * Estimate context pressure and return the token threshold above which
+ * tool_results should be compressed.
+ *
+ * @param {Array} messages - conversation messages
+ * @param {number} modelMaxTokens - model context window size in tokens
+ * @returns {{ ratio: number, threshold: number, level: string }}
+ */
+export function estimatePressure(messages, modelMaxTokens) {
+  const totalTokens = countTokens(JSON.stringify(messages));
+  const ratio = totalTokens / modelMaxTokens;
+
+  if (ratio > 0.9) return { ratio, threshold: 200, level: "critical" };
+  if (ratio > 0.8) return { ratio, threshold: 200, level: "high" };
+  if (ratio >= 0.5) return { ratio, threshold: 500, level: "medium" };
+  return { ratio, threshold: 2000, level: "low" };
+}
+
+// ---------------------------------------------------------------------------
+// Caches
+// ---------------------------------------------------------------------------
+
+/** Cache: hash → compressed text (avoids re-compressing same content). */
+const compressionCache = new Map();
+
+/** Cache: hash → exact original string (preserves identical strings for KV cache warming). */
+const sessionCache = new Map();
+
+/** Reset caches (for testing). */
+export function _resetCaches() {
+  compressionCache.clear();
+  sessionCache.clear();
+}
+
+// ---------------------------------------------------------------------------
+// Deduplication
+// ---------------------------------------------------------------------------
+
+/**
+ * Deduplicate tool results: if the same content (by hash) appeared in an
+ * earlier turn, replace with a back-reference.
+ *
+ * @param {Array<{id: string, toolName: string, text: string, turnIndex: number}>} results
+ * @returns {Map<string, string>} id → replacement text (only for deduped entries)
+ */
+function dedup(results) {
+  const seen = new Map(); // hash → first turnIndex
+  const replacements = new Map();
+
+  for (const r of results) {
+    const hash = sha256(r.text);
+    if (seen.has(hash)) {
+      const firstTurn = seen.get(hash);
+      if (r.turnIndex > firstTurn) {
+        replacements.set(r.id, `[See turn ${firstTurn}, unchanged]`);
+      }
+    } else {
+      seen.set(hash, r.turnIndex);
+    }
+  }
+
+  return replacements;
+}
+
+// ---------------------------------------------------------------------------
+// Persist scheduler
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a debounced persistence scheduler for caches.
+ *
+ * @param {() => Promise<void>} flushFn - function that writes caches to disk
+ * @param {number} intervalMs - debounce interval in ms
+ * @returns {{ markDirty: () => void, flush: () => Promise<void>, destroy: () => void }}
+ */
+export function createPersistScheduler(flushFn, intervalMs = 30_000) {
+  let timer = null;
+  let dirty = false;
+
+  const flush = async () => {
+    if (timer) {
+      clearTimeout(timer);
+      timer = null;
+    }
+    dirty = false;
+    await flushFn();
+  };
+
+  const markDirty = () => {
+    dirty = true;
+    if (timer) clearTimeout(timer);
+    timer = setTimeout(() => flush(), intervalMs);
+  };
+
+  const onSignal = () => {
+    if (dirty) {
+      flush();
+    }
+  };
+
+  process.on("SIGTERM", onSignal);
+  process.on("SIGINT", onSignal);
+
+  const destroy = () => {
+    if (timer) {
+      clearTimeout(timer);
+      timer = null;
+    }
+    process.removeListener("SIGTERM", onSignal);
+    process.removeListener("SIGINT", onSignal);
+  };
+
+  return { markDirty, flush, destroy };
+}
+
+// ---------------------------------------------------------------------------
+// Main pipeline
+// ---------------------------------------------------------------------------
+
+/**
+ * Compress tool_results in a request body using the 5-step pipeline.
+ *
+ * @param {object} body - parsed JSON request body
+ * @param {object} adapter - provider adapter (extractToolResults, rebuildMessages)
+ * @param {object} [config={}] - pipeline configuration
+ * @param {number} [config.modelMaxTokens=200000] - model context window
+ * @param {boolean} [config.aiEnabled=false] - whether AI compression is enabled
+ * @param {Function} [config.compressWithAI] - AI compression function(text, toolName) → string
+ * @returns {Promise<{ body: object, stats: object }>}
+ */
+export async function compressRequest(body, adapter, config = {}) {
+  const {
+    modelMaxTokens = 200_000,
+    aiEnabled = false,
+    compressWithAI = null,
+  } = config;
+
+  const messages = body.messages || [];
+
+  const stats = {
+    originalTokens: 0,
+    compressedTokens: 0,
+    cacheHits: 0,
+    deterministicHits: 0,
+    aiHits: 0,
+  };
+
+  // Step 1: Extract
+  const toolResults = adapter.extractToolResults(messages);
+  if (!toolResults || toolResults.length === 0) {
+    return { body, stats };
+  }
+
+  // Adaptive pressure
+  const { threshold } = estimatePressure(messages, modelMaxTokens);
+
+  // Step 2: Dedup
+  const dedupMap = dedup(toolResults);
+
+  // Build compressed map: id → compressed text
+  const compressedMap = new Map();
+
+  for (const result of toolResults) {
+    const originalTokens = countTokens(result.text);
+    stats.originalTokens += originalTokens;
+
+    // Deduped?
+    if (dedupMap.has(result.id)) {
+      const replacement = dedupMap.get(result.id);
+      compressedMap.set(result.id, replacement);
+      stats.compressedTokens += countTokens(replacement);
+      continue;
+    }
+
+    // Below threshold? Keep original
+    if (originalTokens < threshold) {
+      stats.compressedTokens += originalTokens;
+      continue;
+    }
+
+    const hash = sha256(result.text);
+
+    // Check compression cache
+    if (compressionCache.has(hash)) {
+      const cached = compressionCache.get(hash);
+      compressedMap.set(result.id, cached);
+      stats.compressedTokens += countTokens(cached);
+      stats.cacheHits += 1;
+      continue;
+    }
+
+    // Check session cache (exact string reuse for KV warming)
+    if (sessionCache.has(hash)) {
+      stats.compressedTokens += originalTokens;
+      stats.cacheHits += 1;
+      continue;
+    }
+
+    // Step 3: Deterministic compression
+    let compressed = result.text;
+    let wasCompressed = false;
+
+    const det = compressDeterministic(result.text, result.toolName);
+    if (det.compressed) {
+      compressed = det.text;
+      wasCompressed = true;
+      stats.deterministicHits += 1;
+    }
+
+    // Step 4: AI compression (optional)
+    const compressedTokens = countTokens(compressed);
+    if (
+      aiEnabled &&
+      compressWithAI &&
+      compressedTokens >= threshold &&
+      !wasCompressed
+    ) {
+      try {
+        const aiResult = await compressWithAI(compressed, result.toolName);
+        if (aiResult && countTokens(aiResult) < compressedTokens) {
+          compressed = aiResult;
+          stats.aiHits += 1;
+        }
+      } catch {
+        // AI compression failed; keep deterministic result
+      }
+    }
+
+    if (compressed !== result.text) {
+      compressedMap.set(result.id, compressed);
+      compressionCache.set(hash, compressed);
+      stats.compressedTokens += countTokens(compressed);
+    } else {
+      // Not compressed — store in session cache for KV warming
+      sessionCache.set(hash, result.text);
+      stats.compressedTokens += originalTokens;
+    }
+  }
+
+  // Step 5: Rebuild
+  const rebuiltMessages = adapter.rebuildMessages(messages, compressedMap);
+  const newBody = { ...body, messages: rebuiltMessages };
+
+  return { body: newBody, stats };
+}