flint-agent 1.14.0

package/src/agent/compression.js

@@ -0,0 +1,310 @@
+// Context compression with configurable threshold
+// Only compresses when total context exceeds config.compressAfterTokens
+//
+// Levels (when compression IS triggered):
+//   Current iteration: full results
+//   Previous iteration (age=1): head/tail truncation
+//   Older iterations (age>=2): one-line type-aware summary
+//
+// Facts extracted SYNCHRONOUSLY before summarizing (await, not fire-and-forget)
+// Action log: deterministic tracking of side-effect tools
+
+import { getSpendLevel, spendSettings, windowShare } from "../spend.js";
+import { extractFacts } from "../memory/extract-facts.js";
+import { saveSessionFacts, saveSessionFact } from "../memory/session-facts.js";
+import { config } from "../config.js";
+import { contextWindow } from "./usage.js";
+import { createLogger } from "../logging/logger.js";
+
+const log = createLogger("compression");
+
+const COMPRESS_THRESHOLD = config.compressThreshold;
+const HEAD_LINES = 8;
+const TAIL_LINES = 3;
+
+// Tools that change state — tracked as action facts
+const SIDE_EFFECT_TOOLS = new Set([
+  "write_file", "edit_file", "delete_file", "copy_file", "move_file", "create_directory",
+  "run_command", "run_background_command",
+]);
+
+// MCP tool patterns that indicate side effects
+const MCP_SIDE_EFFECT_PATTERNS = [
+  /desktop_click/, /desktop_type/, /desktop_key/, /desktop_shell/,
+  /desktop_batch/, /desktop_manage/, /desktop_chrome.*navigate/,
+  /desktop_file/, /gmail_send/, /mesh_add/,
+];
+
+function isSideEffectTool(toolName, toolArgs) {
+  if (SIDE_EFFECT_TOOLS.has(toolName)) return true;
+  for (const pat of MCP_SIDE_EFFECT_PATTERNS) {
+    if (pat.test(toolName)) return true;
+  }
+  // desktop_chrome with action=navigate, type, click
+  if (toolName.includes("desktop_chrome")) {
+    const action = toolArgs?.action || "";
+    if (["navigate", "type", "click", "new_tab", "close_tab"].includes(action)) return true;
+  }
+  return false;
+}
+
+/**
+ * Generate a short action description from a tool call
+ */
+function describeAction(toolName, toolArgs, result) {
+  const shortResult = typeof result === "string" ? result.slice(0, 100) : "";
+
+  if (toolName === "write_file" || toolName === "edit_file") {
+    return `Modified file: ${toolArgs?.path || "?"}`;
+  }
+  if (toolName === "run_command") {
+    const cmd = (toolArgs?.command || "?").slice(0, 60);
+    return `Ran command: ${cmd}`;
+  }
+  if (toolName.includes("desktop_click")) {
+    return `Clicked at (${toolArgs?.x}, ${toolArgs?.y})`;
+  }
+  if (toolName.includes("desktop_shell")) {
+    const cmd = (toolArgs?.command || "?").slice(0, 60);
+    return `Desktop shell: ${cmd}`;
+  }
+  if (toolName.includes("desktop_batch")) {
+    const count = toolArgs?.actions?.length || toolArgs?.action_count || "?";
+    return `Desktop batch: ${count} actions`;
+  }
+  if (toolName.includes("desktop_chrome")) {
+    const action = toolArgs?.action || "?";
+    const url = toolArgs?.url || "";
+    return url ? `Chrome ${action}: ${url.slice(0, 60)}` : `Chrome ${action}`;
+  }
+  if (toolName.includes("desktop_manage")) {
+    const action = toolArgs?.action || "?";
+    const app = toolArgs?.app || "";
+    return app ? `Desktop ${action}: ${app}` : `Desktop ${action}`;
+  }
+  if (toolName.includes("desktop_type")) {
+    return `Typed text (${toolArgs?.text?.length || "?"} chars)`;
+  }
+  if (toolName.includes("desktop_key")) {
+    return `Pressed: ${toolArgs?.keys || "?"}`;
+  }
+  return `${toolName}(${JSON.stringify(toolArgs).slice(0, 80)})`;
+}
+
+export function summarizeToolResult(content, toolName, toolArgs) {
+  const lines = content.split("\n");
+  const chars = content.length;
+  const firstLine = lines[0]?.slice(0, 80) || "";
+
+  switch (toolName) {
+    case "read_file": {
+      const filePath = toolArgs?.path || "unknown";
+      return `[Read ${filePath} (${lines.length} lines, ${chars} chars): ${firstLine}...]`;
+    }
+    case "list_directory": {
+      const dirPath = toolArgs?.path || "unknown";
+      const itemCount = lines.length;
+      const preview = lines.slice(0, 3).join(", ");
+      const more = itemCount > 3 ? ` +${itemCount - 3} more` : "";
+      return `[Listed ${dirPath}: ${itemCount} items — ${preview}${more}]`;
+    }
+    case "run_command": {
+      const cmd = toolArgs?.command || "unknown";
+      const cmdShort = cmd.length > 40 ? cmd.slice(0, 40) + "..." : cmd;
+      const exitMatch = content.match(/exit(?:ed| code)[:\s]*(\d+)/i);
+      const exitCode = exitMatch ? exitMatch[1] : "?";
+      return `[Ran '${cmdShort}' (exit ${exitCode}, ${lines.length} lines): ${firstLine}...]`;
+    }
+    case "search_in_files": {
+      const pat = toolArgs?.pattern || "?";
+      const matchCount = lines.length - 1;
+      const files = new Set(lines.slice(1).map(l => l.split(":")[0]).filter(Boolean));
+      return `[Search /${pat}/: ${matchCount} matches in ${files.size} files — ${[...files].slice(0, 5).join(", ")}${files.size > 5 ? " +" + (files.size - 5) + " more" : ""}]`;
+    }
+    case "think": {
+      return `[Thought: ${firstLine}...]`;
+    }
+    default: {
+      return `[${toolName}(...) → ${lines.length} lines, ${chars} chars: ${firstLine}...]`;
+    }
+  }
+}
+
+function compressHeadTail(content) {
+  const originalLen = content.length;
+  const lines = content.split("\n");
+  if (lines.length > HEAD_LINES + TAIL_LINES + 2) {
+    const head = lines.slice(0, HEAD_LINES).join("\n");
+    const tail = lines.slice(-TAIL_LINES).join("\n");
+    const omitted = lines.length - HEAD_LINES - TAIL_LINES;
+    return `${head}\n[... ${omitted} lines omitted, ${originalLen} chars total ...]\n${tail}`;
+  }
+  return content.slice(0, COMPRESS_THRESHOLD) + `\n[... truncated, ${originalLen} chars total]`;
+}
+
+/**
+ * Estimate token count from messages (rough: 1 token ≈ 4 chars)
+ */
+function estimateTokens(messages) {
+  let total = 0;
+  for (const m of messages) {
+    const content = typeof m.content === "string" ? m.content : JSON.stringify(m.content || "");
+    total += Math.ceil(content.length / 4);
+    if (m.tool_calls) {
+      for (const tc of m.tool_calls) {
+        total += Math.ceil((tc.function?.arguments || "").length / 4) + 10;
+      }
+    }
+  }
+  return total;
+}
+
+/**
+ * When history may be rewritten to make room, in tokens.
+ *
+ * Half the model's window, capped at 128k because a bigger
+ * prompt is paid for on every call even when cached; 64k when the window is
+ * unknown. COMPRESS_AFTER_TOKENS overrides. On the 2026-09-26 auto-budget run
+ * the old fixed point (25k for the eager pass) collapsed sixteen file reads
+ * into one-liners at call 16 and again every few calls after; the agent never
+ * edited anything, it re-read. Room is not the problem at 25k on any current
+ * model, forgetting is.
+ */
+export function compressThreshold() {
+  if (config.compressAfterTokens) return config.compressAfterTokens;
+  return compressThresholdFor(contextWindow(), getSpendLevel());
+}
+
+/** The threshold for a window (null: unknown) at a spend level (spend.js). */
+export function compressThresholdFor(window, level) {
+  return windowShare(spendSettings(level).compress, window);
+}
+
+/**
+ * compressContext — threshold-based, sync fact extraction, action logging
+ *
+ * @param {Array} messages
+ * @param {number} prevIterationStart
+ * @param {number} iterationStart
+ * @param {string} [sessionId]
+ * @returns {Promise<number>} tokens saved
+ */
+export async function compressContext(messages, prevIterationStart, iterationStart, sessionId, { forceCompress = false } = {}) {
+  const estimatedTokens = estimateTokens(messages);
+  const threshold = compressThreshold();
+
+  // === Action logging (always, regardless of compression threshold) ===
+  // Track side-effect tools as session facts for context preservation
+  if (sessionId) {
+    for (let i = prevIterationStart; i < iterationStart; i++) {
+      const m = messages[i];
+      if (m.role === "assistant" && m.tool_calls) {
+        for (const tc of m.tool_calls) {
+          const name = tc.function?.name;
+          let args;
+          try { args = JSON.parse(tc.function?.arguments || "{}"); } catch { args = {}; }
+
+          if (isSideEffectTool(name, args)) {
+            // Find corresponding tool result
+            const resultMsg = messages.find((rm, ri) =>
+              ri > i && rm.role === "tool" && rm.tool_call_id === tc.id
+            );
+            const resultText = resultMsg?.content || "";
+            const actionDesc = describeAction(name, args, resultText);
+
+            saveSessionFact(sessionId, {
+              content: actionDesc,
+              category: "action",
+              source: "action-log",
+            });
+            log.debug("action-logged", { tool: name, action: actionDesc });
+          }
+        }
+      }
+    }
+  }
+
+  // === Check threshold — skip compression if under budget ===
+  if (!forceCompress && estimatedTokens < threshold) {
+    log.debug("compression-skipped", { estimatedTokens, threshold });
+    return 0;
+  }
+
+  log.info("compression-triggered", { estimatedTokens, threshold, messages: messages.length });
+
+  let saved = 0;
+  const messagesToExtract = [];
+
+  for (let i = 0; i < iterationStart; i++) {
+    const m = messages[i];
+
+    // Replace previous iteration images with existing text description
+    // No vision API call — use the OCR/caption text already in the adjacent tool result message.
+    // Only the LAST image in context stays as full base64 (current iteration).
+    if (m._isImage && Array.isArray(m.content) && m.content.some((c) => c.type === "image_url")) {
+      if (!m._compressed) {
+        const toolName = m._imageTool || "unknown";
+        const imagePath = m._imagePath || null;
+        // Use existing text from the image message itself (already has OCR/caption)
+        const textBlock = m.content.find((c) => c.type === "text");
+        const description = textBlock?.text || "[Previously viewed screenshot]";
+        const ref = imagePath ? ` [file: ${imagePath}]` : "";
+        m.content = `[Image from ${toolName}${ref}: ${description}]`;
+        m._compressed = true;
+        saved += 50000; // approximate base64 size saved
+        log.info("image-replaced", { tool: toolName, descLen: description.length });
+      }
+      continue;
+    }
+
+    // Collect uncompressed old messages for fact extraction
+    const isOld = i < prevIterationStart; // age >= 2
+    if (isOld && m.role === "tool" && m._compressed !== "summary") {
+      messagesToExtract.push(m);
+    }
+    if (isOld && (m.role === "user" || m.role === "assistant") && !m._compressed) {
+      messagesToExtract.push(m);
+    }
+
+    if (m.role !== "tool") continue;
+    if (m._compressed === "summary") continue;
+
+    if (!isOld && m.content.length <= COMPRESS_THRESHOLD) continue;
+
+    const originalLen = m.content.length;
+
+    if (isOld) {
+      m.content = summarizeToolResult(m.content, m._toolName, m._toolArgs);
+      m._compressed = "summary";
+    } else if (m._compressed !== "headtail") {
+      m.content = compressHeadTail(m.content);
+      m._compressed = "headtail";
+    }
+
+    saved += originalLen - m.content.length;
+  }
+
+  // Mark user/assistant old messages as compressed
+  for (let i = 0; i < prevIterationStart; i++) {
+    const m = messages[i];
+    if ((m.role === "user" || m.role === "assistant") && !m._compressed) {
+      m._compressed = true;
+    }
+  }
+
+  // Extract facts SYNCHRONOUSLY before they're lost
+  if (sessionId && messagesToExtract.length > 0) {
+    try {
+      const facts = await extractFacts(messagesToExtract);
+      if (facts.length > 0) {
+        saveSessionFacts(sessionId, facts);
+        log.info("facts-extracted", { count: facts.length, facts: facts.map(f => f.content.slice(0, 60)) });
+      }
+    } catch (err) {
+      log.warn("fact-extraction-failed", { error: err.message });
+    }
+  }
+
+  log.info("compression-done", { saved, estimatedAfter: estimateTokens(messages) });
+  return saved;
+}

package/src/agent/content-resolver.js

@@ -0,0 +1,180 @@
+/**
+ * Content Type Resolver — determines the actual type of MCP response data.
+ *
+ * Analyzes raw MCP content blocks by inspecting mimeType, data content,
+ * and magic bytes — NOT by trusting block.type strings.
+ *
+ * Returns a normalized typed result that downstream modules (perception,
+ * agent loop) can work with without knowing MCP internals.
+ *
+ * Architecture:
+ *   MCP raw → resolveContent() → typed result → perception → history
+ */
+
+import { createLogger } from "../logging/logger.js";
+
+const log = createLogger("content-resolver");
+
+// Magic byte signatures for common formats (first bytes of base64-decoded data)
+const MAGIC_BYTES = {
+  // Images
+  "iVBOR":   { type: "image", mimeType: "image/png" },      // PNG (\x89PNG)
+  "/9j/":    { type: "image", mimeType: "image/jpeg" },      // JPEG (\xFF\xD8)
+  "R0lGOD":  { type: "image", mimeType: "image/gif" },       // GIF (GIF8)
+  "UklGR":   { type: "image", mimeType: "image/webp" },      // WebP (RIFF)
+  "AAABAA":  { type: "image", mimeType: "image/x-icon" },    // ICO
+
+  // Audio
+  "SUQz":    { type: "audio", mimeType: "audio/mpeg" },      // MP3 (ID3)
+  "T2dnUw":  { type: "audio", mimeType: "audio/ogg" },       // OGG (OggS)
+  "ZkxhQw":  { type: "audio", mimeType: "audio/flac" },      // FLAC (fLaC)
+  "UklGR":   null, // RIFF — could be WAV or WebP, check further
+
+  // Video
+  "AAAA":    null, // Could be MP4/MOV (ftyp box) — needs more bytes
+  "GkXFo":   { type: "video", mimeType: "video/webm" },      // WebM (\x1A\x45)
+
+  // Documents
+  "JVBER":   { type: "document", mimeType: "application/pdf" }, // PDF (%PDF)
+  "UEsDB":   { type: "document", mimeType: "application/zip" }, // ZIP/XLSX/DOCX (PK)
+};
+
+/**
+ * Detect media type from base64-encoded data by inspecting magic bytes.
+ */
+function detectFromBase64(data) {
+  if (!data || data.length < 4) return null;
+
+  const prefix = data.slice(0, 6);
+
+  for (const [magic, info] of Object.entries(MAGIC_BYTES)) {
+    if (prefix.startsWith(magic) && info) {
+      return info;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Detect media type from mimeType string.
+ */
+function detectFromMimeType(mimeType) {
+  if (!mimeType) return null;
+
+  const major = mimeType.split("/")[0];
+  switch (major) {
+    case "image": return { type: "image", mimeType };
+    case "audio": return { type: "audio", mimeType };
+    case "video": return { type: "video", mimeType };
+    case "application":
+      if (mimeType.includes("pdf")) return { type: "document", mimeType };
+      if (mimeType.includes("json")) return { type: "text", mimeType };
+      if (mimeType.includes("xml")) return { type: "text", mimeType };
+      return { type: "binary", mimeType };
+    case "text": return { type: "text", mimeType };
+    default: return { type: "binary", mimeType };
+  }
+}
+
+/**
+ * Resolve a single MCP content block into a typed part.
+ */
+function resolveBlock(block) {
+  // Text block — always text
+  if (block.text != null) {
+    return {
+      type: "text",
+      mimeType: block.mimeType || "text/plain",
+      text: block.text,
+    };
+  }
+
+  // Data block — determine type by mimeType first, then magic bytes
+  if (block.data != null) {
+    const byMime = detectFromMimeType(block.mimeType);
+    const byMagic = detectFromBase64(block.data);
+
+    const resolved = byMime || byMagic || { type: "binary", mimeType: "application/octet-stream" };
+
+    return {
+      ...resolved,
+      data: block.data,
+      dataLength: block.data.length,
+    };
+  }
+
+  // Resource block
+  if (block.resource) {
+    return {
+      type: "text",
+      mimeType: "text/plain",
+      text: block.resource.text || JSON.stringify(block.resource),
+    };
+  }
+
+  // Unknown block type
+  log.warn("unknown-block", { blockType: block.type, keys: Object.keys(block) });
+  return {
+    type: "text",
+    mimeType: "text/plain",
+    text: JSON.stringify(block),
+  };
+}
+
+/**
+ * Resolve MCP content blocks into a typed result.
+ *
+ * @param {Array} blocks - MCP content blocks from tool result
+ * @returns {{ type: string, mimeType?: string, data?: string, text?: string, parts?: Array }}
+ */
+export function resolveContent(blocks) {
+  if (!blocks || blocks.length === 0) {
+    return { type: "text", text: "OK" };
+  }
+
+  const resolved = blocks.map(resolveBlock);
+
+  // Single block — return directly
+  if (resolved.length === 1) {
+    log.debug("resolved-single", { type: resolved[0].type, mimeType: resolved[0].mimeType });
+    return resolved[0];
+  }
+
+  // Multiple blocks — check if mixed types
+  const types = new Set(resolved.map(r => r.type));
+
+  // All same type — merge
+  if (types.size === 1) {
+    const type = resolved[0].type;
+
+    if (type === "text") {
+      // Merge all text blocks
+      return {
+        type: "text",
+        mimeType: "text/plain",
+        text: resolved.map(r => r.text).filter(Boolean).join("\n"),
+      };
+    }
+
+    // Multiple images/audio of same type — return as mixed
+    log.debug("resolved-multi-same", { type, count: resolved.length });
+    return { type: "mixed", parts: resolved };
+  }
+
+  // Mixed types — common case: image + text (screenshot + OCR)
+  const mediaParts = resolved.filter(r => ["image", "audio", "video"].includes(r.type));
+  const textParts = resolved.filter(r => r.type === "text");
+
+  // Single media + text = media with text metadata
+  if (mediaParts.length === 1 && textParts.length >= 1) {
+    const media = mediaParts[0];
+    media.text = textParts.map(r => r.text).filter(Boolean).join("\n");
+    log.debug("resolved-media-with-text", { type: media.type, mimeType: media.mimeType, textLength: media.text.length });
+    return media;
+  }
+
+  // Everything else — mixed
+  log.debug("resolved-mixed", { types: [...types], count: resolved.length });
+  return { type: "mixed", parts: resolved };
+}