@tyvm/knowhow 0.0.159 → 0.0.160

package/src/processors/TokenCompressor.ts

@@ -1,620 +0,0 @@
-import { Message, Tool } from "../clients/types";
-import { MessageProcessorFunction } from "../services/MessageProcessor";
-import { ToolsService } from "../services";
-import {
-  JsonCompressor,
-  JsonSchema,
-  CompressionMetadata,
-  JsonCompressorStorage,
-} from "./JsonCompressor";
-import { paginateToolResponse } from "./tools/boundedToolResponse";
-
-export interface KeyInfo {
-  key: string;
-  size: number;
-  preview: string;
-  tokens?: number;
-  type?: string;
-  depth?: number;
-  childKeys?: string[];
-  nextChunkKey?: string;
-}
-
-interface TokenCompressorStorage {
-  [key: string]: string;
-}
-
-export class TokenCompressor implements JsonCompressorStorage {
-  private storage: TokenCompressorStorage = {};
-  private keyPrefix: string = "compressed_";
-  private toolName: string = expandTokensDefinition.function.name;
-
-  // Threshold for compression - if content exceeds this size, we compress it
-  private compressionThreshold: number = 4000;
-  private characterLimit: number = this.compressionThreshold * 4;
-
-  // Minimum size (in characters) a standalone chunk should have. Leftover content
-  // smaller than this is merged into an adjacent chunk so we never emit tiny,
-  // low-value chunks that force the agent into extra round-trips to reassemble a
-  // file. Defaults to half of the per-chunk character limit.
-  private minChunkCharacters: number = Math.floor(this.characterLimit / 2);
-
-  // Largest size retrievable without re-compressing
-  public maxTokens: number = this.compressionThreshold * 2;
-
-  // JSON compression handler
-  private jsonCompressor: JsonCompressor;
-
-
-  /**
-   * Tool names whose responses should never be compressed.
-   * These are compression/retrieval tools themselves - compressing their output
-   * would prevent the agent from seeing the uncompressed data it just retrieved.
-   */
-  static DO_NOT_COMPRESS: ReadonlySet<string> = new Set([
-    "expandTokens",
-    "grepToolResponse",
-    "jqToolResponse",
-    "tailToolResponse",
-    "listStoredToolResponses",
-    "loadImageAsBase64" // need a proper way to do this
-  ]);
-
-
-  constructor(toolsService?: ToolsService) {
-    this.jsonCompressor = new JsonCompressor(
-      this,
-      this.compressionThreshold,
-      this.maxTokens,
-      this.toolName
-    );
-    this.registerTool(toolsService);
-  }
-
-  // Rough token estimation (4 chars per token average)
-  public estimateTokens(text: string): number {
-    return Math.ceil(text.length / 4);
-  }
-
-  public setCompressionThreshold(threshold: number): void {
-    this.compressionThreshold = threshold;
-    this.characterLimit = threshold * 4; // Update character limit based on new threshold
-    this.minChunkCharacters = Math.floor(this.characterLimit / 2);
-    this.jsonCompressor.updateSettings(threshold, this.maxTokens);
-  }
-
-  // Internally adjust to ensure we can always retrieve data
-  private setMaxTokens(maxTokens: number): void {
-    if (maxTokens > this.maxTokens) {
-      this.maxTokens = maxTokens;
-      this.jsonCompressor.updateSettings(this.compressionThreshold, maxTokens);
-    }
-  }
-
-  /**
-   * Attempts to parse content as JSON and returns parsed object if successful.
-   * Also handles MCP tool response format where actual data is in content[0].text
-   */
-
-  public tryParseJson(content: string): any | null {
-    return this.jsonCompressor.tryParseJson(content);
-  }
-
-  /**
-   * Compresses a string into chunks from the end, creating a chain of references
-   */
-  public compressStringInChunks(content: string, path: string = ""): string {
-    if (content.length <= this.characterLimit) {
-      return content;
-    }
-
-    const chunks: string[] = [];
-    const chunkKeys: string[] = [];
-    let remaining = content;
-
-    // Split from the end, creating chunks that will be linked.
-    while (remaining.length > this.characterLimit) {
-      const chunkStart = remaining.length - this.characterLimit;
-      const chunk = remaining.substring(chunkStart);
-      chunks.unshift(chunk); // Add to beginning since we're working backwards
-      remaining = remaining.substring(0, chunkStart);
-    }
-
-    // The remaining part becomes the first chunk. To avoid producing a tiny,
-    // low-value leading chunk (which forces the agent into extra round-trips to
-    // reassemble a file), merge it into the next chunk when it falls below the
-    // minimum chunk size. Only split it out as its own chunk when it is large
-    // enough to stand on its own.
-    if (remaining.length > 0) {
-      if (chunks.length > 0 && remaining.length < this.minChunkCharacters) {
-        // Prepend the small leftover onto the first existing chunk so every
-        // emitted chunk carries a meaningful amount of content.
-        chunks[0] = remaining + chunks[0];
-      } else {
-        chunks.unshift(remaining);
-      }
-    }
-
-    // Store chunks and create chain of references
-    // First, generate all keys
-    for (let i = 0; i < chunks.length; i++) {
-      const key = this.generateKey();
-      chunkKeys.push(key);
-    }
-
-    // Then store chunks with proper linking
-    for (let i = 0; i < chunks.length; i++) {
-      let chunkContent = chunks[i];
-
-      // Add reference to next chunk if it exists
-      if (i < chunks.length - 1) {
-        const nextKey = chunkKeys[i + 1];
-        chunkContent += `\n\n[NEXT_CHUNK_KEY: ${nextKey}]`;
-      }
-
-      this.storeString(chunkKeys[i], chunkContent);
-    }
-
-    // Return reference to the first chunk
-    const firstKey = chunkKeys[0];
-    const totalTokens = this.estimateTokens(content);
-    const chunkCount = chunks.length;
-
-    return `[COMPRESSED_STRING - ${totalTokens} tokens in ${chunkCount} chunks]\nKey: ${firstKey}\nPath: ${path}\nPreview: ${content.substring(
-      0,
-      200
-    )}...\n[Use ${
-      this.toolName
-    } tool with key "${firstKey}" to retrieve the FULL content in one call (chunks are auto-stitched; pass fromLine/toLine for a ranged read)]\n[TIP: try jqToolResponse,grepToolResponse,tailToolResponse to filter/search/map with REAL line numbers without expanding at all]`;
-  }
-
-  /**
-   * Check if content is already compressed
-   */
-  private isAlreadyCompressed(content: string): boolean {
-    // Check for a compressed-string marker, but ONLY when the content *starts*
-    // with it. Compressor output always begins with "[COMPRESSED_STRING ...".
-    // A naive `.includes()` produced false positives for large payloads (e.g. a
-    // usage.json / message-history dump) that merely CONTAIN that substring deep
-    // inside their nested data — those files were left completely uncompressed
-    // because this guard incorrectly reported them as already compressed.
-    if (content.trimStart().startsWith("[COMPRESSED_STRING")) {
-      return true;
-    }
-
-    // Check for compressed JSON structure with schema key
-    // Only treat this as compressed when `_schema_key` is an actual TOP-LEVEL
-    // key of the parsed object — not merely present somewhere in the string.
-    const parsed = this.tryParseJson(content);
-    if (
-      parsed &&
-      typeof parsed === "object" &&
-      !Array.isArray(parsed) &&
-      typeof parsed._schema_key === "string"
-    ) {
-      return true;
-    }
-
-    return false;
-  }
-
-  /**
-   * Enhanced content compression that handles both JSON and string chunking
-   */
-  public compressContent(content: string, path: string = ""): string {
-    // Check if already compressed - don't compress again
-    if (this.isAlreadyCompressed(content)) {
-      return content;
-    }
-
-    const tokens = this.estimateTokens(content);
-
-    // For nested properties (path !== ""), use maxTokens to avoid recompressing stored data
-    // For top-level content (path === ""), use compressionThreshold to determine compression
-    const thresholdToUse =
-      path === "" ? this.compressionThreshold : this.maxTokens;
-
-    if (tokens <= thresholdToUse) {
-      return content;
-    }
-
-    // Try to parse as JSON and generate schema
-    const jsonObj = this.tryParseJson(content);
-    if (jsonObj) {
-      // For MCP format, work with the actual data
-      const dataToCompress = jsonObj._mcp_format ? jsonObj._data : jsonObj;
-
-      // Generate and store schema
-      const schema = this.jsonCompressor.generateSchema(jsonObj);
-      const schemaKey = this.generateKey();
-      this.storeString(schemaKey, JSON.stringify(schema));
-
-      // For JSON objects, compress individual properties
-      // Use a non-empty path to ensure compression logic is applied
-      const compressedObj = this.compressJsonProperties(
-        dataToCompress,
-        path || "data"
-      );
-
-      // If this was MCP format, wrap the result back
-      const finalCompressedObj = jsonObj._mcp_format
-        ? {
-            _mcp_format: true,
-            _raw_structure: jsonObj._raw_structure,
-            _data: compressedObj,
-          }
-        : compressedObj;
-
-      // Add schema reference to the compressed result
-      const resultWithSchema =
-        typeof finalCompressedObj === "object" &&
-        !Array.isArray(finalCompressedObj)
-          ? { ...finalCompressedObj, _schema_key: schemaKey }
-          : { _schema_key: schemaKey, data: finalCompressedObj };
-      const compressedContent = JSON.stringify(resultWithSchema, null, 2);
-
-      // Check compression effectiveness
-      const compressedTokens = this.estimateTokens(compressedContent);
-
-      // For MCP format, we've successfully extracted and compressed the data
-      // The wrapper overhead is acceptable because we provide schema + structured access
-      // For non-MCP format, use the standard 60% threshold
-      const compressionThreshold = 0.6;
-
-      if (compressedTokens < tokens * compressionThreshold) {
-        return compressedContent;
-      }
-    }
-
-    // For strings or when JSON compression wasn't effective, use chunking
-    return this.compressStringInChunks(content, path);
-  }
-
-  /**
-   * Compresses large properties within a JSON object using depth-first traversal.
-   * Implements an efficient backward-iterating chunking strategy for large arrays.
-   */
-  public compressJsonProperties(obj: any, path: string = ""): any {
-    return this.jsonCompressor.compressJsonProperties(obj, path);
-  }
-
-  public generateKey(): string {
-    return `${this.keyPrefix}${Date.now()}_${Math.random()
-      .toString(36)
-      .substr(2, 9)}`;
-  }
-
-  public async compressMessage(message: Message) {
-    // Compress content if it's a string
-    if (typeof message.content === "string") {
-      const compressed = this.compressContent(message.content);
-      // If this is a tool message with a tool_call_id, append it to the compressed content
-      // so the agent knows which toolCallId to use for grepToolResponse/jqToolResponse/tailToolResponse
-      if (message.role === "tool" && message.tool_call_id && compressed !== message.content) {
-        message.content = compressed + `\n[toolCallId: ${message.tool_call_id}]`;
-      } else {
-        message.content = compressed;
-      }
-    }
-    // Handle array content (multimodal)
-    else if (Array.isArray(message.content)) {
-      for (const item of message.content) {
-        if (item.type === "text" && item.text) {
-          item.text = this.compressContent(item.text);
-        }
-      }
-    }
-  }
-
-  createProcessor(
-    filterFn?: (msg: Message) => boolean
-  ): MessageProcessorFunction {
-    return async (originalMessages: Message[], modifiedMessages: Message[]) => {
-      // Find the index of the last assistant message so we only compress
-      // the most recent batch of tool responses (after the last agent output).
-      // This prevents re-compressing already-cached historical messages, which
-      // would break prompt caching. Newly-added tool responses (the "current
-      // batch") are the ones that need compressing before they reach the model,
-      // and they appear after the last assistant message at the moment this
-      // processor runs (post_tools / pre_call), when the fresh tool response is
-      // still the tail of the message list.
-      let lastAssistantIndex = -1;
-      for (let i = modifiedMessages.length - 1; i >= 0; i--) {
-        if (modifiedMessages[i].role === "assistant") {
-          lastAssistantIndex = i;
-          break;
-        }
-      }
-
-      for (let i = 0; i < modifiedMessages.length; i++) {
-        const message = modifiedMessages[i];
-
-        if (filterFn && !filterFn(message)) {
-          continue;
-        }
-
-        // Only compress tool messages that are part of the last batch
-        // (i.e., they appear after the last assistant message).
-        // Messages before the last assistant message are historical and must
-        // not be modified to preserve prompt caching.
-        if (message.role === "tool" && i <= lastAssistantIndex) {
-          continue;
-        }
-
-        // Skip responses from compression-related tools — the agent called
-        // these specifically to see uncompressed data, so we must not
-        // re-compress their output.
-        if (
-          message.role === "tool" &&
-          message.name &&
-          TokenCompressor.DO_NOT_COMPRESS.has(message.name)
-        ) {
-          continue;
-        }
-
-        await this.compressMessage(message);
-      }
-    };
-  }
-
-  /**
-   * Retrieves a single chunk of stored data.
-   * If the data was chunked, the returned string will contain a `NEXT_CHUNK_KEY`
-   * that the agent can use to retrieve the subsequent part of the content.
-   */
-  retrieveString(key: string): string | null {
-    return this.storage[key] || null;
-  }
-
-  /**
-   * Retrieves the fully reassembled content for a key, following any
-   * `NEXT_CHUNK_KEY` references and stripping the chunk-linking markers. This
-   * lets callers get the complete content in a single call instead of chasing a
-   * chain of nested keys.
-   */
-  retrieveFullString(key: string): string | null {
-    let currentKey: string | null = key;
-    let result = "";
-    let found = false;
-    const visited = new Set<string>();
-
-    while (currentKey && !visited.has(currentKey)) {
-      visited.add(currentKey);
-      const chunk = this.storage[currentKey];
-      if (chunk === undefined) {
-        break;
-      }
-      found = true;
-
-      const nextMatch = chunk.match(/\n\n\[NEXT_CHUNK_KEY:\s*([^\s\]]+)\]/);
-      const nextKey = nextMatch ? nextMatch[1] : null;
-
-      // Strip the trailing NEXT_CHUNK_KEY marker before stitching.
-      const cleaned = chunk.replace(/\n\n\[NEXT_CHUNK_KEY:\s*[^\s\]]+\]/, "");
-      result += cleaned;
-      currentKey = nextKey;
-    }
-
-    return found ? result : null;
-  }
-
-  storeString(key: string, value: string): void {
-    if (this.estimateTokens(value) > this.maxTokens) {
-      // adjust max tokens so we can always retrieve this without re-compressing
-      this.setMaxTokens(this.estimateTokens(value) + 1);
-    }
-    this.storage[key] = value;
-  }
-
-  clearStorage(): void {
-    this.storage = {};
-    this.jsonCompressor.clearDeduplication();
-  }
-
-  getStorageKeys(): string[] {
-    return Object.keys(this.storage);
-  }
-
-  getStorageSize(): number {
-    return Object.keys(this.storage).length;
-  }
-
-  registerTool(toolsService?: ToolsService): void {
-    if (toolsService) {
-      toolsService.addTools([expandTokensDefinition]);
-      toolsService.addFunctions({
-        [this.toolName]: (
-          key: string,
-          fromLine?: number,
-          toLine?: number,
-          characterOffset?: number,
-          maxCharacters?: number
-        ) => {
-          // Auto-stitch: follow any NEXT_CHUNK_KEY chain and return the full,
-          // reassembled content so the agent never has to chase nested keys.
-          const data = this.retrieveFullString(key);
-
-          if (data === null) {
-            return paginateToolResponse(
-              `Error: No data found for key "${key}". Available keys: ${this.getStorageKeys().join(
-                ", "
-              )}`,
-              { characterOffset, maxCharacters },
-              "available-key list"
-            );
-          }
-
-          // Optional ranged read: return only the requested 1-based, inclusive
-          // line range, prefixed with real line numbers for easy mapping back.
-          const hasRange =
-            typeof fromLine === "number" || typeof toLine === "number";
-          if (!hasRange) {
-            return paginateToolResponse(
-              data,
-              { characterOffset, maxCharacters },
-              `expanded value for ${key}`
-            );
-          }
-
-          const lines = data.split("\n");
-          const totalLines = lines.length;
-          const start = Math.max(1, typeof fromLine === "number" ? fromLine : 1);
-          const end = Math.min(
-            totalLines,
-            typeof toLine === "number" ? toLine : totalLines
-          );
-
-          if (start > end) {
-            return paginateToolResponse(
-              `Error: Invalid line range for key "${key}": fromLine (${start}) is greater than toLine (${end}). Content has ${totalLines} lines.`,
-              { characterOffset, maxCharacters },
-              "line-range error"
-            );
-          }
-
-          const numbered: string[] = [];
-          for (let i = start; i <= end; i++) {
-            numbered.push(`${i}: ${lines[i - 1]}`);
-          }
-          return paginateToolResponse(
-            numbered.join("\n"),
-            { characterOffset, maxCharacters },
-            `expanded lines ${start}-${end} for ${key}`
-          );
-        },
-      });
-    }
-  }
-
-  /**
-   * Get the schema for a compressed object
-   */
-  getSchema(key: string): JsonSchema | null {
-    const schemaKey = `${key}_schema`;
-    const schemaStr = this.storage[schemaKey];
-    if (!schemaStr) {
-      return null;
-    }
-    try {
-      return JSON.parse(schemaStr);
-    } catch (e) {
-      return null;
-    }
-  }
-
-  /**
-   * Get compressed properties for an object
-   */
-  getCompressedProperties(key: string): any | null {
-    const content = this.storage[key];
-    if (!content) {
-      return null;
-    }
-    try {
-      const metadata = JSON.parse(content) as CompressionMetadata;
-      return metadata.compressed_properties || null;
-    } catch (e) {
-      return null;
-    }
-  }
-
-  /**
-   * Get full object by merging high-signal and compressed properties
-   */
-  getFullObject(mainObj: any, compressedKey: string): any {
-    if (!mainObj || typeof mainObj !== "object") {
-      return mainObj;
-    }
-
-    const compressed = this.getCompressedProperties(compressedKey);
-    if (!compressed) {
-      return mainObj;
-    }
-
-    const {
-      _compressed_properties_key,
-      _compressed_property_names,
-      _compression_info,
-      ...highSignal
-    } = mainObj;
-    return { ...highSignal, ...compressed };
-  }
-
-  /**
-   * Extract all keys from compressed content
-   */
-  extractKeys(content: string): string[] {
-    const keys: string[] = [];
-    const keyPattern = /\$expandTokens\[([^\]]+)\]|Key:\s*([^\s\n]+)/g;
-    let match;
-    while ((match = keyPattern.exec(content)) !== null) {
-      const key = match[1] || match[2];
-      if (key && !keys.includes(key)) {
-        keys.push(key);
-      }
-    }
-    return keys;
-  }
-
-  /**
-   * Get the chain of keys for a given key (following NEXT_CHUNK_KEY references)
-   */
-  getKeyChain(key: string): KeyInfo[] {
-    const chain: KeyInfo[] = [];
-    let currentKey: string | null = key;
-
-    while (currentKey) {
-      const content = this.storage[currentKey];
-      if (!content) break;
-
-      chain.push({
-        key: currentKey,
-        size: content.length,
-        preview: content.substring(0, 100),
-      });
-
-      const nextMatch = content.match(/NEXT_CHUNK_KEY:\s*([^\s\n]+)/);
-      currentKey = nextMatch ? nextMatch[1] : null;
-    }
-    return chain;
-  }
-}
-
-export const expandTokensDefinition: Tool = {
-  type: "function",
-  function: {
-    name: "expandTokens",
-    description:
-      "Retrieve a bounded page of compressed data. Chunk chains are automatically reassembled, but output is limited to 20,000 characters by default. Use characterOffset to continue, or fromLine/toLine for line-based selection. Prefer jqToolResponse, grepToolResponse, or tailToolResponse when possible.",
-    parameters: {
-      type: "object",
-      positional: true,
-      properties: {
-        key: {
-          type: "string",
-          description: "The key of the compressed data to retrieve",
-        },
-        fromLine: {
-          type: "number",
-          description:
-            "Optional 1-based start line (inclusive) for a ranged read of the reassembled content.",
-        },
-        toLine: {
-          type: "number",
-          description:
-            "Optional 1-based end line (inclusive). Defaults to the end of the content when omitted.",
-        },
-        characterOffset: {
-          type: "number",
-          description:
-            "Optional zero-based character offset within the expanded value or selected line range.",
-        },
-        maxCharacters: {
-          type: "number",
-          description: "Response size (default 20000, hard maximum 50000).",
-        },
-      },
-      required: ["key"],
-    },
-  },
-};