@tyvm/knowhow 0.0.62 → 0.0.64

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.62",
+  "version": "0.0.64",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {

package/src/chat/modules/AgentModule.ts

@@ -21,10 +21,12 @@ import {
   CustomVariables,
   XmlToolCallProcessor,
   HarmonyToolProcessor,
+  Base64ImageProcessor,
 } from "../../processors/index";
 import { TaskInfo, ChatSession } from "../types";
 import { agents } from "../../agents";
 import { ToolCallEvent } from "../../agents/base/base";
+import { $Command } from "@aws-sdk/client-s3";
 
 export class AgentModule extends BaseChatModule {
   name = "agent";
@@ -582,11 +584,16 @@ Please continue from where you left off and complete the original request.
         10 // Priority level
       );
 
-      agent.messageProcessor.setProcessors("pre_call", [
+      const caching = [
         new ToolResponseCache(agent.tools).createProcessor(),
         new TokenCompressor(agent.tools).createProcessor((msg) =>
           Boolean(msg.role === "tool" && msg.tool_call_id)
         ),
+      ];
+
+      agent.messageProcessor.setProcessors("pre_call", [
+        new Base64ImageProcessor(agent.tools).createProcessor(),
+        ...caching,
         new CustomVariables(agent.tools).createProcessor(),
       ]);
 
@@ -596,9 +603,8 @@ Please continue from where you left off and complete the original request.
       ]);
 
       agent.messageProcessor.setProcessors("post_tools", [
-        new TokenCompressor(agent.tools).createProcessor((msg) =>
-          Boolean(msg.role === "tool" && msg.tool_call_id)
-        ),
+        new Base64ImageProcessor(agent.tools).createProcessor(),
+        ...caching,
       ]);
 
       // Set up event listeners

package/src/clients/anthropic.ts

@@ -184,12 +184,44 @@ export class GenericAnthropicClient implements GenericClient {
             });
           }
 
+          // Convert tool message content to appropriate format
+          let toolResultContent: string | (Anthropic.TextBlockParam | Anthropic.ImageBlockParam)[];
+          
+          if (typeof msg.content === "string") {
+            toolResultContent = msg.content;
+          } else if (Array.isArray(msg.content)) {
+            // Transform image_url format to Anthropic's image format
+            toolResultContent = msg.content.map((item): Anthropic.TextBlockParam | Anthropic.ImageBlockParam => {
+              if (item.type === "image_url") {
+                const url = item.image_url.url;
+                const isDataUrl = url.startsWith("data:");
+                const base64Data = isDataUrl ? url.split(",")[1] : url;
+                const mediaType = isDataUrl ? url.match(/data:([^;]+);/)?.[1] || "image/jpeg" : "image/jpeg";
+                
+                return {
+                  type: "image" as const,
+                  source: {
+                    type: "base64" as const,
+                    media_type: mediaType as any,
+                    data: base64Data,
+                  },
+                };
+              } else if (item.type === "text") {
+                return { type: "text" as const, text: item.text };
+              }
+              // Fallback for unknown types
+              return { type: "text" as const, text: String(item) };
+            }) as (Anthropic.TextBlockParam | Anthropic.ImageBlockParam)[];
+          } else {
+            toolResultContent = String(msg.content);
+          }
+
           toolMessages.push({
             role: "user",
             content: [
               {
                 type: "tool_result",
-                content: msg.content as string,
+                content: toolResultContent,
                 tool_use_id: msg.tool_call_id,
               },
             ],

package/src/processors/Base64ImageDetector.ts

@@ -1,5 +1,8 @@
+import * as fs from "fs";
+import * as path from "path";
 import { Message } from "../clients/types";
 import { MessageProcessorFunction } from "../services/MessageProcessor";
+import { ToolsService } from "../services";
 
 interface ImageContent {
   type: "image_url";
@@ -14,23 +17,23 @@ interface TextContent {
   text: string;
 }
 
-export class Base64ImageDetector {
-  private imageDetail: "auto" | "low" | "high";
-  private supportedFormats: string[];
+export class Base64ImageProcessor {
+  private imageDetail: "auto" | "low" | "high" = "auto";
+  private supportedFormats = ["png", "jpeg", "jpg", "gif", "webp"];
 
-  constructor(
-    imageDetail: "auto" | "low" | "high" = "auto",
-    supportedFormats: string[] = ["png", "jpeg", "jpg", "gif", "webp"]
-  ) {
-    this.imageDetail = imageDetail;
-    this.supportedFormats = supportedFormats;
+  constructor(toolsService?: ToolsService) {
+    this.registerTool(toolsService);
   }
 
-  private isBase64Image(text: string): { isImage: boolean; mimeType?: string; data?: string } {
+  private isBase64Image(text: string): {
+    isImage: boolean;
+    mimeType?: string;
+    data?: string;
+  } {
     // Check for data URL format: data:image/type;base64,actualdata
     const dataUrlPattern = /^data:image\/([a-zA-Z]+);base64,(.+)$/;
     const match = text.match(dataUrlPattern);
-    
+
     if (match) {
       const [, mimeType, data] = match;
       if (this.supportedFormats.includes(mimeType.toLowerCase())) {
@@ -47,14 +50,17 @@ export class Base64ImageDetector {
       try {
         const decoded = atob(header);
         // Check for common image file signatures
-        if (decoded.startsWith('\x89PNG')) {
-          return { isImage: true, mimeType: 'png', data: text };
-        } else if (decoded.startsWith('\xFF\xD8\xFF')) {
-          return { isImage: true, mimeType: 'jpeg', data: text };
-        } else if (decoded.startsWith('GIF87a') || decoded.startsWith('GIF89a')) {
-          return { isImage: true, mimeType: 'gif', data: text };
-        } else if (decoded.startsWith('RIFF') && decoded.includes('WEBP')) {
-          return { isImage: true, mimeType: 'webp', data: text };
+        if (decoded.startsWith("\x89PNG")) {
+          return { isImage: true, mimeType: "png", data: text };
+        } else if (decoded.startsWith("\xFF\xD8\xFF")) {
+          return { isImage: true, mimeType: "jpeg", data: text };
+        } else if (
+          decoded.startsWith("GIF87a") ||
+          decoded.startsWith("GIF89a")
+        ) {
+          return { isImage: true, mimeType: "gif", data: text };
+        } else if (decoded.startsWith("RIFF") && decoded.includes("WEBP")) {
+          return { isImage: true, mimeType: "webp", data: text };
         }
       } catch (e) {
         // Not valid base64 or not an image
@@ -66,26 +72,26 @@ export class Base64ImageDetector {
 
   private convertBase64ToImageContent(text: string): ImageContent | null {
     const detection = this.isBase64Image(text);
-    
+
     if (!detection.isImage) {
       return null;
     }
 
-    const dataUrl = detection.data!.startsWith('data:') 
-      ? detection.data 
+    const dataUrl = detection.data!.startsWith("data:")
+      ? detection.data
       : `data:image/${detection.mimeType};base64,${detection.data}`;
 
     return {
       type: "image_url",
       image_url: {
         url: dataUrl,
-        detail: this.imageDetail
-      }
+        detail: this.imageDetail,
+      },
     };
   }
 
   private processMessageContent(message: Message): void {
-    if (typeof message.content === 'string') {
+    if (typeof message.content === "string") {
       const imageContent = this.convertBase64ToImageContent(message.content);
       if (imageContent) {
         // Convert string content to multimodal array
@@ -94,9 +100,9 @@ export class Base64ImageDetector {
     } else if (Array.isArray(message.content)) {
       // Process each content item
       const newContent: (TextContent | ImageContent)[] = [];
-      
+
       for (const item of message.content) {
-        if (item.type === 'text' && item.text) {
+        if (item.type === "text" && item.text) {
           const imageContent = this.convertBase64ToImageContent(item.text);
           if (imageContent) {
             newContent.push(imageContent);
@@ -107,7 +113,7 @@ export class Base64ImageDetector {
           newContent.push(item as TextContent | ImageContent);
         }
       }
-      
+
       message.content = newContent;
     }
   }
@@ -119,22 +125,22 @@ export class Base64ImageDetector {
           try {
             const args = JSON.parse(toolCall.function.arguments);
             let modified = false;
-            
+
             // Recursively check all string values in arguments
             const processValue = (obj: any): any => {
-              if (typeof obj === 'string') {
+              if (typeof obj === "string") {
                 const detection = this.isBase64Image(obj);
                 if (detection.isImage) {
                   modified = true;
-                  const dataUrl = detection.data!.startsWith('data:') 
-                    ? detection.data 
+                  const dataUrl = detection.data!.startsWith("data:")
+                    ? detection.data
                     : `data:image/${detection.mimeType};base64,${detection.data}`;
                   return `[CONVERTED TO IMAGE: ${dataUrl.substring(0, 50)}...]`;
                 }
                 return obj;
               } else if (Array.isArray(obj)) {
                 return obj.map(processValue);
-              } else if (obj && typeof obj === 'object') {
+              } else if (obj && typeof obj === "object") {
                 const result = {};
                 for (const [key, value] of Object.entries(obj)) {
                   result[key] = processValue(value);
@@ -143,7 +149,7 @@ export class Base64ImageDetector {
               }
               return obj;
             };
-            
+
             const processedArgs = processValue(args);
             if (modified) {
               toolCall.function.arguments = JSON.stringify(processedArgs);
@@ -152,10 +158,13 @@ export class Base64ImageDetector {
             // Arguments are not valid JSON, treat as string
             const detection = this.isBase64Image(toolCall.function.arguments);
             if (detection.isImage) {
-              const dataUrl = detection.data!.startsWith('data:') 
-                ? detection.data 
+              const dataUrl = detection.data!.startsWith("data:")
+                ? detection.data
                 : `data:image/${detection.mimeType};base64,${detection.data}`;
-              toolCall.function.arguments = `[CONVERTED TO IMAGE: ${dataUrl.substring(0, 50)}...]`;
+              toolCall.function.arguments = `[CONVERTED TO IMAGE: ${dataUrl.substring(
+                0,
+                50
+              )}...]`;
             }
           }
         }
@@ -163,14 +172,45 @@ export class Base64ImageDetector {
     }
   }
 
+  private processToolMessageContent(message: Message): void {
+    // Tool messages have string content that might be a JSON string containing image data
+    if (typeof message.content === "string" && message.content.trim()) {
+      try {
+        // Try to parse as JSON
+        const parsed = JSON.parse(message.content);
+        
+        // Check if it's an image_url object
+        if (parsed.type === "image_url" && parsed.image_url?.url) {
+          // Convert the tool message content from JSON string to an array with the image
+          message.content = [parsed];
+        }
+      } catch (e) {
+        // Not JSON, check if it's a plain base64 string (only if still a string)
+        if (typeof message.content === "string") {
+          const imageContent = this.convertBase64ToImageContent(message.content);
+          if (imageContent) {
+            message.content = [imageContent];
+          }
+        }
+      }
+    }
+  }
+
   createProcessor(): MessageProcessorFunction {
     return (originalMessages: Message[], modifiedMessages: Message[]) => {
       for (const message of modifiedMessages) {
-        // Only process user messages (images typically come from users)
-        if (message.role === 'user') {
+        // Process user messages (images from user input)
+        if (message.role === "user") {
           this.processMessageContent(message);
         }
         
+        // Process tool messages (images from loadImageAsBase64 tool)
+        // Tool responses come back as JSON strings that need to be parsed
+        // and converted to proper image content before the agent sees them
+        if (message.role === "tool") {
+          this.processToolMessageContent(message);
+        }
+
         // Process tool calls in any message
         this.processToolCallArguments(message);
       }
@@ -184,7 +224,120 @@ export class Base64ImageDetector {
   setSupportedFormats(formats: string[]): void {
     this.supportedFormats = formats;
   }
+
+  /**
+   * Registers the loadImageAsBase64 tool with the ToolsService
+   */
+  registerTool(toolsService?: ToolsService): void {
+    if (toolsService) {
+      const toolDefinition = {
+        type: "function" as const,
+        function: {
+          name: "loadImageAsBase64",
+          description:
+            "Load an image file from a file path and return it as a base64 data URL. This enables you to view and analyze images from the filesystem. Use this when the user provides a screenshot path or asks you to look at an image file.",
+          parameters: {
+            type: "object",
+            positional: true,
+            properties: {
+              filePath: {
+                type: "string",
+                description: "The absolute or relative path to the image file",
+              },
+              detail: {
+                type: "string",
+                description:
+                  "The level of detail for image analysis. Options: 'auto' (default), 'low' (faster, less detail), 'high' (slower, more detail)",
+              },
+            },
+            required: ["filePath"],
+          },
+        },
+      };
+
+      toolsService.addTools([toolDefinition]);
+      toolsService.addFunctions({
+        loadImageAsBase64: async (
+          filePath: string,
+          detail?: "auto" | "low" | "high"
+        ) => {
+          return await this.loadImageAsBase64(filePath, detail);
+        },
+      });
+    }
+  }
+
+  /**
+   * Loads an image from a file path and returns it as a base64 data URL
+   */
+  private async loadImageAsBase64(
+    filePath: string,
+    detail: "auto" | "low" | "high" = "auto"
+  ): Promise<string> {
+    try {
+      // Check if file exists
+      if (!fs.existsSync(filePath)) {
+        throw new Error(`File not found: ${filePath}`);
+      }
+
+      // Get file stats to verify it's a file
+      const stats = fs.statSync(filePath);
+      if (!stats.isFile()) {
+        throw new Error(`Path is not a file: ${filePath}`);
+      }
+
+      // Detect MIME type from file extension
+      const ext = path.extname(filePath).toLowerCase().replace(".", "");
+      const mimeTypeMap: { [key: string]: string } = {
+        png: "image/png",
+        jpg: "image/jpeg",
+        jpeg: "image/jpeg",
+        gif: "image/gif",
+        webp: "image/webp",
+        bmp: "image/bmp",
+        svg: "image/svg+xml",
+      };
+
+      const mimeType = mimeTypeMap[ext];
+      if (!mimeType) {
+        throw new Error(
+          `Unsupported image format: ${ext}. Supported formats: ${Object.keys(
+            mimeTypeMap
+          ).join(", ")}`
+        );
+      }
+
+      // Check if format is supported
+      const simpleType = ext === "jpg" ? "jpeg" : ext;
+      if (!this.supportedFormats.includes(simpleType)) {
+        throw new Error(
+          `Image format ${ext} is not in supported formats: ${this.supportedFormats.join(
+            ", "
+          )}`
+        );
+      }
+
+      // Read the file as base64
+      const imageBuffer = fs.readFileSync(filePath);
+      const base64Data = imageBuffer.toString("base64");
+
+      // Create data URL
+      const dataUrl = `data:${mimeType};base64,${base64Data}`;
+
+      // Return in a format that indicates this is an image
+      // The Base64ImageDetector will convert this to proper image content
+      return JSON.stringify({
+        type: "image_url",
+        image_url: {
+          url: dataUrl,
+          detail: detail || this.imageDetail,
+        },
+      });
+    } catch (error) {
+      throw new Error(`Failed to load image: ${error.message}`);
+    }
+  }
 }
 
 // Global instance
-export const globalBase64ImageDetector = new Base64ImageDetector();
+export const globalBase64ImageDetector = new Base64ImageProcessor();

package/src/processors/JsonCompressor.ts

@@ -101,14 +101,14 @@ export class JsonCompressor {
         // Check if it has type: "text" and a text field
         if (firstContent.type === 'text' && typeof firstContent.text === 'string') {
           try {
-            // Try to parse the nested text as JSON
-            const nestedData = JSON.parse(firstContent.text);
+            // Try to parse the nested text as JSON (recursively to handle double-encoding)
+            const nestedData = this.tryParseJson(firstContent.text);
 
             // Return a structured object that preserves the MCP format but exposes the data
             return {
               _mcp_format: true,
               _raw_structure: { content: [{ type: 'text' }] },
-              data: nestedData
+              _data: nestedData
             };
           } catch (e) {
             // If nested text isn't JSON, return original parsed
@@ -133,13 +133,13 @@ export class JsonCompressor {
     }
 
     // Handle MCP format objects
-    if (obj && typeof obj === 'object' && obj._mcp_format === true && obj.data) {
+    if (obj && typeof obj === 'object' && obj._mcp_format === true && obj._data) {
       // Generate schema for the actual data, not the wrapper
-      const dataSchema = this.generateSchema(obj.data, maxDepth, currentDepth);
+      const dataSchema = this.generateSchema(obj._data, maxDepth, currentDepth);
       return {
         type: 'mcp_response',
         properties: {
-          data: dataSchema
+          _data: dataSchema
         }
       };
     }

package/src/processors/TokenCompressor.ts

@@ -176,7 +176,7 @@ export class TokenCompressor implements JsonCompressorStorage {
     const jsonObj = this.tryParseJson(content);
     if (jsonObj) {
       // For MCP format, work with the actual data
-      const dataToCompress = jsonObj._mcp_format ? jsonObj.data : jsonObj;
+      const dataToCompress = jsonObj._mcp_format ? jsonObj._data : jsonObj;
 
       // Generate and store schema
       const schema = this.jsonCompressor.generateSchema(jsonObj);

package/src/processors/ToolResponseCache.ts

@@ -1,6 +1,7 @@
 import { Message } from "../clients/types";
 import { MessageProcessorFunction } from "../services/MessageProcessor";
 import { ToolsService } from "../services";
+import { JsonCompressor } from "./JsonCompressor";
 import {
   jqToolResponseDefinition,
   executeJqQuery,
@@ -32,20 +33,34 @@ export class ToolResponseCache {
   private storage: ToolResponseStorage = {};
   private metadataStorage: ToolResponseMetadataStorage = {};
   private toolNameMap: { [toolCallId: string]: string } = {};
+  private jsonCompressor: JsonCompressor;
 
-  constructor(toolsService: ToolsService) {
+  constructor(toolsService: ToolsService, jsonCompressor?: JsonCompressor) {
+    // Use provided JsonCompressor or create a minimal storage adapter
+    this.jsonCompressor = jsonCompressor || this.createMinimalJsonCompressor();
     this.registerTool(toolsService);
   }
 
   /**
-   * Attempts to parse content as JSON and returns parsed object if successful
+   * Creates a minimal JsonCompressor instance for JSON parsing utilities
+   * This is used when no JsonCompressor is provided to the constructor
    */
-  private tryParseJson(content: string): any | null {
-    try {
-      return JSON.parse(content);
-    } catch {
-      return null;
-    }
+  private createMinimalJsonCompressor(): JsonCompressor {
+    // Create a minimal storage adapter that satisfies JsonCompressorStorage interface
+    const minimalStorage = {
+      storeString: (key: string, value: string) => {
+        // No-op for ToolResponseCache's internal use
+      },
+      generateKey: () => {
+        return `temp_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+      },
+      estimateTokens: (text: string) => {
+        return Math.ceil(text.length / 4);
+      },
+    };
+
+    // Return a JsonCompressor instance with minimal settings
+    return new JsonCompressor(minimalStorage, 4000, 8000, "expandTokens");
   }
 
   /**
@@ -53,7 +68,7 @@ export class ToolResponseCache {
    */
   public parseNestedJsonStrings(obj: any): any {
     if (typeof obj === "string") {
-      const parsed = this.tryParseJson(obj);
+      const parsed = this.jsonCompressor.tryParseJson(obj);
       if (parsed) {
         return this.parseNestedJsonStrings(parsed);
       }
@@ -90,8 +105,46 @@ export class ToolResponseCache {
       return;
     }
 
-    // Store the original content for later JQ/grep manipulation
-    this.storage[toolCallId] = content;
+    // Try to parse the content
+    const parsed = this.jsonCompressor.tryParseJson(content);
+
+    if (parsed && typeof parsed === 'object' && parsed._mcp_format === true && parsed._data) {
+      // For MCP format responses, store the data in a normalized structure
+      // This allows JQ queries to work directly against the data array
+      // Store as JSON string to maintain compatibility with existing query methods
+      this.storage[toolCallId] = JSON.stringify({
+        _mcp_format: true,
+        _raw_structure: parsed._raw_structure,
+        _data: parsed._data
+      });
+    } else if (parsed !== null) {
+      // Check if content is double-encoded by trying to parse again
+      // Only re-stringify if we detected and handled double-encoding
+      try {
+        const outerParse = JSON.parse(content);
+        if (typeof outerParse === 'string') {
+          // This is double-encoded JSON, store the fully parsed result
+          if (typeof parsed === 'object') {
+            this.storage[toolCallId] = JSON.stringify(parsed);
+          } else if (typeof parsed === 'string') {
+            // Parsed to a string, store it as-is
+            this.storage[toolCallId] = parsed;
+          } else {
+            // Store the original if we couldn't parse further
+            this.storage[toolCallId] = content;
+          }
+        } else {
+          // Not double-encoded, store original to preserve formatting
+          this.storage[toolCallId] = content;
+        }
+      } catch {
+        // Not valid JSON, store as-is
+        this.storage[toolCallId] = content;
+      }
+    } else {
+      // Could not parse as JSON, store as-is
+      this.storage[toolCallId] = content;
+    }
 
     // Store metadata for reference
     this.metadataStorage[toolCallId] = {

package/src/processors/index.ts

@@ -1,4 +1,4 @@
-export { Base64ImageDetector } from "./Base64ImageDetector";
+export { Base64ImageProcessor } from "./Base64ImageDetector";
 export { CustomVariables } from "./CustomVariables";
 export { TokenCompressor } from "./TokenCompressor";
 export { JsonCompressor, JsonSchema, CompressionMetadata, JsonCompressorStorage } from "./JsonCompressor";