@tyvm/knowhow 0.0.63 → 0.0.65

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.63",
+  "version": "0.0.65",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {
@@ -40,6 +40,7 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.39.0",
     "@aws-sdk/client-s3": "^3.588.0",
+    "@tyvm/knowhow-tunnel": "^0.0.1",
     "@google/genai": "^0.14.1",
     "@inquirer/editor": "^4.2.18",
     "@linear/sdk": "^12.0.0",

package/src/chat/modules/AgentModule.ts

@@ -21,6 +21,7 @@ import {
   CustomVariables,
   XmlToolCallProcessor,
   HarmonyToolProcessor,
+  Base64ImageProcessor,
 } from "../../processors/index";
 import { TaskInfo, ChatSession } from "../types";
 import { agents } from "../../agents";
@@ -589,7 +590,9 @@ Please continue from where you left off and complete the original request.
           Boolean(msg.role === "tool" && msg.tool_call_id)
         ),
       ];
+
       agent.messageProcessor.setProcessors("pre_call", [
+        new Base64ImageProcessor(agent.tools).createProcessor(),
         ...caching,
         new CustomVariables(agent.tools).createProcessor(),
       ]);
@@ -599,7 +602,10 @@ Please continue from where you left off and complete the original request.
         new HarmonyToolProcessor().createProcessor(),
       ]);
 
-      agent.messageProcessor.setProcessors("post_tools", caching);
+      agent.messageProcessor.setProcessors("post_tools", [
+        new Base64ImageProcessor(agent.tools).createProcessor(),
+        ...caching,
+      ]);
 
       // Set up event listeners
       if (!agent.agentEvents.listenerCount(agent.eventTypes.toolCall)) {
@@ -628,11 +634,19 @@ Please continue from where you left off and complete the original request.
 
       const taskCompleted = new Promise<string>((resolve) => {
         agent.agentEvents.once(agent.eventTypes.done, async (doneMsg) => {
-          console.log("Agent has completed the task.");
+          console.log("🎯 [AgentModule] Task Completed");
           done = true;
           output = doneMsg || "No response from the AI";
           // Update task info
           taskInfo = this.taskRegistry.get(taskId);
+
+          // Wait for AgentSync to finish before resolving
+          if (knowhowTaskId) {
+            console.log("🎯 [AgentModule] Waiting for sync finalization...");
+            await this.agentSync.waitForFinalization();
+            console.log("🎯 [AgentModule] Sync finalization complete");
+          }
+
           if (taskInfo) {
             taskInfo.status = "completed";
             // Update final cost from agent
@@ -641,7 +655,9 @@ Please continue from where you left off and complete the original request.
             this.updateSession(taskId, agent.getThreads());
             taskInfo.endTime = Date.now();
           }
+
           console.log(Marked.parse(output));
+          console.log("🎯 [AgentModule] Task Complete");
           resolve(doneMsg);
         });
       });

package/src/cli.ts

@@ -54,10 +54,11 @@ async function setupServices() {
   // Add Mcp service to tool context directly so MCP management tools can access it
   Tools.addContext("Mcp", Mcp);
 
-  await Promise.all([
-    Mcp.connectToConfigured(Tools),
-    Clients.registerConfiguredModels(),
-  ]);
+  console.log("🔌 Connecting to MCP...");
+  await Mcp.connectToConfigured(Tools);
+  console.log("Connecting to clients...");
+  await Clients.registerConfiguredModels();
+  console.log("✓ Services are set up and ready to go!");
 }
 
 // Utility function to read from stdin

package/src/clients/anthropic.ts

@@ -184,12 +184,44 @@ export class GenericAnthropicClient implements GenericClient {
             });
           }
 
+          // Convert tool message content to appropriate format
+          let toolResultContent: string | (Anthropic.TextBlockParam | Anthropic.ImageBlockParam)[];
+
+          if (typeof msg.content === "string") {
+            toolResultContent = msg.content;
+          } else if (Array.isArray(msg.content)) {
+            // Transform image_url format to Anthropic's image format
+            toolResultContent = msg.content.map((item): Anthropic.TextBlockParam | Anthropic.ImageBlockParam => {
+              if (item.type === "image_url") {
+                const url = item.image_url.url;
+                const isDataUrl = url.startsWith("data:");
+                const base64Data = isDataUrl ? url.split(",")[1] : url;
+                const mediaType = isDataUrl ? url.match(/data:([^;]+);/)?.[1] || "image/jpeg" : "image/jpeg";
+
+                return {
+                  type: "image" as const,
+                  source: {
+                    type: "base64" as const,
+                    media_type: mediaType as any,
+                    data: base64Data,
+                  },
+                };
+              } else if (item.type === "text") {
+                return { type: "text" as const, text: item.text };
+              }
+              // Fallback for unknown types
+              return { type: "text" as const, text: String(item) };
+            }) as (Anthropic.TextBlockParam | Anthropic.ImageBlockParam)[];
+          } else {
+            toolResultContent = String(msg.content);
+          }
+
           toolMessages.push({
             role: "user",
             content: [
               {
                 type: "tool_result",
-                content: msg.content as string,
+                content: toolResultContent,
                 tool_use_id: msg.tool_call_id,
               },
             ],
@@ -327,9 +359,19 @@ export class GenericAnthropicClient implements GenericClient {
     return {
       [Models.anthropic.Opus4_6]: {
         input: 5.0,
+        input_gt_200k: 10.0,
         cache_write: 6.25,
         cache_hit: 0.5,
         output: 25.0,
+        output_gt_200k: 37.5,
+      },
+      [Models.anthropic.Sonnet4_6]: {
+        input: 3.0,
+        input_gt_200k: 6.0,
+        cache_write: 3.75,
+        cache_hit: 0.3,
+        output: 15.0,
+        output_gt_200k: 22.5,
       },
       [Models.anthropic.Opus4_5]: {
         input: 5.0,
@@ -351,15 +393,19 @@ export class GenericAnthropicClient implements GenericClient {
       },
       [Models.anthropic.Sonnet4]: {
         input: 3.0,
+        input_gt_200k: 6.0,
         cache_write: 3.75,
         cache_hit: 0.3,
         output: 15.0,
+        output_gt_200k: 22.5,
       },
       [Models.anthropic.Sonnet4_5]: {
         input: 3.0,
+        input_gt_200k: 6.0,
         cache_write: 3.75,
         cache_hit: 0.3,
         output: 15.0,
+        output_gt_200k: 22.5,
       },
       [Models.anthropic.Haiku4_5]: {
         input: 1,
@@ -407,19 +453,36 @@ export class GenericAnthropicClient implements GenericClient {
       return undefined;
     }
 
+    // Calculate total input tokens (excluding cached tokens)
+    const inputTokens = usage.input_tokens;
+
     const cachedInputTokens = usage.cache_creation_input_tokens;
     const cachedInputCost = (cachedInputTokens * pricing.cache_write) / 1e6;
 
     const cachedReadTokens = usage.cache_read_input_tokens;
     const cachedReadCost = (cachedReadTokens * pricing.cache_hit) / 1e6;
 
-    const inputTokens = usage.input_tokens;
-    const inputCost = ((inputTokens - cachedInputCost) * pricing.input) / 1e6;
+    // Calculate input cost with >200K input token pricing if applicable
+    let inputCost = 0;
+    const totalInputTokens = inputTokens + cachedInputTokens + cachedReadTokens;
+
+    if (totalInputTokens > 200000 && pricing.input_gt_200k) {
+      inputCost = (totalInputTokens * pricing.input_gt_200k) / 1e6;
+    } else {
+      inputCost = (totalInputTokens * pricing.input) / 1e6;
+    }
 
     const outputTokens = usage.output_tokens;
-    const outputCost = (outputTokens * pricing.output) / 1e6;
 
-    const total = cachedInputCost + inputCost + outputCost;
+    // Calculate output cost with >200K input token pricing if applicable
+    let outputCost = 0;
+    if (totalInputTokens > 200000 && pricing.output_gt_200k) {
+      outputCost = (outputTokens * pricing.output_gt_200k) / 1e6;
+    } else {
+      outputCost = (outputTokens * pricing.output) / 1e6;
+    }
+
+    const total = cachedInputCost + cachedReadCost + inputCost + outputCost;
     return total;
   }
 

package/src/config.ts

@@ -87,6 +87,13 @@ const defaultConfig = {
     logLevel: "info",
     completionTimeout: 5000,
   },
+
+  worker: {
+    tunnel: {
+      enabled: false,
+      allowedPorts: [],
+    },
+  },
 } as Config;
 
 const defaultLanguage = {

package/src/processors/Base64ImageDetector.ts

@@ -1,5 +1,8 @@
+import * as fs from "fs";
+import * as path from "path";
 import { Message } from "../clients/types";
 import { MessageProcessorFunction } from "../services/MessageProcessor";
+import { ToolsService } from "../services";
 
 interface ImageContent {
   type: "image_url";
@@ -14,23 +17,23 @@ interface TextContent {
   text: string;
 }
 
-export class Base64ImageDetector {
-  private imageDetail: "auto" | "low" | "high";
-  private supportedFormats: string[];
+export class Base64ImageProcessor {
+  private imageDetail: "auto" | "low" | "high" = "auto";
+  private supportedFormats = ["png", "jpeg", "jpg", "gif", "webp"];
 
-  constructor(
-    imageDetail: "auto" | "low" | "high" = "auto",
-    supportedFormats: string[] = ["png", "jpeg", "jpg", "gif", "webp"]
-  ) {
-    this.imageDetail = imageDetail;
-    this.supportedFormats = supportedFormats;
+  constructor(toolsService?: ToolsService) {
+    this.registerTool(toolsService);
   }
 
-  private isBase64Image(text: string): { isImage: boolean; mimeType?: string; data?: string } {
+  private isBase64Image(text: string): {
+    isImage: boolean;
+    mimeType?: string;
+    data?: string;
+  } {
     // Check for data URL format: data:image/type;base64,actualdata
     const dataUrlPattern = /^data:image\/([a-zA-Z]+);base64,(.+)$/;
     const match = text.match(dataUrlPattern);
-    
+
     if (match) {
       const [, mimeType, data] = match;
       if (this.supportedFormats.includes(mimeType.toLowerCase())) {
@@ -47,14 +50,17 @@ export class Base64ImageDetector {
       try {
         const decoded = atob(header);
         // Check for common image file signatures
-        if (decoded.startsWith('\x89PNG')) {
-          return { isImage: true, mimeType: 'png', data: text };
-        } else if (decoded.startsWith('\xFF\xD8\xFF')) {
-          return { isImage: true, mimeType: 'jpeg', data: text };
-        } else if (decoded.startsWith('GIF87a') || decoded.startsWith('GIF89a')) {
-          return { isImage: true, mimeType: 'gif', data: text };
-        } else if (decoded.startsWith('RIFF') && decoded.includes('WEBP')) {
-          return { isImage: true, mimeType: 'webp', data: text };
+        if (decoded.startsWith("\x89PNG")) {
+          return { isImage: true, mimeType: "png", data: text };
+        } else if (decoded.startsWith("\xFF\xD8\xFF")) {
+          return { isImage: true, mimeType: "jpeg", data: text };
+        } else if (
+          decoded.startsWith("GIF87a") ||
+          decoded.startsWith("GIF89a")
+        ) {
+          return { isImage: true, mimeType: "gif", data: text };
+        } else if (decoded.startsWith("RIFF") && decoded.includes("WEBP")) {
+          return { isImage: true, mimeType: "webp", data: text };
         }
       } catch (e) {
         // Not valid base64 or not an image
@@ -66,26 +72,26 @@ export class Base64ImageDetector {
 
   private convertBase64ToImageContent(text: string): ImageContent | null {
     const detection = this.isBase64Image(text);
-    
+
     if (!detection.isImage) {
       return null;
     }
 
-    const dataUrl = detection.data!.startsWith('data:') 
-      ? detection.data 
+    const dataUrl = detection.data!.startsWith("data:")
+      ? detection.data
       : `data:image/${detection.mimeType};base64,${detection.data}`;
 
     return {
       type: "image_url",
       image_url: {
         url: dataUrl,
-        detail: this.imageDetail
-      }
+        detail: this.imageDetail,
+      },
     };
   }
 
   private processMessageContent(message: Message): void {
-    if (typeof message.content === 'string') {
+    if (typeof message.content === "string") {
       const imageContent = this.convertBase64ToImageContent(message.content);
       if (imageContent) {
         // Convert string content to multimodal array
@@ -94,9 +100,9 @@ export class Base64ImageDetector {
     } else if (Array.isArray(message.content)) {
       // Process each content item
       const newContent: (TextContent | ImageContent)[] = [];
-      
+
       for (const item of message.content) {
-        if (item.type === 'text' && item.text) {
+        if (item.type === "text" && item.text) {
           const imageContent = this.convertBase64ToImageContent(item.text);
           if (imageContent) {
             newContent.push(imageContent);
@@ -107,7 +113,7 @@ export class Base64ImageDetector {
           newContent.push(item as TextContent | ImageContent);
         }
       }
-      
+
       message.content = newContent;
     }
   }
@@ -119,22 +125,22 @@ export class Base64ImageDetector {
           try {
             const args = JSON.parse(toolCall.function.arguments);
             let modified = false;
-            
+
             // Recursively check all string values in arguments
             const processValue = (obj: any): any => {
-              if (typeof obj === 'string') {
+              if (typeof obj === "string") {
                 const detection = this.isBase64Image(obj);
                 if (detection.isImage) {
                   modified = true;
-                  const dataUrl = detection.data!.startsWith('data:') 
-                    ? detection.data 
+                  const dataUrl = detection.data!.startsWith("data:")
+                    ? detection.data
                     : `data:image/${detection.mimeType};base64,${detection.data}`;
                   return `[CONVERTED TO IMAGE: ${dataUrl.substring(0, 50)}...]`;
                 }
                 return obj;
               } else if (Array.isArray(obj)) {
                 return obj.map(processValue);
-              } else if (obj && typeof obj === 'object') {
+              } else if (obj && typeof obj === "object") {
                 const result = {};
                 for (const [key, value] of Object.entries(obj)) {
                   result[key] = processValue(value);
@@ -143,7 +149,7 @@ export class Base64ImageDetector {
               }
               return obj;
             };
-            
+
             const processedArgs = processValue(args);
             if (modified) {
               toolCall.function.arguments = JSON.stringify(processedArgs);
@@ -152,10 +158,13 @@ export class Base64ImageDetector {
             // Arguments are not valid JSON, treat as string
             const detection = this.isBase64Image(toolCall.function.arguments);
             if (detection.isImage) {
-              const dataUrl = detection.data!.startsWith('data:') 
-                ? detection.data 
+              const dataUrl = detection.data!.startsWith("data:")
+                ? detection.data
                 : `data:image/${detection.mimeType};base64,${detection.data}`;
-              toolCall.function.arguments = `[CONVERTED TO IMAGE: ${dataUrl.substring(0, 50)}...]`;
+              toolCall.function.arguments = `[CONVERTED TO IMAGE: ${dataUrl.substring(
+                0,
+                50
+              )}...]`;
             }
           }
         }
@@ -163,14 +172,45 @@ export class Base64ImageDetector {
     }
   }
 
+  private processToolMessageContent(message: Message): void {
+    // Tool messages have string content that might be a JSON string containing image data
+    if (typeof message.content === "string" && message.content.trim()) {
+      try {
+        // Try to parse as JSON
+        const parsed = JSON.parse(message.content);
+        
+        // Check if it's an image_url object
+        if (parsed.type === "image_url" && parsed.image_url?.url) {
+          // Convert the tool message content from JSON string to an array with the image
+          message.content = [parsed];
+        }
+      } catch (e) {
+        // Not JSON, check if it's a plain base64 string (only if still a string)
+        if (typeof message.content === "string") {
+          const imageContent = this.convertBase64ToImageContent(message.content);
+          if (imageContent) {
+            message.content = [imageContent];
+          }
+        }
+      }
+    }
+  }
+
   createProcessor(): MessageProcessorFunction {
     return (originalMessages: Message[], modifiedMessages: Message[]) => {
       for (const message of modifiedMessages) {
-        // Only process user messages (images typically come from users)
-        if (message.role === 'user') {
+        // Process user messages (images from user input)
+        if (message.role === "user") {
           this.processMessageContent(message);
         }
         
+        // Process tool messages (images from loadImageAsBase64 tool)
+        // Tool responses come back as JSON strings that need to be parsed
+        // and converted to proper image content before the agent sees them
+        if (message.role === "tool") {
+          this.processToolMessageContent(message);
+        }
+
         // Process tool calls in any message
         this.processToolCallArguments(message);
       }
@@ -184,7 +224,120 @@ export class Base64ImageDetector {
   setSupportedFormats(formats: string[]): void {
     this.supportedFormats = formats;
   }
+
+  /**
+   * Registers the loadImageAsBase64 tool with the ToolsService
+   */
+  registerTool(toolsService?: ToolsService): void {
+    if (toolsService) {
+      const toolDefinition = {
+        type: "function" as const,
+        function: {
+          name: "loadImageAsBase64",
+          description:
+            "Load an image file from a file path and return it as a base64 data URL. This enables you to view and analyze images from the filesystem. Use this when the user provides a screenshot path or asks you to look at an image file.",
+          parameters: {
+            type: "object",
+            positional: true,
+            properties: {
+              filePath: {
+                type: "string",
+                description: "The absolute or relative path to the image file",
+              },
+              detail: {
+                type: "string",
+                description:
+                  "The level of detail for image analysis. Options: 'auto' (default), 'low' (faster, less detail), 'high' (slower, more detail)",
+              },
+            },
+            required: ["filePath"],
+          },
+        },
+      };
+
+      toolsService.addTools([toolDefinition]);
+      toolsService.addFunctions({
+        loadImageAsBase64: async (
+          filePath: string,
+          detail?: "auto" | "low" | "high"
+        ) => {
+          return await this.loadImageAsBase64(filePath, detail);
+        },
+      });
+    }
+  }
+
+  /**
+   * Loads an image from a file path and returns it as a base64 data URL
+   */
+  private async loadImageAsBase64(
+    filePath: string,
+    detail: "auto" | "low" | "high" = "auto"
+  ): Promise<string> {
+    try {
+      // Check if file exists
+      if (!fs.existsSync(filePath)) {
+        throw new Error(`File not found: ${filePath}`);
+      }
+
+      // Get file stats to verify it's a file
+      const stats = fs.statSync(filePath);
+      if (!stats.isFile()) {
+        throw new Error(`Path is not a file: ${filePath}`);
+      }
+
+      // Detect MIME type from file extension
+      const ext = path.extname(filePath).toLowerCase().replace(".", "");
+      const mimeTypeMap: { [key: string]: string } = {
+        png: "image/png",
+        jpg: "image/jpeg",
+        jpeg: "image/jpeg",
+        gif: "image/gif",
+        webp: "image/webp",
+        bmp: "image/bmp",
+        svg: "image/svg+xml",
+      };
+
+      const mimeType = mimeTypeMap[ext];
+      if (!mimeType) {
+        throw new Error(
+          `Unsupported image format: ${ext}. Supported formats: ${Object.keys(
+            mimeTypeMap
+          ).join(", ")}`
+        );
+      }
+
+      // Check if format is supported
+      const simpleType = ext === "jpg" ? "jpeg" : ext;
+      if (!this.supportedFormats.includes(simpleType)) {
+        throw new Error(
+          `Image format ${ext} is not in supported formats: ${this.supportedFormats.join(
+            ", "
+          )}`
+        );
+      }
+
+      // Read the file as base64
+      const imageBuffer = fs.readFileSync(filePath);
+      const base64Data = imageBuffer.toString("base64");
+
+      // Create data URL
+      const dataUrl = `data:${mimeType};base64,${base64Data}`;
+
+      // Return in a format that indicates this is an image
+      // The Base64ImageDetector will convert this to proper image content
+      return JSON.stringify({
+        type: "image_url",
+        image_url: {
+          url: dataUrl,
+          detail: detail || this.imageDetail,
+        },
+      });
+    } catch (error) {
+      throw new Error(`Failed to load image: ${error.message}`);
+    }
+  }
 }
 
 // Global instance
-export const globalBase64ImageDetector = new Base64ImageDetector();
+export const globalBase64ImageDetector = new Base64ImageProcessor();

package/src/processors/index.ts

@@ -1,4 +1,4 @@
-export { Base64ImageDetector } from "./Base64ImageDetector";
+export { Base64ImageProcessor } from "./Base64ImageDetector";
 export { CustomVariables } from "./CustomVariables";
 export { TokenCompressor } from "./TokenCompressor";
 export { JsonCompressor, JsonSchema, CompressionMetadata, JsonCompressorStorage } from "./JsonCompressor";

package/src/services/AgentSynchronization.ts

@@ -32,6 +32,7 @@ export class AgentSynchronization {
   private baseUrl: string;
   private knowhowTaskId: string | undefined;
   private eventHandlersSetup: boolean = false;
+  private finalizationPromise: Promise<void> | null = null;
 
   constructor(baseUrl: string = KNOWHOW_API_URL) {
     this.baseUrl = baseUrl;
@@ -248,28 +249,44 @@ export class AgentSynchronization {
     // Listen to completion event to finalize task
     agent.agentEvents.on(agent.eventTypes.done, async (result: string) => {
       if (!this.knowhowTaskId || !this.baseUrl) {
+        console.warn(`⚠️ [AgentSync] Cannot finalize: knowhowTaskId=${this.knowhowTaskId}, baseUrl=${this.baseUrl}`);
         return;
       }
 
-      try {
-        await wait(200);
-        console.log(
-          `Updating Knowhow chat task on completion..., ${this.knowhowTaskId}`
-        );
-        await this.updateChatTask(this.knowhowTaskId, agent, false, result);
-        console.log(`✅ Completed Knowhow chat task: ${this.knowhowTaskId}`);
-      } catch (error) {
-        console.error(`❌ Error finalizing task:`, error);
-      }
+      console.log(`🎯 [AgentSync] Done event received for task: ${this.knowhowTaskId}`);
+      
+      // Create a promise that tracks finalization
+      this.finalizationPromise = (async () => {
+        try {
+          console.log(
+            `Updating Knowhow chat task on completion..., ${this.knowhowTaskId}`
+          );
+          await this.updateChatTask(this.knowhowTaskId!, agent, false, result);
+          console.log(`✅ Completed Knowhow chat task: ${this.knowhowTaskId}`);
+        } catch (error) {
+          console.error(`❌ Error finalizing task:`, error);
+          throw error; // Re-throw so CLI can handle it
+        }
+      })();
     });
   }
 
+  /**
+   * Wait for finalization to complete (for CLI usage)
+   */
+  async waitForFinalization(): Promise<void> {
+    if (this.finalizationPromise) {
+      await this.finalizationPromise;
+    }
+  }
+
   /**
    * Reset synchronization state (useful for reusing the service)
    */
   reset(): void {
     this.knowhowTaskId = undefined;
     this.eventHandlersSetup = false;
+    this.finalizationPromise = null;
   }
 
   /**

package/src/types.ts

@@ -66,6 +66,16 @@ export type Config = {
     sandbox?: boolean;
     volumes?: string[];
     envFile?: string;
+    tunnel?: {
+      enabled?: boolean;
+      allowedPorts?: number[];
+      maxConcurrentStreams?: number;
+      portMapping?: {
+        [containerPort: number]: number; // containerPort -> hostPort
+      };
+      localHost?: string; // Default: "127.0.0.1", can be "host.docker.internal" for Docker
+      enableUrlRewriting?: boolean; // Enable URL rewriting for localhost URLs (default: true)
+    };
   };
 };
 
@@ -143,6 +153,7 @@ export type ChatInteraction = {
 export const Models = {
   anthropic: {
     Opus4_6: "claude-opus-4-6",
+    Sonnet4_6: "claude-sonnet-4-6",
     Opus4_5: "claude-opus-4-5-20251101",
     Opus4: "claude-opus-4-20250514",
     Opus4_1: "claude-opus-4-1-20250805",