@tyvm/knowhow 0.0.159 → 0.0.161

package/src/processors/Base64ImageDetector.ts

@@ -1,502 +0,0 @@
-import * as fs from "fs";
-import * as path from "path";
-import { Message } from "../clients/types";
-import { MessageProcessorFunction } from "../services/MessageProcessor";
-import { ToolsService } from "../services";
-
-interface ImageContent {
-  type: "image_url";
-  image_url: {
-    url: string;
-    detail?: "auto" | "low" | "high";
-  };
-}
-
-interface TextContent {
-  type: "text";
-  text: string;
-}
-
-/**
- * Regex that matches common image file paths (absolute, relative, or just filenames)
- * ending in a known image extension.
- */
-const IMAGE_PATH_REGEX =
-  /(?:^|[\s"'(,\[{])([^\s"'(,\[{]*?\.(?:png|jpe?g|gif|webp|bmp|svg))(?=$|[\s"'),\]}])/gi;
-
-const IMAGE_EXTENSIONS = ["png", "jpeg", "jpg", "gif", "webp", "bmp", "svg"];
-
-/**
- * Sentinel prefix used to identify image-path hint blocks injected by this processor.
- * Any existing hint block starting with this prefix is stripped before re-evaluation,
- * ensuring the hint is idempotent and never duplicated across processor invocations.
- */
-const TIP_SENTINEL = "[TIP: An image file path was detected:";
-const TIP_SENTINEL_MULTI = "[TIP: Image file paths were detected.";
-
-export class Base64ImageProcessor {
-  private imageDetail: "auto" | "low" | "high" = "auto";
-  private supportedFormats = ["png", "jpeg", "jpg", "gif", "webp"];
-
-  constructor(toolsService?: ToolsService) {
-    this.registerTool(toolsService);
-  }
-
-  private isBase64Image(text: string): {
-    isImage: boolean;
-    mimeType?: string;
-    data?: string;
-  } {
-    // Check for data URL format: data:image/type;base64,actualdata
-    const dataUrlPattern = /^data:image\/([a-zA-Z]+);base64,(.+)$/;
-    const match = text.match(dataUrlPattern);
-
-    if (match) {
-      const [, mimeType, data] = match;
-      if (this.supportedFormats.includes(mimeType.toLowerCase())) {
-        return { isImage: true, mimeType, data };
-      }
-    }
-
-    // Check for plain base64 that might be an image
-    // This is a heuristic - look for long base64 strings that might be images
-    const base64Pattern = /^[A-Za-z0-9+/]+=*$/;
-    if (base64Pattern.test(text) && text.length > 100) {
-      // Try to detect image type from base64 header
-      const header = text.substring(0, 50);
-      try {
-        const decoded = atob(header);
-        // Check for common image file signatures
-        if (decoded.startsWith("\x89PNG")) {
-          return { isImage: true, mimeType: "png", data: text };
-        } else if (decoded.startsWith("\xFF\xD8\xFF")) {
-          return { isImage: true, mimeType: "jpeg", data: text };
-        } else if (
-          decoded.startsWith("GIF87a") ||
-          decoded.startsWith("GIF89a")
-        ) {
-          return { isImage: true, mimeType: "gif", data: text };
-        } else if (decoded.startsWith("RIFF") && decoded.includes("WEBP")) {
-          return { isImage: true, mimeType: "webp", data: text };
-        }
-      } catch (e) {
-        // Not valid base64 or not an image
-      }
-    }
-
-    return { isImage: false };
-  }
-
-  private convertBase64ToImageContent(text: string): ImageContent | null {
-    const detection = this.isBase64Image(text);
-
-    if (!detection.isImage) {
-      return null;
-    }
-
-    const dataUrl = detection.data!.startsWith("data:")
-      ? detection.data
-      : `data:image/${detection.mimeType};base64,${detection.data}`;
-
-    return {
-      type: "image_url",
-      image_url: {
-        url: dataUrl,
-        detail: this.imageDetail,
-      },
-    };
-  }
-
-  private processMessageContent(message: Message): void {
-    if (typeof message.content === "string") {
-      const imageContent = this.convertBase64ToImageContent(message.content);
-      if (imageContent) {
-        // Convert string content to multimodal array
-        message.content = [imageContent];
-      }
-    } else if (Array.isArray(message.content)) {
-      // Process each content item
-      const newContent: (TextContent | ImageContent)[] = [];
-
-      for (const item of message.content) {
-        if (item.type === "text" && item.text) {
-          const imageContent = this.convertBase64ToImageContent(item.text);
-          if (imageContent) {
-            newContent.push(imageContent);
-          } else {
-            newContent.push(item as TextContent);
-          }
-        } else {
-          newContent.push(item as TextContent | ImageContent);
-        }
-      }
-
-      message.content = newContent;
-    }
-  }
-
-  /**
-   * Finds all image file paths mentioned in a text string.
-   * Returns deduplicated list of paths like "/tmp/screenshot.png" or "screenshot.jpg".
-   */
-  private findImageFilePaths(text: string): string[] {
-    const found: string[] = [];
-    const seen = new Set<string>();
-    IMAGE_PATH_REGEX.lastIndex = 0;
-    let match: RegExpExecArray | null;
-    while ((match = IMAGE_PATH_REGEX.exec(text)) !== null) {
-      const filePath = match[1];
-      if (filePath && !seen.has(filePath)) {
-        seen.add(filePath);
-        found.push(filePath);
-      }
-    }
-    return found;
-  }
-
-  /**
-   * Appends a hint to a text string if image file paths are detected.
-   * The hint tells the model it can use loadImageAsBase64 to view the image.
-   */
-  private addImagePathHint(text: string): string {
-    // If a hint was already injected into this text, leave it unchanged.
-    // Since we only process the last message per call, a hint is added once
-    // and then the message becomes historical — it is never processed again.
-    if (text.includes(TIP_SENTINEL) || text.includes(TIP_SENTINEL_MULTI)) {
-      return text;
-    }
-
-    const paths = this.findImageFilePaths(text);
-    if (paths.length === 0) return text;
-
-    const hints = paths.map(
-      (p) =>
-        `loadImageAsBase64("${p}") to view the image at ${p}`
-    );
-
-    const hintBlock =
-      paths.length === 1
-        ? `\n\n[TIP: An image file path was detected: ${paths[0]}. Use the \`loadImageAsBase64\` tool with this path to load and view the image: ${hints[0]}]`
-        : `\n\n[TIP: Image file paths were detected. Use the \`loadImageAsBase64\` tool to load and view them:\n${hints.map((h) => `  - ${h}`).join("\n")}]`;
-
-    return text + hintBlock;
-  }
-
-  /**
-   * Applies image path hints to a message's text content items.
-   */
-  private applyImagePathHintsToMessage(message: Message): void {
-    if (typeof message.content === "string") {
-      message.content = this.addImagePathHint(message.content);
-    } else if (Array.isArray(message.content)) {
-      for (const item of message.content) {
-        if (item && (item as TextContent).type === "text" && typeof (item as TextContent).text === "string") {
-          (item as TextContent).text = this.addImagePathHint((item as TextContent).text);
-        }
-      }
-    }
-  }
-
-  private processToolCallArguments(message: Message): void {
-    if (message.tool_calls) {
-      for (const toolCall of message.tool_calls) {
-        if (toolCall.function.arguments) {
-          try {
-            const args = JSON.parse(toolCall.function.arguments);
-            let modified = false;
-
-            // Recursively check all string values in arguments
-            const processValue = (obj: any): any => {
-              if (typeof obj === "string") {
-                const detection = this.isBase64Image(obj);
-                if (detection.isImage) {
-                  modified = true;
-                  const dataUrl = detection.data!.startsWith("data:")
-                    ? detection.data
-                    : `data:image/${detection.mimeType};base64,${detection.data}`;
-                  return `[CONVERTED TO IMAGE: ${dataUrl.substring(0, 50)}...]`;
-                }
-                return obj;
-              } else if (Array.isArray(obj)) {
-                return obj.map(processValue);
-              } else if (obj && typeof obj === "object") {
-                const result = {};
-                for (const [key, value] of Object.entries(obj)) {
-                  result[key] = processValue(value);
-                }
-                return result;
-              }
-              return obj;
-            };
-
-            const processedArgs = processValue(args);
-            if (modified) {
-              toolCall.function.arguments = JSON.stringify(processedArgs);
-            }
-          } catch (e) {
-            // Arguments are not valid JSON, treat as string
-            const detection = this.isBase64Image(toolCall.function.arguments);
-            if (detection.isImage) {
-              const dataUrl = detection.data!.startsWith("data:")
-                ? detection.data
-                : `data:image/${detection.mimeType};base64,${detection.data}`;
-              toolCall.function.arguments = `[CONVERTED TO IMAGE: ${dataUrl.substring(
-                0,
-                50
-              )}...]`;
-            }
-          }
-        }
-      }
-    }
-  }
-
-  private isUsableImageUrl(url: unknown): url is string {
-    if (typeof url !== "string") return false;
-
-    // Data URLs are only usable when they contain an image format this processor
-    // recognizes. In particular, reject TokenCompressor placeholders that happen
-    // to be located inside an otherwise valid-looking image_url content part.
-    if (url.startsWith("data:")) {
-      return this.isBase64Image(url).isImage;
-    }
-
-    try {
-      const parsed = new URL(url);
-      return parsed.protocol === "http:" || parsed.protocol === "https:";
-    } catch {
-      return false;
-    }
-  }
-
-  private processToolMessageContent(message: Message): void {
-    // Tool messages have string content that might be a JSON string containing image data
-    if (typeof message.content === "string" && message.content.trim()) {
-      try {
-        // Try to parse as JSON
-        const parsed = JSON.parse(message.content);
-
-        // Normalize the parsed value into an array of candidate parts so we can
-        // handle BOTH shapes a tool might return:
-        //   1. a single image_url object:  {type:"image_url", image_url:{url}}
-        //   2. an array of parts:          [{type:"image_url", image_url:{url}}, {type:"text", ...}]
-        // (Computer-use screenshot tools and loadWebpage return the array form,
-        //  which the single-object check above previously missed — leaving the
-        //  model to see a raw base64 JSON string instead of an actual image.)
-        const candidates = Array.isArray(parsed) ? parsed : [parsed];
-        const isImagePart = (p: any) =>
-          p &&
-          p.type === "image_url" &&
-          this.isUsableImageUrl(p.image_url?.url);
-        const isTextPart = (p: any) =>
-          p && p.type === "text" && typeof p.text === "string";
-        const hasInvalidImagePart = candidates.some(
-          (p: any) => p?.type === "image_url" && !isImagePart(p)
-        );
-
-        // Only rewrite content containing valid provider-ready images. If an
-        // image URL was compressed or is malformed, preserve the JSON as text so
-        // the agent can inspect/expand it instead of failing the next API call.
-        if (!hasInvalidImagePart && candidates.some(isImagePart)) {
-          message.content = candidates.filter(
-            (p) => isImagePart(p) || isTextPart(p)
-          );
-        }
-      } catch (e) {
-        // Not JSON, check if it's a plain base64 string (only if still a string)
-        if (typeof message.content === "string") {
-          const imageContent = this.convertBase64ToImageContent(message.content);
-          if (imageContent) {
-            message.content = [imageContent];
-          }
-        }
-      }
-    }
-  }
-
-  createProcessor(): MessageProcessorFunction {
-    return (originalMessages: Message[], modifiedMessages: Message[]) => {
-      const lastIndex = modifiedMessages.length - 1;
-      if (lastIndex < 0) return;
-
-      const lastMessage = modifiedMessages[lastIndex];
-
-      // Every request starts with a fresh clone of the persisted thread. Normalize
-      // all images: parallel calls can place another tool result after an image,
-      // while historical image results remain persisted as JSON strings.
-      for (const message of modifiedMessages) {
-        if (message.role === "user") {
-          this.processMessageContent(message);
-        }
-        if (message.role === "tool") {
-          this.processToolMessageContent(message);
-        }
-      }
-
-      // Add path hints only to the newest message. Re-appending hints to history
-      // would multiply them and invalidate the provider's prompt cache.
-      if (lastMessage.role === "tool") {
-        // After processing tool content (which may not convert to image if it's plain text
-        // describing a screenshot path), add hints for any image file paths found in the text.
-        this.applyImagePathHintsToMessage(lastMessage);
-      }
-
-      // Also apply hints to assistant messages — e.g. when an assistant message
-      // contains the result of a screenshot tool that returned a file path.
-      if (lastMessage.role === "assistant") {
-        this.applyImagePathHintsToMessage(lastMessage);
-      }
-
-      // Process tool calls in any message
-      this.processToolCallArguments(lastMessage);
-    };
-  }
-
-  setImageDetail(detail: "auto" | "low" | "high"): void {
-    this.imageDetail = detail;
-  }
-
-  setSupportedFormats(formats: string[]): void {
-    this.supportedFormats = formats;
-  }
-
-  /**
-   * Registers the loadImageAsBase64 tool with the ToolsService
-   */
-  registerTool(toolsService?: ToolsService): void {
-    if (toolsService) {
-      const toolDefinition = {
-        type: "function" as const,
-        function: {
-          name: "loadImageAsBase64",
-          description:
-            "Load an image file from a file path and return it as a base64 data URL. This enables you to view and analyze images from the filesystem. Use this when the user provides a screenshot path or asks you to look at an image file.",
-          parameters: {
-            type: "object",
-            positional: true,
-            properties: {
-              filePath: {
-                type: "string",
-                description: "The absolute or relative path to the image file",
-              },
-              detail: {
-                type: "string",
-                description:
-                  "The level of detail for image analysis. Options: 'auto' (default), 'low' (faster, less detail), 'high' (slower, more detail)",
-              },
-            },
-            required: ["filePath"],
-          },
-        },
-      };
-
-      toolsService.addTools([toolDefinition]);
-      toolsService.addFunctions({
-        loadImageAsBase64: async (
-          filePath: string,
-          detail?: "auto" | "low" | "high"
-        ) => {
-          return await this.loadImageAsBase64(filePath, detail);
-        },
-      });
-    }
-  }
-
-  /**
-   * Loads an image from a file path and returns it as a base64 data URL
-   */
-  private async loadImageAsBase64(
-    filePath: string,
-    detail: "auto" | "low" | "high" = "auto"
-  ): Promise<string> {
-    try {
-      // Check if file exists
-      if (!fs.existsSync(filePath)) {
-        throw new Error(`File not found: ${filePath}`);
-      }
-
-      // Get file stats to verify it's a file
-      const stats = fs.statSync(filePath);
-      if (!stats.isFile()) {
-        throw new Error(`Path is not a file: ${filePath}`);
-      }
-
-      // Detect MIME type from file extension
-      const ext = path.extname(filePath).toLowerCase().replace(".", "");
-      const mimeTypeMap: { [key: string]: string } = {
-        png: "image/png",
-        jpg: "image/jpeg",
-        jpeg: "image/jpeg",
-        gif: "image/gif",
-        webp: "image/webp",
-        bmp: "image/bmp",
-        svg: "image/svg+xml",
-      };
-
-      const mimeType = mimeTypeMap[ext];
-      if (!mimeType) {
-        throw new Error(
-          `Unsupported image format: ${ext}. Supported formats: ${Object.keys(
-            mimeTypeMap
-          ).join(", ")}`
-        );
-      }
-
-      // Check if format is supported
-      const simpleType = ext === "jpg" ? "jpeg" : ext;
-      if (!this.supportedFormats.includes(simpleType)) {
-        throw new Error(
-          `Image format ${ext} is not in supported formats: ${this.supportedFormats.join(
-            ", "
-          )}`
-        );
-      }
-
-      // Read the file as base64
-      const imageBuffer = fs.readFileSync(filePath);
-
-      // Detect actual MIME type from magic bytes (overrides extension-based detection)
-      // This handles cases where a file is saved with the wrong extension (e.g. JPEG saved as .png)
-      let detectedMimeType = mimeType;
-      if (imageBuffer.length >= 4) {
-        // JPEG: starts with FF D8 FF
-        if (imageBuffer[0] === 0xff && imageBuffer[1] === 0xd8 && imageBuffer[2] === 0xff) {
-          detectedMimeType = "image/jpeg";
-        }
-        // PNG: starts with 89 50 4E 47 0D 0A 1A 0A
-        else if (imageBuffer[0] === 0x89 && imageBuffer[1] === 0x50 && imageBuffer[2] === 0x4e && imageBuffer[3] === 0x47) {
-          detectedMimeType = "image/png";
-        }
-        // GIF: starts with 47 49 46 38 (GIF8)
-        else if (imageBuffer[0] === 0x47 && imageBuffer[1] === 0x49 && imageBuffer[2] === 0x46 && imageBuffer[3] === 0x38) {
-          detectedMimeType = "image/gif";
-        }
-        // WebP: starts with 52 49 46 46 (RIFF) ... 57 45 42 50 (WEBP) at offset 8
-        else if (imageBuffer.length >= 12 && imageBuffer[0] === 0x52 && imageBuffer[1] === 0x49 && imageBuffer[2] === 0x46 && imageBuffer[3] === 0x46 &&
-                 imageBuffer[8] === 0x57 && imageBuffer[9] === 0x45 && imageBuffer[10] === 0x42 && imageBuffer[11] === 0x50) {
-          detectedMimeType = "image/webp";
-        }
-      }
-
-      const base64Data = imageBuffer.toString("base64");
-
-      // Create data URL using the actual detected MIME type
-      const dataUrl = `data:${detectedMimeType};base64,${base64Data}`;
-
-      // Return in a format that indicates this is an image
-      // The Base64ImageDetector will convert this to proper image content
-      return JSON.stringify({
-        type: "image_url",
-        image_url: {
-          url: dataUrl,
-          detail: detail || this.imageDetail,
-        },
-      });
-    } catch (error) {
-      throw new Error(`Failed to load image: ${error.message}`);
-    }
-  }
-}
-
-// Global instance
-export const globalBase64ImageDetector = new Base64ImageProcessor();