@tenex-chat/backend 0.9.1

package/src/conversations/utils/image-placeholder.ts

@@ -0,0 +1,281 @@
+/**
+ * Image Placeholder Strategy for Token Cost Reduction
+ *
+ * Images in tool results accumulate massive token costs (~1,600 tokens per image).
+ * This module implements a placeholder strategy:
+ *
+ * 1. First appearance: Image is shown in full (agent can see/analyze it)
+ * 2. Subsequent appearances: Image is replaced with a text placeholder
+ *    that references fs_read(tool='<eventId>') for retrieval
+ *
+ * This reduces token costs by 95-99% for image-heavy conversations while
+ * maintaining retrieval capability.
+ */
+
+import type { ToolResultPart } from "ai";
+import { isImageUrl } from "./image-url-utils";
+
+/**
+ * Prefix used for image placeholders - allows detection of replaced images
+ */
+export const IMAGE_PLACEHOLDER_PREFIX = "[Image:";
+
+/**
+ * Tracker interface for managing which images have been seen in the conversation
+ */
+export interface ImageTracker {
+    /**
+     * Check if an image URL has already been seen in this conversation
+     */
+    hasBeenSeen(imageUrl: string): boolean;
+
+    /**
+     * Mark an image URL as seen (typically after first display)
+     */
+    markAsSeen(imageUrl: string): void;
+
+    /**
+     * Get all seen image URLs
+     */
+    getSeenUrls(): Set<string>;
+}
+
+/**
+ * Create a new image tracker for a conversation
+ */
+export function createImageTracker(): ImageTracker {
+    const seenUrls = new Set<string>();
+
+    return {
+        hasBeenSeen(imageUrl: string): boolean {
+            return seenUrls.has(imageUrl);
+        },
+
+        markAsSeen(imageUrl: string): void {
+            seenUrls.add(imageUrl);
+        },
+
+        getSeenUrls(): Set<string> {
+            return seenUrls;
+        },
+    };
+}
+
+/**
+ * Create a placeholder text for an image that has been seen before.
+ *
+ * @param imageUrlOrName - The image URL or filename
+ * @param eventId - The event ID for retrieval via fs_read
+ * @returns Placeholder text with retrieval instructions
+ */
+export function createImagePlaceholder(
+    imageUrlOrName: string,
+    eventId: string | undefined
+): string {
+    // Extract filename from URL if it looks like a URL
+    let filename = imageUrlOrName;
+    try {
+        const url = new URL(imageUrlOrName);
+        const pathParts = url.pathname.split("/");
+        const lastPart = pathParts[pathParts.length - 1];
+        if (lastPart) {
+            filename = lastPart;
+        }
+    } catch {
+        // Not a URL, use as-is
+    }
+
+    if (eventId) {
+        return `${IMAGE_PLACEHOLDER_PREFIX} ${filename} - use fs_read(tool="${eventId}") to retrieve]`;
+    }
+
+    // Fallback when eventId is not available (shouldn't happen normally)
+    return `${IMAGE_PLACEHOLDER_PREFIX} ${filename} - original context lost, cannot retrieve]`;
+}
+
+/**
+ * Regex pattern to match HTTP(S) URLs in text.
+ * Must match the raw URL as it appears in text, including trailing chars.
+ */
+const URL_PATTERN = /https?:\/\/[^\s<>"]+/gi;
+
+/**
+ * Normalize a raw URL match by removing trailing punctuation and validating.
+ * Returns null if the normalized URL is invalid.
+ *
+ * @param rawUrl - The raw URL as matched from text
+ * @returns Object with rawUrl (as matched) and normalizedUrl, or null if invalid
+ */
+function normalizeImageUrl(rawUrl: string): { rawUrl: string; normalizedUrl: string } | null {
+    // Remove trailing punctuation that might have been captured (., ), ], etc.)
+    let cleanUrl = rawUrl.replace(/[).,\]]+$/, "");
+
+    try {
+        // Validate and normalize via URL constructor
+        const normalized = new URL(cleanUrl).href;
+        if (isImageUrl(normalized)) {
+            return { rawUrl: cleanUrl, normalizedUrl: normalized };
+        }
+    } catch {
+        // Invalid URL after cleanup
+    }
+
+    return null;
+}
+
+/**
+ * Extract image URLs from tool result parts.
+ * Returns normalized, deduplicated URLs in order of first occurrence.
+ *
+ * @param toolData - Array of tool result parts
+ * @returns Array of unique normalized image URL strings
+ */
+export function extractImageUrlsFromToolResult(toolData: ToolResultPart[]): string[] {
+    const seen = new Set<string>();
+    const imageUrls: string[] = [];
+
+    for (const part of toolData) {
+        const text = extractTextFromOutput(part.output);
+        if (!text) continue;
+
+        const matches = text.matchAll(new RegExp(URL_PATTERN.source, "gi"));
+        for (const match of matches) {
+            const result = normalizeImageUrl(match[0]);
+            if (result && !seen.has(result.normalizedUrl)) {
+                seen.add(result.normalizedUrl);
+                imageUrls.push(result.normalizedUrl);
+            }
+        }
+    }
+
+    return imageUrls;
+}
+
+/**
+ * Count the number of images in a tool result.
+ *
+ * @param toolData - Array of tool result parts
+ * @returns Number of images found
+ */
+export function countImagesInToolResult(toolData: ToolResultPart[]): number {
+    return extractImageUrlsFromToolResult(toolData).length;
+}
+
+/**
+ * Result from processing tool results with image tracking
+ */
+export interface ProcessToolResultOutput {
+    /** Processed tool result parts with placeholders applied */
+    processedParts: ToolResultPart[];
+    /** Number of individual URL occurrences replaced with placeholders */
+    replacedCount: number;
+    /** Number of unique URLs that were replaced (each may appear multiple times) */
+    uniqueReplacedCount: number;
+}
+
+/**
+ * Process a tool result with image tracking.
+ *
+ * - First-seen images: Preserved as-is, URL is marked as seen
+ * - Previously-seen images: URL replaced with placeholder text
+ *
+ * Uses a match-driven approach: finds all URL occurrences in text via regex,
+ * normalizes each match, and replaces ALL occurrences of seen URLs.
+ *
+ * @param toolData - Array of tool result parts
+ * @param tracker - Image tracker for the conversation
+ * @param eventId - Event ID for retrieval reference
+ * @returns Object with processed tool results and replacement statistics
+ */
+export function processToolResultWithImageTracking(
+    toolData: ToolResultPart[],
+    tracker: ImageTracker,
+    eventId: string | undefined
+): ProcessToolResultOutput {
+    let totalReplacedCount = 0;
+    const uniqueReplacedUrls = new Set<string>();
+
+    const processedParts = toolData.map((part) => {
+        const text = extractTextFromOutput(part.output);
+        if (!text) {
+            // Non-text output (e.g., JSON), pass through unchanged
+            return part;
+        }
+
+        // Use regex replace callback for match-driven processing
+        // This ensures we find and process every URL occurrence in the text
+        const processedText = text.replace(
+            new RegExp(URL_PATTERN.source, "gi"),
+            (rawMatch) => {
+                const result = normalizeImageUrl(rawMatch);
+                if (!result) {
+                    // Not a valid image URL, keep as-is
+                    return rawMatch;
+                }
+
+                const { rawUrl, normalizedUrl } = result;
+
+                if (tracker.hasBeenSeen(normalizedUrl)) {
+                    // Image already seen - replace with placeholder
+                    totalReplacedCount++;
+                    uniqueReplacedUrls.add(normalizedUrl);
+                    return createImagePlaceholder(normalizedUrl, eventId);
+                } else {
+                    // First time seeing this image - mark as seen but preserve URL
+                    tracker.markAsSeen(normalizedUrl);
+                    return rawUrl;
+                }
+            }
+        );
+
+        // Return modified part with updated text
+        return {
+            ...part,
+            output: updateOutputText(part.output, processedText),
+        };
+    });
+
+    return {
+        processedParts: processedParts as ToolResultPart[],
+        replacedCount: totalReplacedCount,
+        uniqueReplacedCount: uniqueReplacedUrls.size,
+    };
+}
+
+/**
+ * Extract text from a tool result output (various formats)
+ */
+function extractTextFromOutput(output: unknown): string | null {
+    if (typeof output === "string") {
+        return output;
+    }
+
+    if (output && typeof output === "object" && "value" in output) {
+        const value = (output as { value: unknown }).value;
+        if (typeof value === "string") {
+            return value;
+        }
+    }
+
+    return null;
+}
+
+/**
+ * Update the text in a tool result output, preserving the output structure
+ */
+function updateOutputText(output: unknown, newText: string): unknown {
+    if (typeof output === "string") {
+        return newText;
+    }
+
+    if (output && typeof output === "object" && "value" in output) {
+        const existingOutput = output as { type?: string; value: unknown };
+        return {
+            ...existingOutput,
+            value: newText,
+        };
+    }
+
+    // Unknown format - return as-is
+    return output;
+}

package/src/conversations/utils/image-url-utils.ts

@@ -0,0 +1,171 @@
+/**
+ * Utilities for detecting and extracting image URLs from message content.
+ *
+ * These utilities help convert text messages containing image URLs into
+ * multimodal messages compatible with the AI SDK's ImagePart format.
+ */
+
+/**
+ * Supported image file extensions (lowercase, with leading dot)
+ */
+export const IMAGE_EXTENSIONS = [
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".gif",
+    ".webp",
+    ".svg",
+] as const;
+
+/**
+ * Regex pattern to match HTTP(S) URLs
+ * Captures URLs that start with http:// or https://
+ */
+const URL_PATTERN = /https?:\/\/[^\s<>"\]()]+/gi;
+
+/**
+ * Domains that should be skipped for image fetching.
+ * These are either reserved example domains (RFC 2606), localhost/development domains,
+ * or other non-routable domains that will fail to fetch.
+ */
+const SKIP_DOMAINS = new Set([
+    // RFC 2606 reserved example domains (only .com, .net, .org are reserved)
+    "example.com",
+    "example.org",
+    "example.net",
+    // Localhost variants
+    "localhost",
+    "0.0.0.0",
+    // IPv6 loopback - URL.hostname returns "[::1]" with brackets in Bun/Node
+    "[::1]",
+    // Common development/test domains
+    "test",
+    "invalid",
+    "local",
+]);
+
+/**
+ * Check if a hostname is a loopback address (127.0.0.0/8 range).
+ * This covers all 127.x.x.x addresses, not just 127.0.0.1.
+ *
+ * @param hostname - The hostname to check
+ * @returns true if the hostname is a loopback address
+ */
+function isLoopbackAddress(hostname: string): boolean {
+    // Match any 127.x.x.x address (the entire 127.0.0.0/8 block is reserved for loopback)
+    return /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
+}
+
+/**
+ * Check if a URL should be skipped for image fetching.
+ * Returns true for URLs on reserved/example/localhost domains that
+ * will fail to fetch and could crash the agent.
+ *
+ * @param url - The URL string to check
+ * @returns true if the URL should be skipped, false if it can be fetched
+ */
+export function shouldSkipImageUrl(url: string): boolean {
+    if (!url) return true;
+
+    try {
+        const parsedUrl = new URL(url);
+        const hostname = parsedUrl.hostname.toLowerCase();
+
+        // Check exact match against skip domains
+        if (SKIP_DOMAINS.has(hostname)) {
+            return true;
+        }
+
+        // Check for any 127.x.x.x loopback address (entire /8 block)
+        if (isLoopbackAddress(hostname)) {
+            return true;
+        }
+
+        // Check for subdomains of example domains (e.g., cdn.example.com, www.example.org)
+        for (const skipDomain of SKIP_DOMAINS) {
+            if (hostname.endsWith(`.${skipDomain}`)) {
+                return true;
+            }
+        }
+
+        // Check for .local, .test, .invalid, .localhost TLDs (RFC 2606 / RFC 6761)
+        if (
+            hostname.endsWith(".local") ||
+            hostname.endsWith(".test") ||
+            hostname.endsWith(".invalid") ||
+            hostname.endsWith(".localhost") ||
+            hostname.endsWith(".example")
+        ) {
+            return true;
+        }
+
+        return false;
+    } catch {
+        // Invalid URL - skip it
+        return true;
+    }
+}
+
+/**
+ * Check if a URL string points to an image based on its file extension.
+ * Handles URLs with query parameters and fragments.
+ *
+ * @param url - The URL string to check
+ * @returns true if the URL appears to be an image, false otherwise
+ */
+export function isImageUrl(url: string): boolean {
+    if (!url) return false;
+
+    try {
+        const parsedUrl = new URL(url);
+        // Only accept http/https protocols
+        if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
+            return false;
+        }
+
+        // Get the pathname and check the extension (ignoring query params and fragments)
+        const pathname = parsedUrl.pathname.toLowerCase();
+
+        return IMAGE_EXTENSIONS.some((ext) => pathname.endsWith(ext));
+    } catch {
+        // Invalid URL
+        return false;
+    }
+}
+
+/**
+ * Extract all image URLs from a text string.
+ * Returns deduplicated URLs in order of first occurrence.
+ *
+ * @param text - The text content to search for image URLs
+ * @returns Array of unique image URL strings
+ */
+export function extractImageUrls(text: string): string[] {
+    if (!text) return [];
+
+    // Find all URLs in the text
+    const allUrls = text.match(URL_PATTERN) || [];
+
+    // Filter to only image URLs and deduplicate
+    const seen = new Set<string>();
+    const imageUrls: string[] = [];
+
+    for (const url of allUrls) {
+        // Clean up URL (remove trailing punctuation that might have been captured)
+        let cleanUrl = url.replace(/[),.]+$/, "");
+
+        try {
+            // Validate and normalize
+            cleanUrl = new URL(cleanUrl).href;
+        } catch {
+            continue; // Skip invalid URLs encountered during cleanup
+        }
+
+        if (isImageUrl(cleanUrl) && !seen.has(cleanUrl)) {
+            seen.add(cleanUrl);
+            imageUrls.push(cleanUrl);
+        }
+    }
+
+    return imageUrls;
+}

package/src/conversations/utils/multimodal-content.ts

@@ -0,0 +1,90 @@
+/**
+ * Utilities for converting text content with image URLs to AI SDK multimodal format.
+ *
+ * The AI SDK supports multimodal user messages with content arrays containing
+ * TextPart and ImagePart objects. This module converts plain text messages
+ * containing image URLs into the appropriate multimodal format.
+ *
+ * @see https://ai-sdk.dev/docs/foundations/prompts#multi-modal-messages
+ */
+
+import type { TextPart, ImagePart } from "ai";
+import { extractImageUrls, shouldSkipImageUrl } from "./image-url-utils";
+
+/**
+ * Type representing multimodal content for user messages.
+ * Can be a simple string (no images) or an array of text/image parts.
+ */
+export type MultimodalContent = string | Array<TextPart | ImagePart>;
+
+/**
+ * Check if text content contains any image URLs.
+ * Use this to determine if multimodal conversion is needed.
+ *
+ * @param content - The text content to check
+ * @returns true if the content contains image URLs
+ */
+export function hasImageUrls(content: string): boolean {
+    return extractImageUrls(content).length > 0;
+}
+
+/**
+ * Convert text content to multimodal content if it contains image URLs.
+ *
+ * If the content contains image URLs:
+ * - Returns an array with:
+ *   1. A TextPart containing the full original text (preserves context)
+ *   2. ImageParts for each unique image URL (using URL objects for AI SDK to fetch)
+ *
+ * If no image URLs are found:
+ * - Returns the original string unchanged
+ *
+ * @param content - The text content to convert
+ * @returns Either the original string or a multimodal content array
+ */
+export function convertToMultimodalContent(content: string): MultimodalContent {
+    if (!content) return content;
+
+    const imageUrls = extractImageUrls(content);
+
+    // No images - return as-is
+    if (imageUrls.length === 0) {
+        return content;
+    }
+
+    // Build multimodal content array
+    const parts: Array<TextPart | ImagePart> = [];
+
+    // Add the full text content first (preserves context and attribution)
+    parts.push({
+        type: "text",
+        text: content,
+    } satisfies TextPart);
+
+    // Add each image as an ImagePart with URL reference
+    // The AI SDK will fetch the images automatically
+    // Skip URLs on non-routable domains (example.com, localhost, etc.) that would fail
+    for (const imageUrl of imageUrls) {
+        // Skip URLs that cannot be fetched (example domains, localhost, etc.)
+        // These would cause the AI SDK fetch to fail and crash the agent
+        if (shouldSkipImageUrl(imageUrl)) {
+            continue;
+        }
+
+        // Note: new URL() here should never throw because extractImageUrls
+        // already validates and normalizes URLs. The ImagePart uses URL objects
+        // as per AI SDK spec, allowing the SDK to fetch the images.
+        parts.push({
+            type: "image",
+            image: new URL(imageUrl),
+        } satisfies ImagePart);
+    }
+
+    // If all image URLs were skipped (non-fetchable domains), return original string
+    // parts[0] is always the TextPart, so if length is 1, no images were added
+    if (parts.length === 1) {
+        return content;
+    }
+
+    return parts;
+}

package/src/conversations/utils/tool-result-truncator.ts

@@ -0,0 +1,159 @@
+/**
+ * ToolResultTruncator - Truncates large/old tool results to save context
+ *
+ * Rules:
+ * - Tool results < 1k chars: never truncated
+ * - Tool results > 10k chars: include inline for 3 messages, then truncate
+ * - Tool results 1k-10k chars: include inline for 6 messages, then truncate
+ *
+ * Truncated results are replaced with a reference that allows re-reading via
+ * the event ID using fs_read(tool=<eventId>).
+ *
+ * EMERGENCY CONTENT GUARD: If truncation criteria are met but no eventId is
+ * available, content is STILL omitted to protect the context window. This
+ * prioritizes context window safety over content retrieval.
+ */
+
+import type { ToolResultPart } from "ai";
+
+// Thresholds for truncation
+const NEVER_TRUNCATE_THRESHOLD = 1000; // 1k chars - never truncate below this
+const LARGE_RESULT_THRESHOLD = 10000; // 10k chars
+const LARGE_RESULT_BURIAL_LIMIT = 3; // Messages before truncation for large results
+const SMALL_RESULT_BURIAL_LIMIT = 6; // Messages before truncation for small results
+
+export interface TruncationContext {
+    /** Current message index being processed */
+    currentIndex: number;
+    /** Total number of messages in the conversation */
+    totalMessages: number;
+    /** Event ID for referencing the tool result (required for truncation) */
+    eventId?: string;
+}
+
+/**
+ * Calculate the original size of a tool result
+ */
+function getToolResultSize(toolData: ToolResultPart[]): number {
+    let totalSize = 0;
+    for (const part of toolData) {
+        const output = part.output as unknown;
+        if (output) {
+            if (typeof output === "string") {
+                totalSize += output.length;
+            } else if (typeof output === "object" && output !== null && "value" in output) {
+                const value = (output as { value: unknown }).value;
+                if (typeof value === "string") {
+                    totalSize += value.length;
+                } else {
+                    try {
+                        totalSize += (JSON.stringify(value) ?? "").length;
+                    } catch {
+                        // Fallback for non-serializable values (BigInt, circular refs, etc.)
+                        // Bias toward truncation when we can't measure - safer for context window
+                        totalSize += LARGE_RESULT_THRESHOLD;
+                    }
+                }
+            } else {
+                try {
+                    totalSize += (JSON.stringify(output) ?? "").length;
+                } catch {
+                    // Fallback for non-serializable values (BigInt, circular refs, etc.)
+                    // Bias toward truncation when we can't measure - safer for context window
+                    totalSize += LARGE_RESULT_THRESHOLD;
+                }
+            }
+        }
+    }
+    return totalSize;
+}
+
+/**
+ * Check if a tool result should be truncated based on its size and burial depth.
+ * EMERGENCY CONTENT GUARD: Truncation decision is independent of eventId availability.
+ * Context window safety takes priority over content retrieval.
+ */
+export function shouldTruncateToolResult(
+    toolData: ToolResultPart[],
+    context: TruncationContext
+): boolean {
+    const size = getToolResultSize(toolData);
+
+    // Never truncate small results
+    if (size < NEVER_TRUNCATE_THRESHOLD) {
+        return false;
+    }
+
+    // NOTE: We no longer check for eventId here. Truncation decision is purely
+    // based on size and burial depth. The processToolResult function handles
+    // the case where eventId is missing by providing a placeholder message.
+
+    const burialDepth = context.totalMessages - context.currentIndex - 1;
+
+    if (size > LARGE_RESULT_THRESHOLD) {
+        return burialDepth >= LARGE_RESULT_BURIAL_LIMIT;
+    }
+
+    return burialDepth >= SMALL_RESULT_BURIAL_LIMIT;
+}
+
+/**
+ * Create a truncated tool result that references the original by event ID
+ */
+export function createTruncatedToolResult(
+    toolData: ToolResultPart[],
+    eventId: string
+): ToolResultPart[] {
+    const size = getToolResultSize(toolData);
+
+    return toolData.map((part) => ({
+        type: "tool-result" as const,
+        toolCallId: part.toolCallId,
+        toolName: part.toolName,
+        output: {
+            type: "text" as const,
+            value: `[Tool executed, ${size} chars output truncated. Use fs_read(tool="${eventId}") to retrieve full output if needed]`,
+        },
+    }));
+}
+
+/**
+ * Create an omitted tool result when no eventId is available for retrieval.
+ * EMERGENCY CONTENT GUARD: This protects the context window from overflow
+ * when truncation criteria are met but content cannot be retrieved later.
+ */
+export function createOmittedToolResult(toolData: ToolResultPart[]): ToolResultPart[] {
+    const size = getToolResultSize(toolData);
+
+    return toolData.map((part) => ({
+        type: "tool-result" as const,
+        toolCallId: part.toolCallId,
+        toolName: part.toolName,
+        output: {
+            type: "text" as const,
+            value: `[Tool output omitted to save context (${size} chars) - no reference available for retrieval]`,
+        },
+    }));
+}
+
+/**
+ * Process tool result, potentially truncating if buried too deep.
+ * EMERGENCY CONTENT GUARD: If truncation is needed but no eventId is available,
+ * content is omitted with a placeholder to protect the context window.
+ */
+export function processToolResult(
+    toolData: ToolResultPart[],
+    context: TruncationContext
+): ToolResultPart[] {
+    if (shouldTruncateToolResult(toolData, context)) {
+        if (context.eventId) {
+            // Normal case: truncate with retrieval reference
+            return createTruncatedToolResult(toolData, context.eventId);
+        } else {
+            // EMERGENCY CONTENT GUARD: No eventId available, but we still
+            // must protect the context window. Omit content with placeholder.
+            return createOmittedToolResult(toolData);
+        }
+    }
+    return toolData;
+}