@compilr-dev/agents 0.5.2 → 0.5.3

package/dist/agent.d.ts

@@ -1,7 +1,7 @@
 /**
  * Agent - The main class for running AI agents with tool use
  */
-import type { LLMProvider, Message, ChatOptions, StreamChunk } from './providers/types.js';
+import type { LLMProvider, Message, ChatOptions, StreamChunk, ContentBlock } from './providers/types.js';
 import type { Tool, ToolDefinition, ToolRegistry, ToolExecutionResult, ToolExecutionContext } from './tools/types.js';
 import type { ContextStats, VerbosityLevel, SmartCompactionResult } from './context/types.js';
 import type { AgentState, Checkpointer, SessionMetadata } from './state/types.js';
@@ -1650,7 +1650,7 @@ export declare class Agent {
     /**
      * Run the agent with a user message
      */
-    run(userMessage: string, options?: RunOptions): Promise<AgentRunResult>;
+    run(userMessage: string | ContentBlock[], options?: RunOptions): Promise<AgentRunResult>;
     /**
      * Stream the agent's response with full tool use support
      *

package/dist/agent.js

@@ -2091,18 +2091,33 @@ export class Agent {
                         const finalTokens = this.contextManager.estimateTokens(toolResultContent);
                         this.contextManager.addToCategory('toolResults', finalTokens);
                     }
+                    // Build content blocks: tool_result + optional image blocks
+                    const contentBlocks = [
+                        {
+                            type: 'tool_result',
+                            toolUseId: toolUse.id,
+                            content: toolResultContent,
+                            isError: !result.success,
+                        },
+                    ];
+                    // Inject image blocks from tool result (e.g., view_image tool)
+                    if (result.imageBlocks?.length) {
+                        for (const img of result.imageBlocks) {
+                            contentBlocks.push({
+                                type: 'image',
+                                data: img.data,
+                                mediaType: img.mediaType,
+                                filename: img.filename,
+                                width: img.width,
+                                height: img.height,
+                            });
+                        }
+                    }
                     return {
                         result,
                         toolResultMsg: {
                             role: 'user',
-                            content: [
-                                {
-                                    type: 'tool_result',
-                                    toolUseId: toolUse.id,
-                                    content: toolResultContent,
-                                    isError: !result.success,
-                                },
-                            ],
+                            content: contentBlocks,
                         },
                         skipped: false,
                         aborted: false,
@@ -2352,8 +2367,9 @@ export class Agent {
             // Context management: increment turn count and update token count
             if (this.contextManager) {
                 this.contextManager.incrementTurn();
-                // Observation masking: mask old tool results in-place before token update
+                // Observation masking: stamp new images, then mask old results + images
                 if (this.observationMasker) {
+                    this.observationMasker.stampImages(messages, this.contextManager.getTurnCount());
                     this.observationMasker.maskHistory(messages, this.contextManager.getTurnCount());
                 }
                 // Dead message pruning: prune superseded errors and permission exchanges

package/dist/context/index.d.ts

@@ -21,7 +21,7 @@ export type { ToolResultDelegatorOptions } from './tool-result-delegator.js';
 export { DEFAULT_DELEGATION_CONFIG } from './delegation-types.js';
 export type { DelegationConfig, StoredResult, DelegationEvent } from './delegation-types.js';
 export { compactToolResult } from './result-compactor.js';
-export { ObservationMasker, DEFAULT_MASK_CONFIG, DEFAULT_INPUT_COMPACTION, extractInputSummary, buildMaskText, isMasked, } from './observation-masker.js';
+export { ObservationMasker, DEFAULT_MASK_CONFIG, DEFAULT_INPUT_COMPACTION, extractInputSummary, buildMaskText, isMasked, maskImageBlock, } from './observation-masker.js';
 export type { InputCompactionRule, ObservationMaskConfig, MaskResult, ObservationMaskStats, } from './observation-masker.js';
 export { DeadMessagePruner, DEFAULT_PRUNE_CONFIG, isPruned } from './dead-message-pruner.js';
 export type { PruneConfig, PruneResult, PruneStats } from './dead-message-pruner.js';

package/dist/context/index.js

@@ -18,7 +18,7 @@ export { DEFAULT_DELEGATION_CONFIG } from './delegation-types.js';
 // Compact Tool Result Formatting (Phase 2 Token Optimization)
 export { compactToolResult } from './result-compactor.js';
 // Observation Masking (Phase 1 Token Optimization) + Tool Input Compaction (Phase 1b)
-export { ObservationMasker, DEFAULT_MASK_CONFIG, DEFAULT_INPUT_COMPACTION, extractInputSummary, buildMaskText, isMasked, } from './observation-masker.js';
+export { ObservationMasker, DEFAULT_MASK_CONFIG, DEFAULT_INPUT_COMPACTION, extractInputSummary, buildMaskText, isMasked, maskImageBlock, } from './observation-masker.js';
 // Dead Message Pruning (Phase 4 Token Optimization)
 export { DeadMessagePruner, DEFAULT_PRUNE_CONFIG, isPruned } from './dead-message-pruner.js';
 // Smart Windowing (Programmatic Context Compaction)

package/dist/context/observation-masker.d.ts

@@ -8,7 +8,7 @@
  * Strategy: In-place masking of conversationHistory after N turns.
  * The agent can re-read from the environment if needed (files, git, etc.).
  */
-import type { Message } from '../providers/types.js';
+import type { Message, ImageBlock, TextBlock } from '../providers/types.js';
 /**
  * Defines which input fields to keep when compacting a tool_use input.
  * All other fields are removed.
@@ -62,6 +62,8 @@ export declare class ObservationMasker {
     private readonly stamps;
     private readonly config;
     private stats;
+    /** Tracks which image blocks have been stamped (by identity) to avoid re-stamping */
+    private readonly stampedImages;
     constructor(config?: Partial<ObservationMaskConfig>);
     /**
      * Register a tool result with its turn number and input context.
@@ -69,8 +71,16 @@ export declare class ObservationMasker {
      */
     stamp(toolUseId: string, toolName: string, input: Record<string, unknown>, contentLength: number, turn: number): void;
     /**
-     * Mask old tool results and compact old tool_use inputs in-place.
+     * Stamp all image blocks in a message array with the current turn.
+     * Call this after adding user messages that may contain images.
+     */
+    stampImages(messages: Message[], turn: number): void;
+    /** Turn at which each image block was first seen */
+    private readonly imageStamps;
+    /**
+     * Mask old tool results, images, and compact old tool_use inputs in-place.
      * - tool_result: replaces content with compact mask text (Phase 1)
+     * - image: replaces with text placeholder after maskAfterTurns (Phase 2)
      * - tool_use input: strips large fields, keeping only identifying fields (Phase 1b)
      */
     maskHistory(messages: Message[], currentTurn: number): MaskResult;
@@ -101,4 +111,9 @@ export declare function buildMaskText(stamp: TurnStamp): string;
  * Check if a tool result content string is already masked.
  */
 export declare function isMasked(content: string): boolean;
+/**
+ * Replace an image content block with a text placeholder.
+ * Preserves filename and dimensions for context.
+ */
+export declare function maskImageBlock(block: ImageBlock, turn: number): TextBlock;
 export {};

package/dist/context/observation-masker.js

@@ -26,6 +26,8 @@ export class ObservationMasker {
     stamps = new Map();
     config;
     stats = { maskedCount: 0, tokensSaved: 0, inputsCompacted: 0 };
+    /** Tracks which image blocks have been stamped (by identity) to avoid re-stamping */
+    stampedImages = new WeakSet();
     constructor(config) {
         this.config = { ...DEFAULT_MASK_CONFIG, ...config };
     }
@@ -45,11 +47,33 @@ export class ObservationMasker {
         });
     }
     // ----------------------------------------------------------
+    // Image stamping — called when messages with images are added
+    // ----------------------------------------------------------
+    /**
+     * Stamp all image blocks in a message array with the current turn.
+     * Call this after adding user messages that may contain images.
+     */
+    stampImages(messages, turn) {
+        for (const msg of messages) {
+            if (typeof msg.content === 'string')
+                continue;
+            for (const block of msg.content) {
+                if (block.type === 'image' && !this.stampedImages.has(block)) {
+                    this.imageStamps.set(block, turn);
+                    this.stampedImages.add(block);
+                }
+            }
+        }
+    }
+    /** Turn at which each image block was first seen */
+    imageStamps = new WeakMap();
+    // ----------------------------------------------------------
     // Masking — called after incrementTurn()
     // ----------------------------------------------------------
     /**
-     * Mask old tool results and compact old tool_use inputs in-place.
+     * Mask old tool results, images, and compact old tool_use inputs in-place.
      * - tool_result: replaces content with compact mask text (Phase 1)
+     * - image: replaces with text placeholder after maskAfterTurns (Phase 2)
      * - tool_use input: strips large fields, keeping only identifying fields (Phase 1b)
      */
     maskHistory(messages, currentTurn) {
@@ -90,6 +114,34 @@ export class ObservationMasker {
                     this.stamps.delete(block.toolUseId);
                 }
             }
+            // Phase 2: Replace old image blocks with text placeholders
+            const contentArr = msg.content;
+            for (let i = 0; i < contentArr.length; i++) {
+                const block = contentArr[i];
+                if (block.type !== 'image')
+                    continue;
+                // Stamp if not already stamped (images added before stampImages existed)
+                if (!this.stampedImages.has(block)) {
+                    this.imageStamps.set(block, currentTurn);
+                    this.stampedImages.add(block);
+                    continue; // Don't mask on the same turn we stamp
+                }
+                const imageTurn = this.imageStamps.get(block);
+                if (imageTurn === undefined)
+                    continue;
+                const age = currentTurn - imageTurn;
+                if (age < this.config.maskAfterTurns)
+                    continue;
+                // Replace image block with text placeholder
+                const placeholder = maskImageBlock(block, imageTurn);
+                contentArr[i] = placeholder;
+                // Estimate tokens saved: base64 image data is ~4 chars per 3 bytes
+                // A typical image is 1000-5000 tokens; the placeholder is ~20 tokens
+                const imageTokens = Math.ceil(block.data.length / 4);
+                const savedTokens = Math.max(0, imageTokens - 20);
+                tokensSaved += savedTokens;
+                maskedCount++;
+            }
             // Phase 1b: Compact old tool_use inputs in assistant messages
             if (msg.role === 'assistant') {
                 for (const block of msg.content) {
@@ -252,3 +304,15 @@ export function buildMaskText(stamp) {
 export function isMasked(content) {
     return content.startsWith('[') && content.endsWith(']') && content.includes('@turn:');
 }
+/**
+ * Replace an image content block with a text placeholder.
+ * Preserves filename and dimensions for context.
+ */
+export function maskImageBlock(block, turn) {
+    const name = block.filename ?? 'image';
+    const dims = block.width && block.height ? `, ${String(block.width)}x${String(block.height)}` : '';
+    return {
+        type: 'text',
+        text: `[Image: ${name}${dims}, sent@turn:${String(turn)}]`,
+    };
+}

package/dist/index.d.ts

@@ -39,7 +39,7 @@ export type { ToolPairingValidation } from './messages/index.js';
 export { generateId, sleep, retry, truncate, withRetryGenerator, calculateBackoffDelay, DEFAULT_RETRY_CONFIG, countTokens, countMessageTokens, } from './utils/index.js';
 export type { RetryConfig as LLMRetryConfig, WithRetryOptions } from './utils/index.js';
 export { AgentError, ProviderError, ToolError, ToolTimeoutError, ToolLoopError, ValidationError, MaxIterationsError, AbortError, ContextOverflowError, isAgentError, isProviderError, isToolError, isToolTimeoutError, isToolLoopError, isContextOverflowError, wrapError, } from './errors.js';
-export { ContextManager, DEFAULT_CONTEXT_CONFIG, FileAccessTracker, createFileTrackingHook, TRACKED_TOOLS, DelegatedResultStore, ToolResultDelegator, DELEGATION_SYSTEM_PROMPT, DEFAULT_DELEGATION_CONFIG, compactToolResult, ObservationMasker, DEFAULT_MASK_CONFIG, DEFAULT_INPUT_COMPACTION, extractInputSummary, buildMaskText, isMasked, DeadMessagePruner, DEFAULT_PRUNE_CONFIG, isPruned, } from './context/index.js';
+export { ContextManager, DEFAULT_CONTEXT_CONFIG, FileAccessTracker, createFileTrackingHook, TRACKED_TOOLS, DelegatedResultStore, ToolResultDelegator, DELEGATION_SYSTEM_PROMPT, DEFAULT_DELEGATION_CONFIG, compactToolResult, ObservationMasker, DEFAULT_MASK_CONFIG, DEFAULT_INPUT_COMPACTION, extractInputSummary, buildMaskText, isMasked, maskImageBlock, DeadMessagePruner, DEFAULT_PRUNE_CONFIG, isPruned, } from './context/index.js';
 export type { ContextManagerOptions, ContextCategory, BudgetAllocation, CategoryBudgetInfo, PreflightResult, VerbosityLevel, VerbosityConfig, ContextConfig, FilteringConfig, CompactionConfig, SummarizationConfig, CompactionResult, SummarizationResult, FilteringResult, ContextEvent, ContextEventHandler, ContextStats, FileAccessType, FileAccess, FileAccessTrackerOptions, FormatHintsOptions, FileAccessStats, RestorationHintMessage, DelegatedResultStoreStats, ToolResultDelegatorOptions, DelegationConfig, StoredResult, DelegationEvent, InputCompactionRule, ObservationMaskConfig, MaskResult, ObservationMaskStats, PruneConfig, PruneResult, PruneStats, WindowingConfig, WindowingResult, ImportanceLevel, } from './context/index.js';
 export { SkillRegistry, defineSkill, createSkillRegistry, builtinSkills, getDefaultSkillRegistry, resetDefaultSkillRegistry, } from './skills/index.js';
 export type { Skill, SkillInvocationResult, SkillInvokeOptions } from './skills/index.js';

package/dist/index.js

@@ -51,7 +51,7 @@ DelegatedResultStore, ToolResultDelegator, DELEGATION_SYSTEM_PROMPT, DEFAULT_DEL
 // Compact tool result formatting (Phase 2 Token Optimization)
 compactToolResult, 
 // Observation masking (Phase 1 Token Optimization) + Tool Input Compaction (Phase 1b)
-ObservationMasker, DEFAULT_MASK_CONFIG, DEFAULT_INPUT_COMPACTION, extractInputSummary, buildMaskText, isMasked, 
+ObservationMasker, DEFAULT_MASK_CONFIG, DEFAULT_INPUT_COMPACTION, extractInputSummary, buildMaskText, isMasked, maskImageBlock, 
 // Dead message pruning (Phase 4 Token Optimization)
 DeadMessagePruner, DEFAULT_PRUNE_CONFIG, isPruned, } from './context/index.js';
 // Skills system

package/dist/providers/claude.js

@@ -228,6 +228,15 @@ export class ClaudeProvider {
                     // Thinking blocks are passed through as text for now
                     // The API expects thinking in a specific format during beta
                     return { type: 'text', text: `<thinking>${block.thinking}</thinking>` };
+                case 'image':
+                    return {
+                        type: 'image',
+                        source: {
+                            type: 'base64',
+                            media_type: block.mediaType,
+                            data: block.data,
+                        },
+                    };
                 default: {
                     // Exhaustive check - this should never happen
                     const _exhaustive = block;

package/dist/providers/gemini-native.js

@@ -232,6 +232,15 @@ export class GeminiNativeProvider {
                     // They are internal model reasoning. Only the signature on function calls matters.
                     // Skip - do not add to parts.
                     break;
+                case 'image':
+                    // Convert to Gemini's inlineData format
+                    parts.push({
+                        inlineData: {
+                            mimeType: block.mediaType,
+                            data: block.data,
+                        },
+                    });
+                    break;
                 default: {
                     // Exhaustive check
                     const _exhaustive = block;

package/dist/providers/openai-compatible.js

@@ -221,12 +221,20 @@ export class OpenAICompatibleProvider {
             else if (Array.isArray(msg.content)) {
                 // Handle content blocks
                 const blocks = msg.content;
-                const textParts = [];
+                const contentParts = [];
                 const toolCallsList = [];
                 const toolResults = [];
+                let hasImages = false;
                 for (const block of blocks) {
                     if (block.type === 'text') {
-                        textParts.push(block.text);
+                        contentParts.push({ type: 'text', text: block.text });
+                    }
+                    else if (block.type === 'image') {
+                        contentParts.push({
+                            type: 'image_url',
+                            image_url: { url: `data:${block.mediaType};base64,${block.data}` },
+                        });
+                        hasImages = true;
                     }
                     else if (block.type === 'tool_use') {
                         toolCallsList.push({
@@ -247,6 +255,7 @@ export class OpenAICompatibleProvider {
                     }
                     // Note: 'thinking' blocks are ignored (Claude-specific)
                 }
+                const textParts = contentParts.filter((p) => p.type === 'text').map((p) => p.text ?? '');
                 // Handle tool results - each needs its own message
                 if (toolResults.length > 0) {
                     for (const tr of toolResults) {
@@ -265,6 +274,13 @@ export class OpenAICompatibleProvider {
                         tool_calls: toolCallsList,
                     });
                 }
+                else if (hasImages) {
+                    // Message with images — send as content parts array
+                    result.push({
+                        role: this.mapRole(msg.role),
+                        content: contentParts,
+                    });
+                }
                 else if (textParts.length > 0) {
                     // Regular text message
                     result.push({

package/dist/providers/types.d.ts

@@ -51,10 +51,26 @@ export interface ThinkingBlock {
      */
     signature?: string;
 }
+/**
+ * Image content block (user-attached or tool-provided image for vision)
+ */
+export interface ImageBlock {
+    type: 'image';
+    /** Base64-encoded image data */
+    data: string;
+    /** MIME type: image/png, image/jpeg, image/webp, image/gif */
+    mediaType: string;
+    /** Original filename (for display and observation masking placeholder) */
+    filename?: string;
+    /** Image width in pixels */
+    width?: number;
+    /** Image height in pixels */
+    height?: number;
+}
 /**
  * Union of all content block types
  */
-export type ContentBlock = TextBlock | ToolUseBlock | ToolResultBlock | ThinkingBlock;
+export type ContentBlock = TextBlock | ToolUseBlock | ToolResultBlock | ThinkingBlock | ImageBlock;
 /**
  * A message in a conversation
  */

package/dist/tools/types.d.ts

@@ -24,6 +24,19 @@ export interface ToolExecutionResult {
     success: boolean;
     result?: unknown;
     error?: string;
+    /**
+     * Optional image blocks to inject alongside the tool result.
+     * When present, these are added as sibling content blocks in the
+     * tool result message, enabling vision-capable LLMs to see images.
+     * Used by tools like view_image that return visual content.
+     */
+    imageBlocks?: Array<{
+        data: string;
+        mediaType: string;
+        filename?: string;
+        width?: number;
+        height?: number;
+    }>;
 }
 /**
  * Context passed to tool execution for streaming output

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@compilr-dev/agents",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "description": "Lightweight multi-LLM agent library for building CLI AI assistants",
   "type": "module",
   "main": "dist/index.js",