@alejandroroman/agent-kit 0.1.4 → 0.2.0

package/dist/gateways/slack/handler.js

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { extractUserText } from "../../llm/types.js";
 import { markdownToBlocks, markdownToPlainText } from "./format.js";
 import { BudgetStatusSchema } from "./types.js";
 import { createLogger } from "../../logger.js";
@@ -16,7 +17,7 @@ function findBudgetToolResults(messages) {
     const results = [];
     for (const msg of messages) {
         if (msg.role === "tool_result" && toolCallNames.get(msg.tool_call_id) === "get_budget_status") {
-            results.push(msg.content);
+            results.push(extractUserText(msg.content));
         }
     }
     return results;

package/dist/gateways/slack/index.js

@@ -4,7 +4,25 @@ import { createSlackListener } from "./listener.js";
 import { createThreadSessionManager } from "./sessions.js";
 import { markdownToBlocks, markdownToPlainText } from "./format.js";
 import { createLogger } from "../../logger.js";
+import { context, trace, SpanStatusCode } from "@opentelemetry/api";
+import { getTracer, ATTR } from "../../telemetry/index.js";
 const log = createLogger("slack");
+function buildUserMessage(msg) {
+    if (!msg.files?.length) {
+        return { role: "user", content: msg.userText };
+    }
+    const blocks = [];
+    if (msg.userText) {
+        blocks.push({ type: "text", text: msg.userText });
+    }
+    for (const file of msg.files) {
+        blocks.push({
+            type: "image",
+            source: { type: "base64", media_type: file.mimetype, data: file.base64 },
+        });
+    }
+    return { role: "user", content: blocks };
+}
 export function createSlackGateway(config, options) {
     const botToken = options?.botToken ?? process.env.SLACK_BOT_TOKEN;
     const appToken = options?.appToken ?? process.env.SLACK_APP_TOKEN;
@@ -40,37 +58,65 @@ export function createSlackGateway(config, options) {
     async function processMessage(msg) {
         if (!executor)
             return;
-        const userMsg = { role: "user", content: msg.userText };
-        sessions.append(msg.threadTs, userMsg);
-        const messages = sessions.get(msg.threadTs);
-        try {
-            const start = Date.now();
-            log.info({ agent: msg.agentName, thread: msg.threadTs, history: messages.length }, "executing agent");
-            const result = await executor(msg.agentName, messages);
-            const elapsed = ((Date.now() - start) / 1000).toFixed(1);
-            log.info({ agent: msg.agentName, thread: msg.threadTs, elapsed, tokens: result.usage.inputTokens + result.usage.outputTokens }, "agent completed");
-            // Replace session with full message history from agent
-            sessions.set(msg.threadTs, result.messages);
-            const text = result.text || "(completed with no text response)";
-            const blocks = markdownToBlocks(text);
-            const fallback = markdownToPlainText(text);
-            await client.postMessage(msg.channelId, fallback, blocks, msg.threadTs);
-            log.info({ agent: msg.agentName, thread: msg.threadTs }, "reply posted");
-        }
-        catch (err) {
-            log.error({ err, agent: msg.agentName, thread: msg.threadTs }, "error handling inbound message");
-            // Roll back user message on failure to avoid malformed conversation history
-            const current = sessions.get(msg.threadTs);
-            if (current.length > 0 && current[current.length - 1].role === "user") {
-                sessions.set(msg.threadTs, current.slice(0, -1));
-            }
+        const tracer = getTracer("slack");
+        const span = tracer.startSpan("source.slack", {
+            attributes: {
+                [ATTR.CHANNEL]: msg.channelId,
+                [ATTR.AGENT]: msg.agentName,
+                [ATTR.THREAD_TS]: msg.threadTs,
+                [ATTR.SOURCE]: "slack",
+            },
+        });
+        const spanCtx = trace.setSpan(context.active(), span);
+        await context.with(spanCtx, async () => {
+            const userMsg = buildUserMessage(msg);
+            sessions.append(msg.threadTs, userMsg);
+            const messages = sessions.get(msg.threadTs);
             try {
-                await client.postMessage(msg.channelId, "Sorry, I encountered an error processing your message. Please try again in a moment.", undefined, msg.threadTs);
+                const start = Date.now();
+                log.info({ agent: msg.agentName, thread: msg.threadTs, history: messages.length }, "executing agent");
+                const result = await executor(msg.agentName, messages);
+                const elapsed = ((Date.now() - start) / 1000).toFixed(1);
+                log.info({ agent: msg.agentName, thread: msg.threadTs, elapsed, tokens: result.usage.inputTokens + result.usage.outputTokens }, "agent completed");
+                // Replace session with full message history from agent
+                sessions.set(msg.threadTs, result.messages);
+                const text = result.text || "(completed with no text response)";
+                const blocks = markdownToBlocks(text);
+                const fallback = markdownToPlainText(text);
+                const postSpan = tracer.startSpan("slack.post", {}, spanCtx);
+                try {
+                    await client.postMessage(msg.channelId, fallback, blocks, msg.threadTs);
+                    log.info({ agent: msg.agentName, thread: msg.threadTs }, "reply posted");
+                }
+                catch (postErr) {
+                    postSpan.setStatus({ code: SpanStatusCode.ERROR, message: String(postErr) });
+                    postSpan.recordException(postErr instanceof Error ? postErr : new Error(String(postErr)));
+                    throw postErr;
+                }
+                finally {
+                    postSpan.end();
+                }
             }
-            catch (postErr) {
-                log.error({ err: postErr, thread: msg.threadTs }, "failed to send error reply");
+            catch (err) {
+                span.setStatus({ code: SpanStatusCode.ERROR, message: String(err) });
+                span.recordException(err instanceof Error ? err : new Error(String(err)));
+                log.error({ err, agent: msg.agentName, thread: msg.threadTs }, "error handling inbound message");
+                // Roll back user message on failure to avoid malformed conversation history
+                const current = sessions.get(msg.threadTs);
+                if (current.length > 0 && current[current.length - 1].role === "user") {
+                    sessions.set(msg.threadTs, current.slice(0, -1));
+                }
+                try {
+                    await client.postMessage(msg.channelId, "Sorry, I encountered an error processing your message. Please try again in a moment.", undefined, msg.threadTs);
+                }
+                catch (postErr) {
+                    log.error({ err: postErr, thread: msg.threadTs }, "failed to send error reply");
+                }
             }
-        }
+            finally {
+                span.end();
+            }
+        });
     }
     // Eviction timer handle
     let evictionTimer;
@@ -78,7 +124,7 @@ export function createSlackGateway(config, options) {
         async start() {
             await client.start();
             if (executor) {
-                createSlackListener(client.app, gatewayConfig, handleInbound);
+                createSlackListener(client.app, gatewayConfig, handleInbound, client);
                 // Evict stale threads every hour
                 evictionTimer = setInterval(() => {
                     sessions.evict();

package/dist/gateways/slack/listener.d.ts

@@ -1,10 +1,17 @@
 import type { App } from "@slack/bolt";
 import type { SlackGatewayConfig } from "./types.js";
+import type { SlackClient } from "./client.js";
+export interface InboundFile {
+    mimetype: string;
+    name: string;
+    base64: string;
+}
 export interface InboundMessage {
     agentName: string;
     userText: string;
     threadTs: string;
     channelId: string;
+    files?: InboundFile[];
 }
 export type OnMessageCallback = (message: InboundMessage) => Promise<void>;
-export declare function createSlackListener(app: App, config: SlackGatewayConfig, onMessage: OnMessageCallback): void;
+export declare function createSlackListener(app: App, config: SlackGatewayConfig, onMessage: OnMessageCallback, client?: SlackClient): void;

package/dist/gateways/slack/listener.js

@@ -1,33 +1,59 @@
+import { sanitizeImage } from "../../media/sanitize.js";
 import { createLogger } from "../../logger.js";
 const log = createLogger("slack:listener");
-export function createSlackListener(app, config, onMessage) {
-    // Build reverse map: channelId → agentName
+const IMAGE_MIMES = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
+export function createSlackListener(app, config, onMessage, client) {
     const channelToAgent = new Map();
     for (const [agentName, binding] of Object.entries(config.channels)) {
         channelToAgent.set(binding.channelId, agentName);
     }
     app.event("message", async (payload) => {
-        // Handle both Bolt's { event } wrapper and direct event (tests)
         const msg = payload.event ?? payload;
         // Filter out bot messages and subtypes (edits, deletes, etc.)
-        if (msg.bot_id || msg.subtype)
+        // Allow file_share through so we can process image attachments
+        if (msg.bot_id)
             return;
-        if (!msg.text || typeof msg.text !== "string")
+        if (msg.subtype && msg.subtype !== "file_share")
+            return;
+        const hasText = msg.text && typeof msg.text === "string";
+        const hasFiles = Array.isArray(msg.files) && msg.files.length > 0;
+        if (!hasText && !hasFiles)
             return;
         const channelId = msg.channel;
         const agentName = channelToAgent.get(channelId);
         if (!agentName)
             return;
-        // Use thread_ts if in a thread, otherwise use ts as the thread parent
         const threadTs = msg.thread_ts ?? msg.ts;
-        // Fire-and-forget: don't await the handler so the Bolt event pipeline
-        // returns immediately. This prevents long-running agent loops from
-        // stalling WebSocket ping/pong and causing socket timeouts.
+        let files;
+        if (hasFiles && client) {
+            files = [];
+            for (const file of msg.files) {
+                if (!IMAGE_MIMES.has(file.mimetype)) {
+                    log.debug({ name: file.name, mimetype: file.mimetype }, "skipping non-image file");
+                    continue;
+                }
+                try {
+                    const buffer = await client.downloadFile(file.url_private_download);
+                    const sanitized = await sanitizeImage(buffer, file.mimetype);
+                    files.push({
+                        mimetype: sanitized.mediaType,
+                        name: file.name ?? "image",
+                        base64: sanitized.base64,
+                    });
+                }
+                catch (err) {
+                    log.warn({ err, name: file.name }, "failed to download/sanitize image");
+                }
+            }
+            if (files.length === 0)
+                files = undefined;
+        }
         onMessage({
             agentName,
-            userText: msg.text,
+            userText: msg.text ?? "",
             threadTs,
             channelId,
+            files,
         }).catch((err) => {
             log.error({ err, channelId }, "unhandled error processing message");
         });

package/dist/heartbeat/runner.js

@@ -5,9 +5,11 @@ import { setupAgentSession } from "../agent/setup.js";
 import { resolveAgent, resolveModelAlias, resolveWebSearch } from "../config/resolve.js";
 import { createBuiltinRegistry } from "../tools/builtin/index.js";
 import { registerSpawnWrappers } from "../tools/builtin/spawn.js";
-import { createActivateSkillTool } from "../skills/index.js";
+import { createActivateSkillTool, preActivateSkills } from "../skills/index.js";
 import { createLogger } from "../logger.js";
 import { dateContext } from "../text.js";
+import { context, trace, SpanStatusCode } from "@opentelemetry/api";
+import { getTracer, ATTR } from "../telemetry/index.js";
 const log = createLogger("heartbeat");
 const HEARTBEAT_OK = "HEARTBEAT_OK";
 export function isHeartbeatSuppressed(text) {
@@ -64,93 +66,99 @@ export class HeartbeatRunner {
         if (!agentDef?.heartbeat)
             return undefined;
         const hb = agentDef.heartbeat;
-        if (!isWithinActiveHours(hb.activeHours)) {
-            log.debug({ agent: agentName }, "outside active hours, skipping");
-            return undefined;
-        }
-        const instructions = this.loadHeartbeatInstructions(agentName);
-        if (!instructions) {
-            log.warn({ agent: agentName }, "no HEARTBEAT.md found, skipping");
-            return undefined;
-        }
-        // Isolated tool registry per tick (same pattern as CronScheduler)
-        const sandbox = agentDef.sandbox ?? this.config.defaults.sandbox;
-        const memoryConfig = this.config.defaults.memory;
-        const tickRegistry = createBuiltinRegistry({
-            allowedCommands: sandbox?.allowedCommands,
-            allowedPaths: sandbox?.allowedPaths,
-            memoryConfig,
-            webSearch: resolveWebSearch(agentName, this.config),
+        const tracer = getTracer("heartbeat");
+        const span = tracer.startSpan("source.heartbeat", {
+            attributes: {
+                [ATTR.AGENT]: agentName,
+                [ATTR.SOURCE]: "heartbeat",
+            },
         });
-        const resolved = resolveAgent(agentName, this.config, tickRegistry, this.skillsDir);
-        // Skills setup — auto-activate all skills upfront so Haiku doesn't need to call activate_skill
-        const promptFragments = [];
-        let skillsIndex = "";
-        if (resolved.skills.length > 0) {
-            const ctx = {
-                manifests: resolved.skills,
-                skillsDir: this.skillsDir,
-                toolRegistry: tickRegistry,
-                promptFragments,
-                activatedSkills: new Set(),
-            };
-            const activateTool = createActivateSkillTool(ctx);
-            tickRegistry.register(activateTool);
-            // Auto-activate all skills upfront
-            for (const manifest of resolved.skills) {
-                try {
-                    const result = await activateTool.execute({ skill_name: manifest.name });
-                    if (typeof result === "string" && result.startsWith("Error")) {
-                        log.error({ agent: agentName, skill: manifest.name, result }, "failed to auto-activate skill");
-                    }
-                }
-                catch (err) {
-                    log.error({ err, agent: agentName, skill: manifest.name }, "skill auto-activation threw");
-                }
+        const spanCtx = trace.setSpan(context.active(), span);
+        try {
+            if (!isWithinActiveHours(hb.activeHours)) {
+                log.debug({ agent: agentName }, "outside active hours, skipping");
+                span.setAttribute(ATTR.SUPPRESSED, "outside_active_hours");
+                return undefined;
             }
-            const activatedNames = [...ctx.activatedSkills];
-            if (activatedNames.length > 0) {
-                skillsIndex = "\n\nThe following skills are pre-activated: "
-                    + activatedNames.join(", ")
-                    + ".\nTheir tools are already available — you do not need to call activate_skill.";
+            const instructions = this.loadHeartbeatInstructions(agentName);
+            if (!instructions) {
+                log.warn({ agent: agentName }, "no HEARTBEAT.md found, skipping");
+                span.setAttribute(ATTR.SUPPRESSED, "no_instructions");
+                return undefined;
             }
+            return await context.with(spanCtx, async () => {
+                // Isolated tool registry per tick (same pattern as CronScheduler)
+                const sandbox = agentDef.sandbox ?? this.config.defaults.sandbox;
+                const memoryConfig = this.config.defaults.memory;
+                const tickRegistry = createBuiltinRegistry({
+                    allowedCommands: sandbox?.allowedCommands,
+                    allowedPaths: sandbox?.allowedPaths,
+                    memoryConfig,
+                    webSearch: resolveWebSearch(agentName, this.config),
+                });
+                const resolved = resolveAgent(agentName, this.config, tickRegistry, this.skillsDir);
+                // Skills setup — auto-activate all skills upfront so Haiku doesn't need to call activate_skill
+                const promptFragments = [];
+                let skillsIndex = "";
+                if (resolved.skills.length > 0) {
+                    const ctx = {
+                        manifests: resolved.skills,
+                        skillsDir: this.skillsDir,
+                        toolRegistry: tickRegistry,
+                        promptFragments,
+                        activatedSkills: new Set(),
+                    };
+                    const activateTool = createActivateSkillTool(ctx);
+                    tickRegistry.register(activateTool);
+                    // Heartbeat always auto-activates (Haiku can't reliably call activate_skill)
+                    skillsIndex = await preActivateSkills(ctx, activateTool, log);
+                }
+                // Spawn wrappers
+                if (resolved.canSpawn.length > 0) {
+                    registerSpawnWrappers(resolved.canSpawn, this.config, this.agentRegistry, tickRegistry, this.usageStore);
+                }
+                // Ephemeral session: no history accumulation, append-only audit log
+                const sessionId = `heartbeat-${agentName}`;
+                const { soul, session } = setupAgentSession(this.dataDir, agentName, sessionId);
+                // Build tick prompt with fresh instructions + timestamp
+                const tickPrompt = `[Heartbeat tick — ${new Date().toISOString()}]\n\n${instructions}`;
+                const userMsg = { role: "user", content: tickPrompt };
+                session.append(userMsg); // audit only
+                const messages = [userMsg]; // ephemeral — no history
+                // Model: heartbeat.model overrides agent model
+                const model = hb.model
+                    ? resolveModelAlias(hb.model, this.config.models)
+                    : resolved.model;
+                const systemPrompt = [soul, dateContext(), skillsIndex, ...promptFragments]
+                    .filter(Boolean)
+                    .join("\n\n") || undefined;
+                const result = await runAgentLoop(messages, {
+                    model,
+                    fallbacks: resolved.fallbacks,
+                    systemPrompt,
+                    toolRegistry: tickRegistry,
+                    maxIterations: hb.maxIterations ?? resolved.maxIterations,
+                    compactionThreshold: resolved.compactionThreshold,
+                    maxToolResultSize: resolved.maxToolResultSize,
+                    agentName,
+                    usageStore: this.usageStore,
+                    source: "heartbeat",
+                });
+                // Audit: append all new messages from the agent loop
+                for (const msg of result.messages.slice(1)) {
+                    session.append(msg);
+                }
+                return result;
+            });
         }
-        // Spawn wrappers
-        if (resolved.canSpawn.length > 0) {
-            registerSpawnWrappers(resolved.canSpawn, this.config, this.agentRegistry, tickRegistry, this.usageStore);
+        catch (err) {
+            span.setStatus({ code: SpanStatusCode.ERROR, message: String(err) });
+            span.recordException(err instanceof Error ? err : new Error(String(err)));
+            throw err;
         }
-        // Ephemeral session: no history accumulation, append-only audit log
-        const sessionId = `heartbeat-${agentName}`;
-        const { soul, session } = setupAgentSession(this.dataDir, agentName, sessionId);
-        // Build tick prompt with fresh instructions + timestamp
-        const tickPrompt = `[Heartbeat tick — ${new Date().toISOString()}]\n\n${instructions}`;
-        const userMsg = { role: "user", content: tickPrompt };
-        session.append(userMsg); // audit only
-        const messages = [userMsg]; // ephemeral — no history
-        // Model: heartbeat.model overrides agent model
-        const model = hb.model
-            ? resolveModelAlias(hb.model, this.config.models)
-            : resolved.model;
-        const systemPrompt = [soul, dateContext(), skillsIndex, ...promptFragments]
-            .filter(Boolean)
-            .join("\n\n") || undefined;
-        const result = await runAgentLoop(messages, {
-            model,
-            fallbacks: resolved.fallbacks,
-            systemPrompt,
-            toolRegistry: tickRegistry,
-            maxIterations: hb.maxIterations ?? resolved.maxIterations,
-            compactionThreshold: resolved.compactionThreshold,
-            maxToolResultSize: resolved.maxToolResultSize,
-            agentName,
-            usageStore: this.usageStore,
-            source: "heartbeat",
-        });
-        // Audit: append all new messages from the agent loop
-        for (const msg of result.messages.slice(1)) {
-            session.append(msg);
+        finally {
+            span.end();
         }
-        return result;
     }
     start(callbacks) {
         for (const [agentName, agentDef] of Object.entries(this.config.agents)) {
@@ -162,6 +170,15 @@ export class HeartbeatRunner {
             const runTick = async () => {
                 if (this.running.has(agentName)) {
                     log.info({ agent: agentName }, "previous tick still running, skipping");
+                    const tracer = getTracer("heartbeat");
+                    const skipSpan = tracer.startSpan("source.heartbeat", {
+                        attributes: {
+                            [ATTR.AGENT]: agentName,
+                            [ATTR.SOURCE]: "heartbeat",
+                            [ATTR.SUPPRESSED]: "already_running",
+                        },
+                    });
+                    skipSpan.end();
                     return;
                 }
                 this.running.add(agentName);

package/dist/llm/anthropic.d.ts

@@ -7,6 +7,7 @@ export declare class AnthropicProvider implements LLMProvider {
     complete(model: string, messages: Message[], options?: CompleteOptions): Promise<LLMResponse>;
     stream(model: string, messages: Message[], options?: CompleteOptions): AsyncIterable<StreamEvent>;
     private toAnthropicMessages;
+    private mapUserContent;
     private fromAnthropicResponse;
     private mapStopReason;
 }

package/dist/llm/anthropic.js

@@ -39,7 +39,7 @@ export class AnthropicProvider {
     toAnthropicMessages(messages) {
         return messages.map((msg) => {
             if (msg.role === "user")
-                return { role: "user", content: msg.content };
+                return { role: "user", content: this.mapUserContent(msg.content) };
             if (msg.role === "assistant") {
                 return {
                     role: "assistant",
@@ -51,11 +51,20 @@ export class AnthropicProvider {
                 };
             }
             if (msg.role === "tool_result") {
-                return { role: "user", content: [{ type: "tool_result", tool_use_id: msg.tool_call_id, content: msg.content }] };
+                return { role: "user", content: [{ type: "tool_result", tool_use_id: msg.tool_call_id, content: this.mapUserContent(msg.content) }] };
             }
             return msg;
         });
     }
+    mapUserContent(content) {
+        if (typeof content === "string")
+            return content;
+        return content.map((block) => {
+            if (block.type === "text")
+                return { type: "text", text: block.text };
+            return { type: "image", source: block.source };
+        });
+    }
     fromAnthropicResponse(response) {
         const content = response.content.map((block) => {
             if (block.type === "text")

package/dist/llm/fallback.js

@@ -1,4 +1,6 @@
 import { createLogger } from "../logger.js";
+import { getTracer, ATTR } from "../telemetry/index.js";
+import { SpanStatusCode, context } from "@opentelemetry/api";
 const log = createLogger("llm");
 const RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503]);
 export function isRetryableError(err) {
@@ -12,17 +14,47 @@ export function isRetryableError(err) {
     return false;
 }
 export async function completeWithFallback(primary, fallbacks, messages, options, completeFn) {
+    const tracer = getTracer("llm");
     const models = [primary, ...fallbacks];
     let lastError;
-    for (const model of models) {
+    for (let idx = 0; idx < models.length; idx++) {
+        const model = models[idx];
+        const [provider = "unknown"] = model.split(":");
+        const attemptSpan = tracer.startSpan("llm.attempt", {}, context.active());
+        attemptSpan.setAttribute(ATTR.PROVIDER, provider);
+        attemptSpan.setAttribute(ATTR.MODEL, model);
         try {
-            return await completeFn(model, messages, options);
+            const response = await completeFn(model, messages, options);
+            // Set token attributes on successful attempt
+            attemptSpan.setAttribute(ATTR.INPUT_TOKENS, response.usage.inputTokens);
+            attemptSpan.setAttribute(ATTR.OUTPUT_TOKENS, response.usage.outputTokens);
+            attemptSpan.setAttribute(ATTR.CACHE_CREATION_TOKENS, response.usage.cacheCreationTokens);
+            attemptSpan.setAttribute(ATTR.CACHE_READ_TOKENS, response.usage.cacheReadTokens);
+            attemptSpan.setAttribute(ATTR.STOP_REASON, response.stopReason);
+            if (response.latencyMs != null) {
+                attemptSpan.setAttribute(ATTR.LATENCY_MS, response.latencyMs);
+            }
+            attemptSpan.end();
+            return response;
         }
         catch (err) {
             lastError = err;
+            const errMsg = err instanceof Error ? err.message : String(err);
+            attemptSpan.setStatus({ code: SpanStatusCode.ERROR, message: errMsg });
+            if (err instanceof Error)
+                attemptSpan.recordException(err);
+            attemptSpan.end();
             if (!isRetryableError(err) || model === models[models.length - 1]) {
                 throw err;
             }
+            // Record fallback span for the transition to the next model
+            const nextModel = models[idx + 1];
+            const fallbackSpan = tracer.startSpan("llm.fallback", {}, context.active());
+            fallbackSpan.setAttribute(ATTR.FALLBACK_ORIGINAL, model);
+            fallbackSpan.setAttribute(ATTR.FALLBACK_MODEL, nextModel);
+            fallbackSpan.setAttribute(ATTR.FALLBACK_ERROR, errMsg);
+            fallbackSpan.setStatus({ code: SpanStatusCode.ERROR, message: errMsg });
+            fallbackSpan.end();
             log.warn({ err, model }, "model failed, trying next");
         }
     }

package/dist/llm/openai.d.ts

@@ -7,6 +7,8 @@ export declare class OpenAIProvider implements LLMProvider {
     complete(model: string, messages: Message[], options?: CompleteOptions): Promise<LLMResponse>;
     stream(model: string, messages: Message[], options?: CompleteOptions): AsyncIterable<StreamEvent>;
     private toOpenAIMessages;
+    private mapImageBlock;
+    private mapUserContentToOpenAI;
     private fromOpenAIResponse;
     private mapStopReason;
 }

package/dist/llm/openai.js

@@ -30,7 +30,7 @@ export class OpenAIProvider {
             result.push({ role: "system", content: systemPrompt });
         for (const msg of messages) {
             if (msg.role === "user") {
-                result.push({ role: "user", content: msg.content });
+                result.push({ role: "user", content: this.mapUserContentToOpenAI(msg.content) });
             }
             else if (msg.role === "assistant") {
                 const textParts = msg.content.filter((b) => b.type === "text").map((b) => b.text).join("");
@@ -45,11 +45,42 @@ export class OpenAIProvider {
                 result.push(openaiMsg);
             }
             else if (msg.role === "tool_result") {
-                result.push({ role: "tool", tool_call_id: msg.tool_call_id, content: msg.content });
+                if (typeof msg.content === "string") {
+                    result.push({ role: "tool", tool_call_id: msg.tool_call_id, content: msg.content });
+                }
+                else {
+                    const textParts = msg.content.filter((b) => b.type === "text").map((b) => b.text).join("");
+                    const imageBlocks = msg.content.filter((b) => b.type === "image");
+                    result.push({ role: "tool", tool_call_id: msg.tool_call_id, content: textParts });
+                    if (imageBlocks.length) {
+                        result.push({
+                            role: "user",
+                            content: [
+                                { type: "text", text: "Image(s) from tool result:" },
+                                ...imageBlocks.map((b) => this.mapImageBlock(b)),
+                            ],
+                        });
+                    }
+                }
             }
         }
         return result;
     }
+    mapImageBlock(block) {
+        if (block.source.type === "base64") {
+            return { type: "image_url", image_url: { url: `data:${block.source.media_type};base64,${block.source.data}` } };
+        }
+        return { type: "image_url", image_url: { url: block.source.url } };
+    }
+    mapUserContentToOpenAI(content) {
+        if (typeof content === "string")
+            return content;
+        return content.map((block) => {
+            if (block.type === "text")
+                return { type: "text", text: block.text };
+            return this.mapImageBlock(block);
+        });
+    }
     fromOpenAIResponse(response) {
         const choice = response.choices[0];
         const content = [];

package/dist/llm/types.d.ts

@@ -8,10 +8,22 @@ export interface ToolCallBlock {
     name: string;
     arguments: Record<string, unknown>;
 }
+export interface ImageBlock {
+    type: "image";
+    source: {
+        type: "base64";
+        media_type: string;
+        data: string;
+    } | {
+        type: "url";
+        url: string;
+    };
+}
+export type UserContentBlock = TextBlock | ImageBlock;
 export type ContentBlock = TextBlock | ToolCallBlock;
 export interface UserMessage {
     role: "user";
-    content: string;
+    content: string | UserContentBlock[];
 }
 export interface AssistantMessage {
     role: "assistant";
@@ -20,7 +32,7 @@ export interface AssistantMessage {
 export interface ToolResultMessage {
     role: "tool_result";
     tool_call_id: string;
-    content: string;
+    content: string | UserContentBlock[];
 }
 export type Message = UserMessage | AssistantMessage | ToolResultMessage;
 export interface ToolDefinition {
@@ -71,3 +83,5 @@ export interface ErrorEvent {
 }
 export type StreamEvent = TextDeltaEvent | ToolCallStartEvent | ToolCallDeltaEvent | DoneEvent | ErrorEvent;
 export declare function extractText(content: ContentBlock[]): string;
+/** Extract text from user content (string or UserContentBlock[]) */
+export declare function extractUserText(content: string | UserContentBlock[]): string;