@alejandroroman/agent-kit 0.1.3 → 0.2.0

package/dist/heartbeat/runner.js

@@ -5,9 +5,11 @@ import { setupAgentSession } from "../agent/setup.js";
 import { resolveAgent, resolveModelAlias, resolveWebSearch } from "../config/resolve.js";
 import { createBuiltinRegistry } from "../tools/builtin/index.js";
 import { registerSpawnWrappers } from "../tools/builtin/spawn.js";
-import { createActivateSkillTool } from "../skills/index.js";
+import { createActivateSkillTool, preActivateSkills } from "../skills/index.js";
 import { createLogger } from "../logger.js";
 import { dateContext } from "../text.js";
+import { context, trace, SpanStatusCode } from "@opentelemetry/api";
+import { getTracer, ATTR } from "../telemetry/index.js";
 const log = createLogger("heartbeat");
 const HEARTBEAT_OK = "HEARTBEAT_OK";
 export function isHeartbeatSuppressed(text) {
@@ -64,93 +66,99 @@ export class HeartbeatRunner {
         if (!agentDef?.heartbeat)
             return undefined;
         const hb = agentDef.heartbeat;
-        if (!isWithinActiveHours(hb.activeHours)) {
-            log.debug({ agent: agentName }, "outside active hours, skipping");
-            return undefined;
-        }
-        const instructions = this.loadHeartbeatInstructions(agentName);
-        if (!instructions) {
-            log.warn({ agent: agentName }, "no HEARTBEAT.md found, skipping");
-            return undefined;
-        }
-        // Isolated tool registry per tick (same pattern as CronScheduler)
-        const sandbox = agentDef.sandbox ?? this.config.defaults.sandbox;
-        const memoryConfig = this.config.defaults.memory;
-        const tickRegistry = createBuiltinRegistry({
-            allowedCommands: sandbox?.allowedCommands,
-            allowedPaths: sandbox?.allowedPaths,
-            memoryConfig,
-            webSearch: resolveWebSearch(agentName, this.config),
+        const tracer = getTracer("heartbeat");
+        const span = tracer.startSpan("source.heartbeat", {
+            attributes: {
+                [ATTR.AGENT]: agentName,
+                [ATTR.SOURCE]: "heartbeat",
+            },
         });
-        const resolved = resolveAgent(agentName, this.config, tickRegistry, this.skillsDir);
-        // Skills setup — auto-activate all skills upfront so Haiku doesn't need to call activate_skill
-        const promptFragments = [];
-        let skillsIndex = "";
-        if (resolved.skills.length > 0) {
-            const ctx = {
-                manifests: resolved.skills,
-                skillsDir: this.skillsDir,
-                toolRegistry: tickRegistry,
-                promptFragments,
-                activatedSkills: new Set(),
-            };
-            const activateTool = createActivateSkillTool(ctx);
-            tickRegistry.register(activateTool);
-            // Auto-activate all skills upfront
-            for (const manifest of resolved.skills) {
-                try {
-                    const result = await activateTool.execute({ skill_name: manifest.name });
-                    if (typeof result === "string" && result.startsWith("Error")) {
-                        log.error({ agent: agentName, skill: manifest.name, result }, "failed to auto-activate skill");
-                    }
-                }
-                catch (err) {
-                    log.error({ err, agent: agentName, skill: manifest.name }, "skill auto-activation threw");
-                }
+        const spanCtx = trace.setSpan(context.active(), span);
+        try {
+            if (!isWithinActiveHours(hb.activeHours)) {
+                log.debug({ agent: agentName }, "outside active hours, skipping");
+                span.setAttribute(ATTR.SUPPRESSED, "outside_active_hours");
+                return undefined;
             }
-            const activatedNames = [...ctx.activatedSkills];
-            if (activatedNames.length > 0) {
-                skillsIndex = "\n\nThe following skills are pre-activated: "
-                    + activatedNames.join(", ")
-                    + ".\nTheir tools are already available — you do not need to call activate_skill.";
+            const instructions = this.loadHeartbeatInstructions(agentName);
+            if (!instructions) {
+                log.warn({ agent: agentName }, "no HEARTBEAT.md found, skipping");
+                span.setAttribute(ATTR.SUPPRESSED, "no_instructions");
+                return undefined;
             }
+            return await context.with(spanCtx, async () => {
+                // Isolated tool registry per tick (same pattern as CronScheduler)
+                const sandbox = agentDef.sandbox ?? this.config.defaults.sandbox;
+                const memoryConfig = this.config.defaults.memory;
+                const tickRegistry = createBuiltinRegistry({
+                    allowedCommands: sandbox?.allowedCommands,
+                    allowedPaths: sandbox?.allowedPaths,
+                    memoryConfig,
+                    webSearch: resolveWebSearch(agentName, this.config),
+                });
+                const resolved = resolveAgent(agentName, this.config, tickRegistry, this.skillsDir);
+                // Skills setup — auto-activate all skills upfront so Haiku doesn't need to call activate_skill
+                const promptFragments = [];
+                let skillsIndex = "";
+                if (resolved.skills.length > 0) {
+                    const ctx = {
+                        manifests: resolved.skills,
+                        skillsDir: this.skillsDir,
+                        toolRegistry: tickRegistry,
+                        promptFragments,
+                        activatedSkills: new Set(),
+                    };
+                    const activateTool = createActivateSkillTool(ctx);
+                    tickRegistry.register(activateTool);
+                    // Heartbeat always auto-activates (Haiku can't reliably call activate_skill)
+                    skillsIndex = await preActivateSkills(ctx, activateTool, log);
+                }
+                // Spawn wrappers
+                if (resolved.canSpawn.length > 0) {
+                    registerSpawnWrappers(resolved.canSpawn, this.config, this.agentRegistry, tickRegistry, this.usageStore);
+                }
+                // Ephemeral session: no history accumulation, append-only audit log
+                const sessionId = `heartbeat-${agentName}`;
+                const { soul, session } = setupAgentSession(this.dataDir, agentName, sessionId);
+                // Build tick prompt with fresh instructions + timestamp
+                const tickPrompt = `[Heartbeat tick — ${new Date().toISOString()}]\n\n${instructions}`;
+                const userMsg = { role: "user", content: tickPrompt };
+                session.append(userMsg); // audit only
+                const messages = [userMsg]; // ephemeral — no history
+                // Model: heartbeat.model overrides agent model
+                const model = hb.model
+                    ? resolveModelAlias(hb.model, this.config.models)
+                    : resolved.model;
+                const systemPrompt = [soul, dateContext(), skillsIndex, ...promptFragments]
+                    .filter(Boolean)
+                    .join("\n\n") || undefined;
+                const result = await runAgentLoop(messages, {
+                    model,
+                    fallbacks: resolved.fallbacks,
+                    systemPrompt,
+                    toolRegistry: tickRegistry,
+                    maxIterations: hb.maxIterations ?? resolved.maxIterations,
+                    compactionThreshold: resolved.compactionThreshold,
+                    maxToolResultSize: resolved.maxToolResultSize,
+                    agentName,
+                    usageStore: this.usageStore,
+                    source: "heartbeat",
+                });
+                // Audit: append all new messages from the agent loop
+                for (const msg of result.messages.slice(1)) {
+                    session.append(msg);
+                }
+                return result;
+            });
         }
-        // Spawn wrappers
-        if (resolved.canSpawn.length > 0) {
-            registerSpawnWrappers(resolved.canSpawn, this.config, this.agentRegistry, tickRegistry, this.usageStore);
+        catch (err) {
+            span.setStatus({ code: SpanStatusCode.ERROR, message: String(err) });
+            span.recordException(err instanceof Error ? err : new Error(String(err)));
+            throw err;
         }
-        // Ephemeral session: no history accumulation, append-only audit log
-        const sessionId = `heartbeat-${agentName}`;
-        const { soul, session } = setupAgentSession(this.dataDir, agentName, sessionId);
-        // Build tick prompt with fresh instructions + timestamp
-        const tickPrompt = `[Heartbeat tick — ${new Date().toISOString()}]\n\n${instructions}`;
-        const userMsg = { role: "user", content: tickPrompt };
-        session.append(userMsg); // audit only
-        const messages = [userMsg]; // ephemeral — no history
-        // Model: heartbeat.model overrides agent model
-        const model = hb.model
-            ? resolveModelAlias(hb.model, this.config.models)
-            : resolved.model;
-        const systemPrompt = [soul, dateContext(), skillsIndex, ...promptFragments]
-            .filter(Boolean)
-            .join("\n\n") || undefined;
-        const result = await runAgentLoop(messages, {
-            model,
-            fallbacks: resolved.fallbacks,
-            systemPrompt,
-            toolRegistry: tickRegistry,
-            maxIterations: hb.maxIterations ?? resolved.maxIterations,
-            compactionThreshold: resolved.compactionThreshold,
-            maxToolResultSize: resolved.maxToolResultSize,
-            agentName,
-            usageStore: this.usageStore,
-            source: "heartbeat",
-        });
-        // Audit: append all new messages from the agent loop
-        for (const msg of result.messages.slice(1)) {
-            session.append(msg);
+        finally {
+            span.end();
         }
-        return result;
     }
     start(callbacks) {
         for (const [agentName, agentDef] of Object.entries(this.config.agents)) {
@@ -162,6 +170,15 @@ export class HeartbeatRunner {
             const runTick = async () => {
                 if (this.running.has(agentName)) {
                     log.info({ agent: agentName }, "previous tick still running, skipping");
+                    const tracer = getTracer("heartbeat");
+                    const skipSpan = tracer.startSpan("source.heartbeat", {
+                        attributes: {
+                            [ATTR.AGENT]: agentName,
+                            [ATTR.SOURCE]: "heartbeat",
+                            [ATTR.SUPPRESSED]: "already_running",
+                        },
+                    });
+                    skipSpan.end();
                     return;
                 }
                 this.running.add(agentName);

package/dist/index.js

@@ -1,209 +1,4 @@
-import * as readline from "readline";
-import * as path from "path";
-import { runAgentLoop } from "./agent/loop.js";
-import { compactMessages } from "./session/compaction.js";
-import { loadConfig } from "./config/index.js";
-import { CronScheduler } from "./cron/scheduler.js";
-import { HeartbeatRunner } from "./heartbeat/index.js";
-import { createSlackGateway } from "./gateways/slack/index.js";
-import { createLogger } from "./logger.js";
-import { createApiServer } from "./api/router.js";
-import { buildAgentRuntime, buildSystemPrompt, initUsageStore, createAgentExecutor, } from "./bootstrap.js";
-const log = createLogger("startup");
-const CONFIG_PATH = path.join(process.cwd(), "agent-kit.json");
-const DATA_DIR = path.join(process.cwd(), "data");
-const DEFAULT_AGENT = "default";
-const SKILLS_DIR = path.join(process.cwd(), "skills");
-async function main() {
-    const config = loadConfig(CONFIG_PATH);
-    const { usageStore, apiPort } = initUsageStore(config);
-    const apiServer = createApiServer({
-        config,
-        configPath: CONFIG_PATH,
-        usageStore,
-        dataDir: DATA_DIR,
-    }, apiPort);
-    const agentName = process.argv[2] ?? DEFAULT_AGENT;
-    // Validate all agent tool references at startup
-    const runtime = buildAgentRuntime(agentName, config, {
-        dataDir: DATA_DIR,
-        skillsDir: SKILLS_DIR,
-        usageStore,
-    });
-    const toolWarnings = runtime.toolRegistry.validateAgents(config.agents);
-    for (const w of toolWarnings)
-        console.warn(`  ⚠ ${w}`);
-    const { resolved, toolRegistry, agentRegistry, promptFragments, skillsIndex, soul, session } = runtime;
-    const sandbox = config.agents[agentName]?.sandbox ?? config.defaults.sandbox;
-    console.log("Agent Kit");
-    console.log(`  Model: ${resolved.model}`);
-    if (resolved.fallbacks.length > 0) {
-        console.log(`  Fallbacks: ${resolved.fallbacks.join(" → ")}`);
-    }
-    console.log(`  Agent: ${agentName}`);
-    console.log(`  SOUL: ${soul ? "loaded" : "none (create data/agents/default/SOUL.md)"}`);
-    console.log(`  Tools: ${resolved.tools.map((t) => t.name).join(", ")}`);
-    if (resolved.skills.length > 0) {
-        console.log(`  Skills: ${resolved.skills.map((s) => s.name).join(", ")}`);
-    }
-    const sandboxSummary = (() => {
-        if (!sandbox)
-            return "off (unrestricted)";
-        const parts = [];
-        if (sandbox.allowedCommands?.length) {
-            parts.push(`${sandbox.allowedCommands.length} commands allowed`);
-        }
-        if (sandbox.allowedPaths?.length) {
-            parts.push(`${sandbox.allowedPaths.length} paths allowed`);
-        }
-        return parts.length > 0 ? parts.join(", ") : "configured (no restrictions)";
-    })();
-    console.log(`  Sandbox: ${sandboxSummary}`);
-    if (config.defaults.usage) {
-        console.log(`  Usage: tracking enabled (${config.defaults.usage.dbPath})`);
-    }
-    const scheduler = new CronScheduler(config, toolRegistry, agentRegistry, DATA_DIR, SKILLS_DIR, usageStore);
-    const enabledJobs = scheduler.getJobs().filter((j) => j.enabled);
-    // Agent executor for inbound Slack messages
-    const executeAgent = createAgentExecutor(config, {
-        dataDir: DATA_DIR,
-        skillsDir: SKILLS_DIR,
-        agentRegistry,
-        usageStore,
-        source: "slack",
-    });
-    // Slack gateway (optional — requires both tokens + agent slack bindings)
-    let gateway;
-    const hasSlackBindings = Object.values(config.agents).some((a) => a.slack);
-    if (hasSlackBindings && process.env.SLACK_BOT_TOKEN && process.env.SLACK_APP_TOKEN) {
-        try {
-            gateway = createSlackGateway(config, { onAgentRequest: executeAgent });
-            await gateway.start();
-            const bindingCount = Object.values(config.agents).filter((a) => a.slack).length;
-            log.info({ bindings: bindingCount }, "Slack connected");
-        }
-        catch (err) {
-            log.warn({ err }, "Slack failed to connect");
-        }
-    }
-    try {
-        const actualPort = await apiServer.start();
-        console.log(`  API: http://localhost:${actualPort}/api`);
-    }
-    catch (err) {
-        log.warn({ err }, "failed to start API server — continuing without it");
-    }
-    if (enabledJobs.length > 0) {
-        const jobList = enabledJobs.map((j) => `${j.id} @ ${j.schedule}`).join(", ");
-        log.info({ jobs: enabledJobs.length, jobList }, "cron scheduled");
-        scheduler.start({
-            onResult: (jobId, agentName, result) => {
-                log.info({ jobId, tokens: result.usage.inputTokens + result.usage.outputTokens }, "cron job completed");
-                process.stdout.write("You: ");
-                const channelOverride = config.cron.find((j) => j.id === jobId)?.slack?.channelId;
-                gateway?.onJobResult(agentName, jobId, result, channelOverride).catch((err) => log.warn({ err, jobId }, "failed to post cron result to Slack"));
-            },
-            onError: (jobId, agentName, error) => {
-                log.error({ err: error, jobId }, "cron job failed");
-                process.stdout.write("You: ");
-                const channelOverride = config.cron.find((j) => j.id === jobId)?.slack?.channelId;
-                gateway?.onJobError(agentName, jobId, error, channelOverride).catch((err) => log.warn({ err, jobId }, "failed to post cron error to Slack"));
-            },
-        });
-    }
-    const heartbeat = new HeartbeatRunner(config, agentRegistry, DATA_DIR, SKILLS_DIR, usageStore);
-    const heartbeatAgents = heartbeat.getHeartbeatAgents();
-    if (heartbeatAgents.length > 0) {
-        log.info({ agents: heartbeatAgents }, "heartbeat started");
-        heartbeat.start({
-            onResult: (agentName, result) => {
-                log.info({ agent: agentName, tokens: result.usage.inputTokens + result.usage.outputTokens }, "heartbeat alert");
-                process.stdout.write("You: ");
-                const channelOverride = config.agents[agentName]?.heartbeat?.slack?.channelId;
-                gateway?.onJobResult(agentName, "heartbeat", result, channelOverride).catch((err) => log.warn({ err, agent: agentName }, "failed to post heartbeat result to Slack"));
-            },
-            onError: (agentName, error) => {
-                log.error({ err: error, agent: agentName }, "heartbeat failed");
-                process.stdout.write("You: ");
-                const channelOverride = config.agents[agentName]?.heartbeat?.slack?.channelId;
-                gateway?.onJobError(agentName, "heartbeat", error, channelOverride).catch((err) => log.warn({ err, agent: agentName }, "failed to post heartbeat error to Slack"));
-            },
-            onSuppressed: (agentName) => {
-                log.debug({ agent: agentName }, "heartbeat OK, suppressed");
-            },
-        });
-    }
-    if (heartbeatAgents.length > 0) {
-        console.log(`  Heartbeat: ${heartbeatAgents.join(", ")}`);
-    }
-    console.log(`  Commands: /new (reset), /quit`);
-    console.log();
-    let messages = session.getMessages();
-    const rl = readline.createInterface({
-        input: process.stdin,
-        output: process.stdout,
-    });
-    const ask = () => {
-        rl.question("You: ", async (input) => {
-            const trimmed = input.trim();
-            if (trimmed === "/quit") {
-                scheduler.stop();
-                heartbeat.stop();
-                await gateway?.stop();
-                await apiServer.stop();
-                try {
-                    usageStore?.close();
-                }
-                catch (err) {
-                    log.warn({ err }, "failed to close usage store");
-                }
-                finally {
-                    rl.close();
-                }
-                return;
-            }
-            if (trimmed === "/new") {
-                messages = [];
-                console.log("  Session reset.\n");
-                ask();
-                return;
-            }
-            if (!trimmed) {
-                ask();
-                return;
-            }
-            const userMsg = { role: "user", content: trimmed };
-            messages.push(userMsg);
-            session.append(userMsg);
-            messages = await compactMessages(messages, resolved.model, resolved.compactionThreshold);
-            try {
-                const systemPrompt = buildSystemPrompt(soul, skillsIndex, promptFragments);
-                const result = await runAgentLoop(messages, {
-                    model: resolved.model,
-                    fallbacks: resolved.fallbacks,
-                    systemPrompt,
-                    toolRegistry,
-                    maxIterations: resolved.maxIterations,
-                    compactionThreshold: resolved.compactionThreshold,
-                    maxToolResultSize: resolved.maxToolResultSize,
-                    agentName,
-                    usageStore,
-                    source: "cli",
-                });
-                const newMessages = result.messages.slice(messages.length - 1);
-                for (const msg of newMessages) {
-                    session.append(msg);
-                }
-                messages = result.messages;
-                console.log(`\nAgent: ${result.text}`);
-                console.log(`  (${result.usage.inputTokens + result.usage.outputTokens} tokens)\n`);
-            }
-            catch (err) {
-                console.error(`\n  Error: ${err instanceof Error ? err.message : err}\n`);
-            }
-            ask();
-        });
-    };
-    ask();
-}
-main();
+// Legacy entry point — delegates to the CLI start command.
+// Kept for backwards compatibility with `pnpm dev` / `pnpm dev:local`.
+import { start } from "./cli/start.js";
+start();

package/dist/llm/anthropic.d.ts

@@ -7,6 +7,7 @@ export declare class AnthropicProvider implements LLMProvider {
     complete(model: string, messages: Message[], options?: CompleteOptions): Promise<LLMResponse>;
     stream(model: string, messages: Message[], options?: CompleteOptions): AsyncIterable<StreamEvent>;
     private toAnthropicMessages;
+    private mapUserContent;
     private fromAnthropicResponse;
     private mapStopReason;
 }

package/dist/llm/anthropic.js

@@ -39,7 +39,7 @@ export class AnthropicProvider {
     toAnthropicMessages(messages) {
         return messages.map((msg) => {
             if (msg.role === "user")
-                return { role: "user", content: msg.content };
+                return { role: "user", content: this.mapUserContent(msg.content) };
             if (msg.role === "assistant") {
                 return {
                     role: "assistant",
@@ -51,11 +51,20 @@ export class AnthropicProvider {
                 };
             }
             if (msg.role === "tool_result") {
-                return { role: "user", content: [{ type: "tool_result", tool_use_id: msg.tool_call_id, content: msg.content }] };
+                return { role: "user", content: [{ type: "tool_result", tool_use_id: msg.tool_call_id, content: this.mapUserContent(msg.content) }] };
             }
             return msg;
         });
     }
+    mapUserContent(content) {
+        if (typeof content === "string")
+            return content;
+        return content.map((block) => {
+            if (block.type === "text")
+                return { type: "text", text: block.text };
+            return { type: "image", source: block.source };
+        });
+    }
     fromAnthropicResponse(response) {
         const content = response.content.map((block) => {
             if (block.type === "text")

package/dist/llm/fallback.js

@@ -1,4 +1,6 @@
 import { createLogger } from "../logger.js";
+import { getTracer, ATTR } from "../telemetry/index.js";
+import { SpanStatusCode, context } from "@opentelemetry/api";
 const log = createLogger("llm");
 const RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503]);
 export function isRetryableError(err) {
@@ -12,17 +14,47 @@ export function isRetryableError(err) {
     return false;
 }
 export async function completeWithFallback(primary, fallbacks, messages, options, completeFn) {
+    const tracer = getTracer("llm");
     const models = [primary, ...fallbacks];
     let lastError;
-    for (const model of models) {
+    for (let idx = 0; idx < models.length; idx++) {
+        const model = models[idx];
+        const [provider = "unknown"] = model.split(":");
+        const attemptSpan = tracer.startSpan("llm.attempt", {}, context.active());
+        attemptSpan.setAttribute(ATTR.PROVIDER, provider);
+        attemptSpan.setAttribute(ATTR.MODEL, model);
         try {
-            return await completeFn(model, messages, options);
+            const response = await completeFn(model, messages, options);
+            // Set token attributes on successful attempt
+            attemptSpan.setAttribute(ATTR.INPUT_TOKENS, response.usage.inputTokens);
+            attemptSpan.setAttribute(ATTR.OUTPUT_TOKENS, response.usage.outputTokens);
+            attemptSpan.setAttribute(ATTR.CACHE_CREATION_TOKENS, response.usage.cacheCreationTokens);
+            attemptSpan.setAttribute(ATTR.CACHE_READ_TOKENS, response.usage.cacheReadTokens);
+            attemptSpan.setAttribute(ATTR.STOP_REASON, response.stopReason);
+            if (response.latencyMs != null) {
+                attemptSpan.setAttribute(ATTR.LATENCY_MS, response.latencyMs);
+            }
+            attemptSpan.end();
+            return response;
         }
         catch (err) {
             lastError = err;
+            const errMsg = err instanceof Error ? err.message : String(err);
+            attemptSpan.setStatus({ code: SpanStatusCode.ERROR, message: errMsg });
+            if (err instanceof Error)
+                attemptSpan.recordException(err);
+            attemptSpan.end();
             if (!isRetryableError(err) || model === models[models.length - 1]) {
                 throw err;
             }
+            // Record fallback span for the transition to the next model
+            const nextModel = models[idx + 1];
+            const fallbackSpan = tracer.startSpan("llm.fallback", {}, context.active());
+            fallbackSpan.setAttribute(ATTR.FALLBACK_ORIGINAL, model);
+            fallbackSpan.setAttribute(ATTR.FALLBACK_MODEL, nextModel);
+            fallbackSpan.setAttribute(ATTR.FALLBACK_ERROR, errMsg);
+            fallbackSpan.setStatus({ code: SpanStatusCode.ERROR, message: errMsg });
+            fallbackSpan.end();
             log.warn({ err, model }, "model failed, trying next");
         }
     }

package/dist/llm/openai.d.ts

@@ -7,6 +7,8 @@ export declare class OpenAIProvider implements LLMProvider {
     complete(model: string, messages: Message[], options?: CompleteOptions): Promise<LLMResponse>;
     stream(model: string, messages: Message[], options?: CompleteOptions): AsyncIterable<StreamEvent>;
     private toOpenAIMessages;
+    private mapImageBlock;
+    private mapUserContentToOpenAI;
     private fromOpenAIResponse;
     private mapStopReason;
 }

package/dist/llm/openai.js

@@ -30,7 +30,7 @@ export class OpenAIProvider {
             result.push({ role: "system", content: systemPrompt });
         for (const msg of messages) {
             if (msg.role === "user") {
-                result.push({ role: "user", content: msg.content });
+                result.push({ role: "user", content: this.mapUserContentToOpenAI(msg.content) });
             }
             else if (msg.role === "assistant") {
                 const textParts = msg.content.filter((b) => b.type === "text").map((b) => b.text).join("");
@@ -45,11 +45,42 @@ export class OpenAIProvider {
                 result.push(openaiMsg);
             }
             else if (msg.role === "tool_result") {
-                result.push({ role: "tool", tool_call_id: msg.tool_call_id, content: msg.content });
+                if (typeof msg.content === "string") {
+                    result.push({ role: "tool", tool_call_id: msg.tool_call_id, content: msg.content });
+                }
+                else {
+                    const textParts = msg.content.filter((b) => b.type === "text").map((b) => b.text).join("");
+                    const imageBlocks = msg.content.filter((b) => b.type === "image");
+                    result.push({ role: "tool", tool_call_id: msg.tool_call_id, content: textParts });
+                    if (imageBlocks.length) {
+                        result.push({
+                            role: "user",
+                            content: [
+                                { type: "text", text: "Image(s) from tool result:" },
+                                ...imageBlocks.map((b) => this.mapImageBlock(b)),
+                            ],
+                        });
+                    }
+                }
             }
         }
         return result;
     }
+    mapImageBlock(block) {
+        if (block.source.type === "base64") {
+            return { type: "image_url", image_url: { url: `data:${block.source.media_type};base64,${block.source.data}` } };
+        }
+        return { type: "image_url", image_url: { url: block.source.url } };
+    }
+    mapUserContentToOpenAI(content) {
+        if (typeof content === "string")
+            return content;
+        return content.map((block) => {
+            if (block.type === "text")
+                return { type: "text", text: block.text };
+            return this.mapImageBlock(block);
+        });
+    }
     fromOpenAIResponse(response) {
         const choice = response.choices[0];
         const content = [];

package/dist/llm/types.d.ts

@@ -8,10 +8,22 @@ export interface ToolCallBlock {
     name: string;
     arguments: Record<string, unknown>;
 }
+export interface ImageBlock {
+    type: "image";
+    source: {
+        type: "base64";
+        media_type: string;
+        data: string;
+    } | {
+        type: "url";
+        url: string;
+    };
+}
+export type UserContentBlock = TextBlock | ImageBlock;
 export type ContentBlock = TextBlock | ToolCallBlock;
 export interface UserMessage {
     role: "user";
-    content: string;
+    content: string | UserContentBlock[];
 }
 export interface AssistantMessage {
     role: "assistant";
@@ -20,7 +32,7 @@ export interface AssistantMessage {
 export interface ToolResultMessage {
     role: "tool_result";
     tool_call_id: string;
-    content: string;
+    content: string | UserContentBlock[];
 }
 export type Message = UserMessage | AssistantMessage | ToolResultMessage;
 export interface ToolDefinition {
@@ -71,3 +83,5 @@ export interface ErrorEvent {
 }
 export type StreamEvent = TextDeltaEvent | ToolCallStartEvent | ToolCallDeltaEvent | DoneEvent | ErrorEvent;
 export declare function extractText(content: ContentBlock[]): string;
+/** Extract text from user content (string or UserContentBlock[]) */
+export declare function extractUserText(content: string | UserContentBlock[]): string;

package/dist/llm/types.js

@@ -4,3 +4,12 @@ export function extractText(content) {
         .map((b) => b.text)
         .join("");
 }
+/** Extract text from user content (string or UserContentBlock[]) */
+export function extractUserText(content) {
+    if (typeof content === "string")
+        return content;
+    return content
+        .filter((b) => b.type === "text")
+        .map((b) => b.text)
+        .join("");
+}

package/dist/logger.js

@@ -1,4 +1,5 @@
 import pino from "pino";
+import { trace } from "@opentelemetry/api";
 const root = pino({
     level: process.env.LOG_LEVEL ?? "info",
     transport: {
@@ -9,6 +10,13 @@ const root = pino({
             colorize: true,
         },
     },
+    mixin() {
+        const span = trace.getActiveSpan();
+        if (!span)
+            return {};
+        const ctx = span.spanContext();
+        return { trace_id: ctx.traceId, span_id: ctx.spanId };
+    },
 });
 export function createLogger(name) {
     return root.child({ component: name });

package/dist/media/sanitize.d.ts

@@ -0,0 +1,5 @@
+export interface SanitizedImage {
+    base64: string;
+    mediaType: string;
+}
+export declare function sanitizeImage(buffer: Buffer, claimedMimeType?: string): Promise<SanitizedImage>;