@meetopenbot/openbot 1.0.0 → 1.0.1

package/dist/history.js

@@ -164,12 +164,16 @@ export function eventsToModelMessages(events, options) {
             }
             case 'agent:invoke': {
                 const invokeEvent = event;
-                if (invokeEvent.data?.content && invokeEvent.data?.role) {
+                if (invokeEvent.data?.role) {
                     const role = invokeEvent.data.role === 'agent' ? 'user' : invokeEvent.data.role;
-                    messages.push({
-                        role,
-                        content: invokeEvent.data.content,
-                    });
+                    const content = invokeEvent.data.content ?? '';
+                    const hasFiles = Boolean(invokeEvent.data.attachments?.length);
+                    if (content || hasFiles) {
+                        messages.push({
+                            role,
+                            content,
+                        });
+                    }
                 }
                 break;
             }

package/dist/images.js

@@ -0,0 +1,64 @@
+import { shouldUseCreditsAuth } from '@meetopenbot/plugin-sdk';
+export function isImageAttachment(file) {
+    if (file.mimeType?.startsWith('image/'))
+        return true;
+    return /\.(png|jpe?g|gif|webp)$/i.test(file.name);
+}
+export function eventsHaveImages(events) {
+    return events.some((event) => {
+        if (event.meta?.parentToolCallId)
+            return false;
+        if (event.type !== 'agent:invoke')
+            return false;
+        return Boolean(event.data?.attachments?.some(isImageAttachment));
+    });
+}
+/** Auto uses DeepSeek, which cannot see pictures. Pick a cheap vision model. */
+export function pickVisionModel(authMode, env = process.env) {
+    if (shouldUseCreditsAuth({ authMode }))
+        return 'google/gemini-3.5-flash';
+    if (env.OPENAI_API_KEY?.trim())
+        return 'openai/gpt-4o-mini';
+    if (env.GOOGLE_GENERATIVE_AI_API_KEY?.trim())
+        return 'google/gemini-3.5-flash';
+    if (env.ANTHROPIC_API_KEY?.trim())
+        return 'anthropic/claude-haiku-4-5';
+    return undefined;
+}
+/** Put image files on matching user messages as AI SDK `file` parts. */
+export async function addImageParts(messages, events, read) {
+    const result = [...messages];
+    let userIndex = 0;
+    for (const event of events) {
+        if (event.meta?.parentToolCallId)
+            continue;
+        if (event.type !== 'agent:invoke' || !event.data?.role)
+            continue;
+        const files = (event.data.attachments ?? []).filter(isImageAttachment);
+        const text = event.data.content ?? '';
+        if (!text && files.length === 0)
+            continue;
+        const current = result[userIndex];
+        userIndex += 1;
+        if (!current || files.length === 0)
+            continue;
+        const parts = [];
+        if (typeof current.content === 'string' && current.content) {
+            parts.push({ type: 'text', text: current.content });
+        }
+        for (const file of files) {
+            const loaded = await read(file.name);
+            if (!loaded?.data.byteLength)
+                continue;
+            parts.push({
+                type: 'file',
+                data: loaded.data,
+                mediaType: loaded.mediaType || 'image/png',
+            });
+        }
+        if (parts.some((part) => part.type === 'file')) {
+            result[userIndex - 1] = { role: current.role, content: parts };
+        }
+    }
+    return result;
+}

package/dist/runtime.js

@@ -1,5 +1,6 @@
 import { streamText, stepCountIs } from 'ai';
 import { eventsToModelMessages } from './history.js';
+import { addImageParts, eventsHaveImages, pickVisionModel } from './images.js';
 import { buildContext } from './context.js';
 import { OPENBOT_SYSTEM_PROMPT } from './system-prompt.js';
 import { isAuthErrorMessage, isCreditsErrorMessage } from '@meetopenbot/plugin-sdk';
@@ -97,11 +98,19 @@ export async function* runOpenbotTurn(args, options) {
         channelId: state.channelId ?? '',
         threadId,
     });
-    const candidates = isAutoModel(configuredModelString)
+    const hasImages = eventsHaveImages(events);
+    let candidates = isAutoModel(configuredModelString)
         ? listAutoModelCandidates({ authMode })
         : [configuredModelString];
     try {
-        const missingKey = missingByokApiKeyMessage(configuredModelString, { authMode });
+        if (hasImages && isAutoModel(configuredModelString)) {
+            const vision = pickVisionModel(authMode);
+            if (!vision) {
+                throw new Error('Looking at images needs an OpenAI, Google, or Anthropic API key. Auto\'s usual model cannot see pictures.');
+            }
+            candidates = [vision];
+        }
+        const missingKey = missingByokApiKeyMessage(candidates[0], { authMode });
         if (missingKey)
             throw new Error(missingKey);
         let lastError;
@@ -116,12 +125,30 @@ export async function* runOpenbotTurn(args, options) {
             };
             const tools = toAiTools(toolDefinitions, args, flushPendingOutput);
             try {
+                let messages = eventsToModelMessages(events, {
+                    includeReasoning: includeReasoningInHistory(candidate),
+                });
+                if (hasImages) {
+                    messages = await addImageParts(messages, events, async (name) => {
+                        try {
+                            const file = await storage.readThreadAttachment({
+                                channelId: state.channelId,
+                                threadId,
+                                name,
+                            });
+                            return { data: new Uint8Array(file.body), mediaType: file.mimeType };
+                        }
+                        catch (error) {
+                            console.warn('[openbot] could not read attachment', name, error);
+                            return null;
+                        }
+                    });
+                    console.info('[openbot] attached images for', candidate);
+                }
                 const result = streamText({
                     model: resolveModel(candidate, { authMode }),
                     system: systemPrompt,
-                    messages: eventsToModelMessages(events, {
-                        includeReasoning: includeReasoningInHistory(candidate),
-                    }),
+                    messages,
                     tools,
                     abortSignal,
                     reasoning: reasoningForModel(candidate),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meetopenbot/openbot",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "OpenBot coordinator runtime: ask specialists, route work into Spaces, and track todos.",
   "type": "module",
   "main": "./dist/index.js",
@@ -24,7 +24,7 @@
     "@ai-sdk/openai": "^4.0.46",
     "ai": "^7.0.77",
     "zod": "^4.3.5",
-    "@meetopenbot/plugin-sdk": "^1.0.0"
+    "@meetopenbot/plugin-sdk": "^1.0.2"
   },
   "devDependencies": {
     "@types/node": "^20.10.1",