@meetopenbot/openbot 1.0.0 → 1.0.2

package/dist/auto-model.js

@@ -1,4 +1,4 @@
-import { shouldUseCreditsAuth } from '@meetopenbot/plugin-sdk';
+import { BYOK_ENV_KEYS, shouldUseCreditsAuth } from '@meetopenbot/plugin-sdk';
 /** Product alias shown in the picker. Never sent to a provider or the credits proxy. */
 export const AUTO_MODEL_ID = 'openbot/auto';
 export const AUTO_MODEL_OPTION = {
@@ -11,12 +11,9 @@ export const AUTO_MODEL_OPTION = {
  * order, failover, intent routing) without changing the `openbot/auto` alias.
  */
 export const AUTO_MODEL_POOL = ['deepseek/deepseek-v4-flash'];
-const PROVIDER_BYOK_ENV = {
-    openai: 'OPENAI_API_KEY',
-    anthropic: 'ANTHROPIC_API_KEY',
-    google: 'GOOGLE_GENERATIVE_AI_API_KEY',
-    deepseek: 'DEEPSEEK_API_KEY',
-};
+function providerByokKeys(provider) {
+    return BYOK_ENV_KEYS[provider];
+}
 export function isAutoModel(modelString) {
     if (!modelString)
         return true;
@@ -27,8 +24,8 @@ function providerOf(modelString) {
     return modelString.split('/')[0] ?? '';
 }
 function providerHasByokKey(provider, env) {
-    const key = PROVIDER_BYOK_ENV[provider];
-    return Boolean(key && env[key]?.trim());
+    const keys = providerByokKeys(provider);
+    return Boolean(keys?.some((key) => env[key]?.trim()));
 }
 /** Concrete models Auto may call, filtered by credits vs BYOK keys. */
 export function listAutoModelCandidates(ctx) {
@@ -57,8 +54,7 @@ export function missingByokApiKeyMessage(modelString, ctx) {
     const provider = providerOf(expandModelString(modelString, ctx));
     if (providerHasByokKey(provider, env))
         return undefined;
-    const envVar = PROVIDER_BYOK_ENV[provider];
-    if (!envVar)
+    if (!providerByokKeys(provider))
         return undefined;
     return `${provider} API key is missing`;
 }

package/dist/history.js

@@ -164,12 +164,16 @@ export function eventsToModelMessages(events, options) {
             }
             case 'agent:invoke': {
                 const invokeEvent = event;
-                if (invokeEvent.data?.content && invokeEvent.data?.role) {
+                if (invokeEvent.data?.role) {
                     const role = invokeEvent.data.role === 'agent' ? 'user' : invokeEvent.data.role;
-                    messages.push({
-                        role,
-                        content: invokeEvent.data.content,
-                    });
+                    const content = invokeEvent.data.content ?? '';
+                    const hasFiles = Boolean(invokeEvent.data.attachments?.length);
+                    if (content || hasFiles) {
+                        messages.push({
+                            role,
+                            content,
+                        });
+                    }
                 }
                 break;
             }

package/dist/images.js

@@ -0,0 +1,67 @@
+import { BYOK_ENV_KEYS, shouldUseCreditsAuth } from '@meetopenbot/plugin-sdk';
+export function isImageAttachment(file) {
+    if (file.mimeType?.startsWith('image/'))
+        return true;
+    return /\.(png|jpe?g|gif|webp)$/i.test(file.name);
+}
+export function eventsHaveImages(events) {
+    return events.some((event) => {
+        if (event.meta?.parentToolCallId)
+            return false;
+        if (event.type !== 'agent:invoke')
+            return false;
+        return Boolean(event.data?.attachments?.some(isImageAttachment));
+    });
+}
+function hasAnyKey(env, keys) {
+    return keys.some((key) => Boolean(env[key]?.trim()));
+}
+/** Auto uses DeepSeek, which cannot see pictures. Pick a cheap vision model. */
+export function pickVisionModel(authMode, env = process.env) {
+    if (shouldUseCreditsAuth({ authMode }))
+        return 'google/gemini-3.5-flash';
+    if (hasAnyKey(env, BYOK_ENV_KEYS.openai))
+        return 'openai/gpt-4o-mini';
+    if (hasAnyKey(env, BYOK_ENV_KEYS.google))
+        return 'google/gemini-3.5-flash';
+    if (hasAnyKey(env, BYOK_ENV_KEYS.anthropic))
+        return 'anthropic/claude-haiku-4-5';
+    return undefined;
+}
+/** Put image files on matching user messages as AI SDK `file` parts. */
+export async function addImageParts(messages, events, read) {
+    const result = [...messages];
+    let userIndex = 0;
+    for (const event of events) {
+        if (event.meta?.parentToolCallId)
+            continue;
+        if (event.type !== 'agent:invoke' || !event.data?.role)
+            continue;
+        const files = (event.data.attachments ?? []).filter(isImageAttachment);
+        const text = event.data.content ?? '';
+        if (!text && files.length === 0)
+            continue;
+        const current = result[userIndex];
+        userIndex += 1;
+        if (!current || files.length === 0)
+            continue;
+        const parts = [];
+        if (typeof current.content === 'string' && current.content) {
+            parts.push({ type: 'text', text: current.content });
+        }
+        for (const file of files) {
+            const loaded = await read(file.name);
+            if (!loaded?.data.byteLength)
+                continue;
+            parts.push({
+                type: 'file',
+                data: loaded.data,
+                mediaType: loaded.mediaType || 'image/png',
+            });
+        }
+        if (parts.some((part) => part.type === 'file')) {
+            result[userIndex - 1] = { role: current.role, content: parts };
+        }
+    }
+    return result;
+}

package/dist/model.js

@@ -1,26 +1,18 @@
 import { createOpenAI, openai as defaultOpenai } from '@ai-sdk/openai';
 import { createAnthropic, anthropic } from '@ai-sdk/anthropic';
 import { createDeepSeek, deepSeek } from '@ai-sdk/deepseek';
+import { createGoogle, google as defaultGoogle } from '@ai-sdk/google';
 import { expandModelString } from './auto-model.js';
-import { CREDITS_API_KEY_PLACEHOLDER, INTEGRATIONS_TOKEN_HEADER, creditsProviderBaseUrl, resolveCreditsAuthConfig, shouldUseCreditsAuth, } from '@meetopenbot/plugin-sdk';
-function googleChat(modelId) {
-    return createOpenAI({
-        baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai',
-        apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY,
-    }).chat(modelId);
-}
-/**
- * OpenAI-compatible Chat Completions. Gemini's gateway only speaks this.
- * Do not use for OpenAI itself — gpt-5.6-luna rejects function tools on
- * `/v1/chat/completions` unless `reasoning_effort` is `none`.
- */
-function openAiChatModel(options, modelId) {
-    return createOpenAI(options).chat(modelId);
-}
+import { CREDITS_API_KEY_PLACEHOLDER, INTEGRATIONS_TOKEN_HEADER, creditsProviderBaseUrl, resolveByokApiKey, resolveCreditsAuthConfig, shouldUseCreditsAuth, } from '@meetopenbot/plugin-sdk';
 /** AI SDK default: OpenAI Responses API (`/v1/responses`), required for tools on Luna. */
 function openAiResponsesModel(options, modelId) {
     return createOpenAI(options)(modelId);
 }
+/** `@ai-sdk/google` only reads `GOOGLE_GENERATIVE_AI_API_KEY`; accept `GEMINI_API_KEY` too. */
+function googleByok(modelId) {
+    const apiKey = resolveByokApiKey('google');
+    return apiKey ? createGoogle({ apiKey })(modelId) : defaultGoogle(modelId);
+}
 function resolveCreditsProvider(provider, modelId) {
     const config = resolveCreditsAuthConfig();
     if (!config) {
@@ -37,7 +29,7 @@ function resolveCreditsProvider(provider, modelId) {
         case 'anthropic':
             return createAnthropic(options)(modelId);
         case 'google':
-            return openAiChatModel(options, modelId);
+            return createGoogle(options)(modelId);
         case 'deepseek':
             return createDeepSeek(options)(modelId);
     }
@@ -56,7 +48,7 @@ export function resolveModel(modelString, options) {
         case 'anthropic':
             return useCredits ? resolveCreditsProvider('anthropic', modelId) : anthropic(modelId);
         case 'google':
-            return useCredits ? resolveCreditsProvider('google', modelId) : googleChat(modelId);
+            return useCredits ? resolveCreditsProvider('google', modelId) : googleByok(modelId);
         case 'deepseek':
             return useCredits ? resolveCreditsProvider('deepseek', modelId) : deepSeek(modelId);
         default:
@@ -66,16 +58,9 @@ export function resolveModel(modelString, options) {
 function providerOf(modelString) {
     return expandModelString(modelString).split('/')[0] ?? '';
 }
-/** Low thinking for DeepSeek (Auto). Google Chat Completions cannot round-trip thinking. */
+/** Low thinking for DeepSeek (Auto). Other providers use their SDK defaults. */
 export function reasoningForModel(modelString) {
-    const provider = providerOf(modelString);
-    if (provider === 'deepseek')
+    if (providerOf(modelString) === 'deepseek')
         return 'low';
-    if (provider === 'google')
-        return 'none';
     return undefined;
 }
-/** Gemini's OpenAI Chat path drops thinking; omit parts so tool turns stay valid. */
-export function includeReasoningInHistory(modelString) {
-    return providerOf(modelString) !== 'google';
-}

package/dist/runtime.js

@@ -1,10 +1,11 @@
 import { streamText, stepCountIs } from 'ai';
 import { eventsToModelMessages } from './history.js';
+import { addImageParts, eventsHaveImages, pickVisionModel } from './images.js';
 import { buildContext } from './context.js';
 import { OPENBOT_SYSTEM_PROMPT } from './system-prompt.js';
 import { isAuthErrorMessage, isCreditsErrorMessage } from '@meetopenbot/plugin-sdk';
 import { AUTO_MODEL_ID, isAutoModel, isRetryableModelError, listAutoModelCandidates, missingByokApiKeyMessage, } from './auto-model.js';
-import { includeReasoningInHistory, reasoningForModel, resolveModel } from './model.js';
+import { reasoningForModel, resolveModel } from './model.js';
 import { fetchModelRegistry, getProviderModelOptions, listApiKeyProviders, PROVIDER_API_KEY_LINKS, } from '@meetopenbot/plugin-sdk';
 import { errorFromStreamPart } from './stream-error.js';
 import { createAgentOutputBuffer, isOutputBoundaryPart } from './output-buffer.js';
@@ -97,11 +98,19 @@ export async function* runOpenbotTurn(args, options) {
         channelId: state.channelId ?? '',
         threadId,
     });
-    const candidates = isAutoModel(configuredModelString)
+    const hasImages = eventsHaveImages(events);
+    let candidates = isAutoModel(configuredModelString)
         ? listAutoModelCandidates({ authMode })
         : [configuredModelString];
     try {
-        const missingKey = missingByokApiKeyMessage(configuredModelString, { authMode });
+        if (hasImages && isAutoModel(configuredModelString)) {
+            const vision = pickVisionModel(authMode);
+            if (!vision) {
+                throw new Error('Looking at images needs an OpenAI, Google, or Anthropic API key. Auto\'s usual model cannot see pictures.');
+            }
+            candidates = [vision];
+        }
+        const missingKey = missingByokApiKeyMessage(candidates[0], { authMode });
         if (missingKey)
             throw new Error(missingKey);
         let lastError;
@@ -116,12 +125,28 @@ export async function* runOpenbotTurn(args, options) {
             };
             const tools = toAiTools(toolDefinitions, args, flushPendingOutput);
             try {
+                let messages = eventsToModelMessages(events);
+                if (hasImages) {
+                    messages = await addImageParts(messages, events, async (name) => {
+                        try {
+                            const file = await storage.readThreadAttachment({
+                                channelId: state.channelId,
+                                threadId,
+                                name,
+                            });
+                            return { data: new Uint8Array(file.body), mediaType: file.mimeType };
+                        }
+                        catch (error) {
+                            console.warn('[openbot] could not read attachment', name, error);
+                            return null;
+                        }
+                    });
+                    console.info('[openbot] attached images for', candidate);
+                }
                 const result = streamText({
                     model: resolveModel(candidate, { authMode }),
                     system: systemPrompt,
-                    messages: eventsToModelMessages(events, {
-                        includeReasoning: includeReasoningInHistory(candidate),
-                    }),
+                    messages,
                     tools,
                     abortSignal,
                     reasoning: reasoningForModel(candidate),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meetopenbot/openbot",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "OpenBot coordinator runtime: ask specialists, route work into Spaces, and track todos.",
   "type": "module",
   "main": "./dist/index.js",
@@ -21,10 +21,11 @@
   "dependencies": {
     "@ai-sdk/anthropic": "^4.0.42",
     "@ai-sdk/deepseek": "^3.0.32",
+    "@ai-sdk/google": "^4.0.58",
     "@ai-sdk/openai": "^4.0.46",
     "ai": "^7.0.77",
     "zod": "^4.3.5",
-    "@meetopenbot/plugin-sdk": "^1.0.0"
+    "@meetopenbot/plugin-sdk": "^1.0.2"
   },
   "devDependencies": {
     "@types/node": "^20.10.1",