@meetopenbot/openbot 1.2.14 → 1.2.15

package/dist/images.js

@@ -1,17 +1,32 @@
 import { BYOK_ENV_KEYS, shouldUseCreditsAuth } from '@meetopenbot/plugin-sdk';
+import { expandModelString, isAutoModel, listAutoModelCandidates, } from './auto-model.js';
 export function isImageAttachment(file) {
     if (file.mimeType?.startsWith('image/'))
         return true;
     return /\.(png|jpe?g|gif|webp)$/i.test(file.name);
 }
-export function eventsHaveImages(events) {
-    return events.some((event) => {
-        if (event.meta?.parentToolCallId)
-            return false;
-        if (event.type !== 'agent:invoke')
-            return false;
-        return Boolean(event.data?.attachments?.some(isImageAttachment));
-    });
+function topLevelUserInvokes(events) {
+    return events.filter((event) => !event.meta?.parentToolCallId &&
+        event.type === 'agent:invoke' &&
+        Boolean(event.data?.role) &&
+        invokeRole(event.data.role) === 'user');
+}
+/** True when the newest top-level user message carries an image. */
+export function latestUserMessageHasImages(events) {
+    const invokes = topLevelUserInvokes(events);
+    const latest = invokes[invokes.length - 1];
+    return Boolean(latest?.data?.attachments?.some(isImageAttachment));
+}
+/** True when any top-level user message carries an attachment. */
+export function eventsHaveAttachments(events) {
+    return topLevelUserInvokes(events).some((event) => Boolean(event.data?.attachments?.length));
+}
+/** Providers whose models in the picker cannot take image input. */
+const NO_IMAGE_INPUT_PROVIDERS = new Set(['deepseek']);
+/** Whether the model for this turn can see images. Auto counts as its concrete pool model. */
+export function modelSupportsImages(modelString, ctx) {
+    const provider = expandModelString(modelString, ctx).split('/')[0] ?? '';
+    return !NO_IMAGE_INPUT_PROVIDERS.has(provider);
 }
 function hasAnyKey(env, keys) {
     return Boolean(keys?.some((key) => Boolean(env[key]?.trim())));
@@ -33,8 +48,57 @@ export function pickVisionModel(authMode, env = process.env) {
 function invokeRole(role) {
     return role === 'agent' ? 'user' : role;
 }
-/** Put image files on matching user messages as AI SDK `file` parts. */
-export async function addImageParts(messages, events, read) {
+/**
+ * Models to try this turn. Auto switches to a cheap vision model only when the
+ * newest user message has an image and a vision model is available. Never throws.
+ */
+export function chooseTurnCandidates({ model, events, authMode, env = process.env, }) {
+    if (!isAutoModel(model))
+        return [model.trim()];
+    if (latestUserMessageHasImages(events)) {
+        const vision = pickVisionModel(authMode, env);
+        if (vision)
+            return [vision];
+    }
+    return listAutoModelCandidates({ authMode, env });
+}
+function formatSize(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    if (bytes < 1024 * 1024)
+        return `${Math.round(bytes / 1024)} KB`;
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+function fileType(file) {
+    if (file.mimeType)
+        return file.mimeType;
+    const match = /\.([a-z0-9]+)$/i.exec(file.name);
+    return match ? match[1].toLowerCase() : 'file';
+}
+/** Text note listing attached files with their full paths. */
+export function attachmentNote(files) {
+    const lines = files.map((file) => {
+        const details = [fileType(file)];
+        if (typeof file.size === 'number' && file.size > 0)
+            details.push(formatSize(file.size));
+        const where = file.path ?? "in this thread's attachments";
+        const suffix = file.imageNotShown
+            ? ' — not shown to you; the current model cannot view images'
+            : '';
+        return `- ${file.name} (${details.join(', ')}): ${where}${suffix}`;
+    });
+    return [
+        '[Attached files, saved on this computer:',
+        ...lines,
+        'You cannot open these files yourself. If a coding agent should use one, give them its full path.]',
+    ].join('\n');
+}
+/**
+ * Put attachments on matching user messages. Images go as AI SDK `file` parts
+ * when `includeImages` is set and the read succeeds; every other file (and any
+ * image not shown) becomes one text note with its full path.
+ */
+export async function addAttachmentParts(messages, events, { includeImages, read, locate, }) {
     const result = [...messages];
     let userCursor = 0;
     const takeNextUserMessageIndex = () => {
@@ -46,6 +110,23 @@ export async function addImageParts(messages, events, read) {
         }
         return -1;
     };
+    const safeRead = async (name) => {
+        try {
+            const loaded = await read(name);
+            return loaded?.data.byteLength ? loaded : null;
+        }
+        catch {
+            return null;
+        }
+    };
+    const safeLocate = async (name) => {
+        try {
+            return (await locate(name)) || null;
+        }
+        catch {
+            return null;
+        }
+    };
     for (const event of events) {
         if (event.meta?.parentToolCallId)
             continue;
@@ -58,27 +139,39 @@ export async function addImageParts(messages, events, read) {
         if (invokeRole(event.data.role) !== 'user')
             continue;
         const index = takeNextUserMessageIndex();
-        const files = attachments.filter(isImageAttachment);
-        if (index === -1 || files.length === 0)
+        if (index === -1 || attachments.length === 0)
             continue;
         const current = result[index];
         const parts = [];
         if (typeof current.content === 'string' && current.content) {
             parts.push({ type: 'text', text: current.content });
         }
-        for (const file of files) {
-            const loaded = await read(file.name);
-            if (!loaded?.data.byteLength)
-                continue;
-            parts.push({
-                type: 'file',
-                data: loaded.data,
-                mediaType: loaded.mediaType || 'image/png',
+        const noted = [];
+        for (const file of attachments) {
+            const isImage = isImageAttachment(file);
+            if (isImage && includeImages) {
+                const loaded = await safeRead(file.name);
+                if (loaded) {
+                    parts.push({
+                        type: 'file',
+                        data: loaded.data,
+                        mediaType: loaded.mediaType || 'image/png',
+                    });
+                    continue;
+                }
+            }
+            noted.push({
+                name: file.name,
+                mimeType: file.mimeType,
+                size: file.size,
+                path: await safeLocate(file.name),
+                imageNotShown: isImage,
             });
         }
-        if (parts.some((part) => part.type === 'file')) {
+        if (noted.length > 0)
+            parts.push({ type: 'text', text: attachmentNote(noted) });
+        if (parts.length > 0)
             result[index] = { role: 'user', content: parts };
-        }
     }
     return result;
 }

package/dist/runtime.js

@@ -1,10 +1,10 @@
 import { streamText, stepCountIs } from 'ai';
 import { BYOK_ENV_KEYS, isAuthErrorMessage, isCreditsErrorMessage, fetchModelRegistry, getProviderModelOptions, listApiKeyProviders, PROVIDER_API_KEY_LINKS, } from '@meetopenbot/plugin-sdk';
 import { eventsToModelMessages } from './history.js';
-import { addImageParts, eventsHaveImages, pickVisionModel } from './images.js';
+import { addAttachmentParts, chooseTurnCandidates, eventsHaveAttachments, modelSupportsImages, } from './images.js';
 import { buildContext } from './context.js';
 import { OPENBOT_SYSTEM_PROMPT, withGithubGitInstructions, withUserInstructions, } from './system-prompt.js';
-import { AUTO_MODEL_ID, isAutoModel, isRetryableModelError, listAutoModelCandidates, missingByokApiKeyMessage, } from './auto-model.js';
+import { AUTO_MODEL_ID, isAutoModel, isRetryableModelError, missingByokApiKeyMessage, } from './auto-model.js';
 import { reasoningForModel, resolveModel } from './model.js';
 import { errorFromStreamPart } from './stream-error.js';
 import { createAgentOutputBuffer, isOutputBoundaryPart } from './output-buffer.js';
@@ -100,18 +100,39 @@ export async function* runOpenbotTurn(args, options) {
         channelId: state.channelId ?? '',
         threadId,
     });
-    const hasImages = eventsHaveImages(events);
-    let candidates = isAutoModel(configuredModelString)
-        ? listAutoModelCandidates({ authMode })
-        : [configuredModelString];
-    try {
-        if (hasImages && isAutoModel(configuredModelString)) {
-            const vision = pickVisionModel(authMode);
-            if (!vision) {
-                throw new Error('Looking at images needs an OpenAI, Google, or Anthropic API key. Auto\'s usual model cannot see pictures.');
-            }
-            candidates = [vision];
+    const hasAttachments = eventsHaveAttachments(events);
+    const candidates = chooseTurnCandidates({ model: configuredModelString, events, authMode });
+    const readImage = async (name) => {
+        try {
+            const file = await storage.readThreadAttachment({
+                channelId: state.channelId,
+                threadId,
+                name,
+            });
+            return { data: new Uint8Array(file.body), mediaType: file.mimeType };
+        }
+        catch (error) {
+            console.warn('[openbot] could not read attachment', name, error);
+            return null;
+        }
+    };
+    const locateAttachment = async (name) => {
+        if (typeof storage.getThreadAttachmentStat !== 'function')
+            return null;
+        try {
+            const stat = await storage.getThreadAttachmentStat({
+                channelId: state.channelId,
+                threadId,
+                name,
+            });
+            return typeof stat?.abs === 'string' ? stat.abs : null;
+        }
+        catch (error) {
+            console.warn('[openbot] could not locate attachment', name, error);
+            return null;
         }
+    };
+    try {
         const missingKey = missingByokApiKeyMessage(candidates[0], { authMode });
         if (missingKey)
             throw new Error(missingKey);
@@ -128,22 +149,14 @@ export async function* runOpenbotTurn(args, options) {
             const tools = toAiTools(toolDefinitions, args, flushPendingOutput);
             try {
                 let messages = eventsToModelMessages(events);
-                if (hasImages) {
-                    messages = await addImageParts(messages, events, async (name) => {
-                        try {
-                            const file = await storage.readThreadAttachment({
-                                channelId: state.channelId,
-                                threadId,
-                                name,
-                            });
-                            return { data: new Uint8Array(file.body), mediaType: file.mimeType };
-                        }
-                        catch (error) {
-                            console.warn('[openbot] could not read attachment', name, error);
-                            return null;
-                        }
+                if (hasAttachments) {
+                    const includeImages = modelSupportsImages(candidate, { authMode });
+                    messages = await addAttachmentParts(messages, events, {
+                        includeImages,
+                        read: readImage,
+                        locate: locateAttachment,
                     });
-                    console.info('[openbot] attached images for', candidate);
+                    console.info('[openbot] attached files for', candidate, { images: includeImages });
                 }
                 const result = streamText({
                     model: resolveModel(candidate, { authMode }),

package/dist/system-prompt.js

@@ -50,6 +50,13 @@ export const OPENBOT_SYSTEM_PROMPT = [
     "- **Hub-and-spoke**: Specialists cannot talk to each other. You pass context from one to the next.",
     "- **Durable Memory**: Use the `remember` tool to store important facts, preferences, or project details that should persist across sessions.",
     "",
+    "# ATTACHMENTS",
+    "- Files the human attaches are saved on this computer. Their message lists each one with its full path.",
+    "- You see an image only when it is shown to you. You cannot open other files (PDF, documents, spreadsheets, archives, code).",
+    "- Never claim you read a file or saw an image you were not shown.",
+    "- When a coding agent should use a file, `ask_agent` them with the exact full path(s) and ask them to open the file there. Do not paraphrase or shorten the path.",
+    "- If an image was not shown to you and no specialist fits, say the current model cannot view images and suggest picking a vision-capable model.",
+    "",
     "# CLOUD COMPUTER",
     "This runtime is a remote cloud computer. The human is in the browser at openbot.one and cannot open localhost, 127.0.0.1, or any port on this machine.",
     "- Never tell them to visit localhost. They need a public HTTPS URL.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meetopenbot/openbot",
-  "version": "1.2.14",
+  "version": "1.2.15",
   "description": "OpenBot coordinator runtime: ask specialists, route work into Spaces, and track todos.",
   "type": "module",
   "main": "./dist/index.js",