@meetopenbot/openbot 1.2.13 → 1.2.15

package/dist/build-tools.js

@@ -5,6 +5,7 @@ import { askAgentTools } from "./tools/ask-agent.js";
 import { installAgentTools } from "./tools/install-agent.js";
 import { threadStatusTools } from "./tools/thread-status.js";
 import { threadTitleTools } from "./tools/thread-title.js";
+import { previewTools } from "./tools/preview.js";
 import { wrapToolsWithApproval } from "./tools/approval.js";
 export const DEFAULT_APPROVAL_ACTIONS = ["update_channel_spec"];
 /**
@@ -20,6 +21,7 @@ export function buildOpenbotTools(args) {
         ...installAgentTools,
         ...threadStatusTools,
         ...threadTitleTools,
+        ...previewTools,
         ...args.context.tools,
     };
     const approvalConfig = args.context.config?.approval;

package/dist/images.js

@@ -1,17 +1,32 @@
 import { BYOK_ENV_KEYS, shouldUseCreditsAuth } from '@meetopenbot/plugin-sdk';
+import { expandModelString, isAutoModel, listAutoModelCandidates, } from './auto-model.js';
 export function isImageAttachment(file) {
     if (file.mimeType?.startsWith('image/'))
         return true;
     return /\.(png|jpe?g|gif|webp)$/i.test(file.name);
 }
-export function eventsHaveImages(events) {
-    return events.some((event) => {
-        if (event.meta?.parentToolCallId)
-            return false;
-        if (event.type !== 'agent:invoke')
-            return false;
-        return Boolean(event.data?.attachments?.some(isImageAttachment));
-    });
+function topLevelUserInvokes(events) {
+    return events.filter((event) => !event.meta?.parentToolCallId &&
+        event.type === 'agent:invoke' &&
+        Boolean(event.data?.role) &&
+        invokeRole(event.data.role) === 'user');
+}
+/** True when the newest top-level user message carries an image. */
+export function latestUserMessageHasImages(events) {
+    const invokes = topLevelUserInvokes(events);
+    const latest = invokes[invokes.length - 1];
+    return Boolean(latest?.data?.attachments?.some(isImageAttachment));
+}
+/** True when any top-level user message carries an attachment. */
+export function eventsHaveAttachments(events) {
+    return topLevelUserInvokes(events).some((event) => Boolean(event.data?.attachments?.length));
+}
+/** Providers whose models in the picker cannot take image input. */
+const NO_IMAGE_INPUT_PROVIDERS = new Set(['deepseek']);
+/** Whether the model for this turn can see images. Auto counts as its concrete pool model. */
+export function modelSupportsImages(modelString, ctx) {
+    const provider = expandModelString(modelString, ctx).split('/')[0] ?? '';
+    return !NO_IMAGE_INPUT_PROVIDERS.has(provider);
 }
 function hasAnyKey(env, keys) {
     return Boolean(keys?.some((key) => Boolean(env[key]?.trim())));
@@ -33,8 +48,57 @@ export function pickVisionModel(authMode, env = process.env) {
 function invokeRole(role) {
     return role === 'agent' ? 'user' : role;
 }
-/** Put image files on matching user messages as AI SDK `file` parts. */
-export async function addImageParts(messages, events, read) {
+/**
+ * Models to try this turn. Auto switches to a cheap vision model only when the
+ * newest user message has an image and a vision model is available. Never throws.
+ */
+export function chooseTurnCandidates({ model, events, authMode, env = process.env, }) {
+    if (!isAutoModel(model))
+        return [model.trim()];
+    if (latestUserMessageHasImages(events)) {
+        const vision = pickVisionModel(authMode, env);
+        if (vision)
+            return [vision];
+    }
+    return listAutoModelCandidates({ authMode, env });
+}
+function formatSize(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    if (bytes < 1024 * 1024)
+        return `${Math.round(bytes / 1024)} KB`;
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+function fileType(file) {
+    if (file.mimeType)
+        return file.mimeType;
+    const match = /\.([a-z0-9]+)$/i.exec(file.name);
+    return match ? match[1].toLowerCase() : 'file';
+}
+/** Text note listing attached files with their full paths. */
+export function attachmentNote(files) {
+    const lines = files.map((file) => {
+        const details = [fileType(file)];
+        if (typeof file.size === 'number' && file.size > 0)
+            details.push(formatSize(file.size));
+        const where = file.path ?? "in this thread's attachments";
+        const suffix = file.imageNotShown
+            ? ' — not shown to you; the current model cannot view images'
+            : '';
+        return `- ${file.name} (${details.join(', ')}): ${where}${suffix}`;
+    });
+    return [
+        '[Attached files, saved on this computer:',
+        ...lines,
+        'You cannot open these files yourself. If a coding agent should use one, give them its full path.]',
+    ].join('\n');
+}
+/**
+ * Put attachments on matching user messages. Images go as AI SDK `file` parts
+ * when `includeImages` is set and the read succeeds; every other file (and any
+ * image not shown) becomes one text note with its full path.
+ */
+export async function addAttachmentParts(messages, events, { includeImages, read, locate, }) {
     const result = [...messages];
     let userCursor = 0;
     const takeNextUserMessageIndex = () => {
@@ -46,6 +110,23 @@ export async function addImageParts(messages, events, read) {
         }
         return -1;
     };
+    const safeRead = async (name) => {
+        try {
+            const loaded = await read(name);
+            return loaded?.data.byteLength ? loaded : null;
+        }
+        catch {
+            return null;
+        }
+    };
+    const safeLocate = async (name) => {
+        try {
+            return (await locate(name)) || null;
+        }
+        catch {
+            return null;
+        }
+    };
     for (const event of events) {
         if (event.meta?.parentToolCallId)
             continue;
@@ -58,27 +139,39 @@ export async function addImageParts(messages, events, read) {
         if (invokeRole(event.data.role) !== 'user')
             continue;
         const index = takeNextUserMessageIndex();
-        const files = attachments.filter(isImageAttachment);
-        if (index === -1 || files.length === 0)
+        if (index === -1 || attachments.length === 0)
             continue;
         const current = result[index];
         const parts = [];
         if (typeof current.content === 'string' && current.content) {
             parts.push({ type: 'text', text: current.content });
         }
-        for (const file of files) {
-            const loaded = await read(file.name);
-            if (!loaded?.data.byteLength)
-                continue;
-            parts.push({
-                type: 'file',
-                data: loaded.data,
-                mediaType: loaded.mediaType || 'image/png',
+        const noted = [];
+        for (const file of attachments) {
+            const isImage = isImageAttachment(file);
+            if (isImage && includeImages) {
+                const loaded = await safeRead(file.name);
+                if (loaded) {
+                    parts.push({
+                        type: 'file',
+                        data: loaded.data,
+                        mediaType: loaded.mediaType || 'image/png',
+                    });
+                    continue;
+                }
+            }
+            noted.push({
+                name: file.name,
+                mimeType: file.mimeType,
+                size: file.size,
+                path: await safeLocate(file.name),
+                imageNotShown: isImage,
             });
         }
-        if (parts.some((part) => part.type === 'file')) {
+        if (noted.length > 0)
+            parts.push({ type: 'text', text: attachmentNote(noted) });
+        if (parts.length > 0)
             result[index] = { role: 'user', content: parts };
-        }
     }
     return result;
 }

package/dist/index.js

@@ -10,6 +10,7 @@ import { askAgentTools } from './tools/ask-agent.js';
 import { installAgentTools } from './tools/install-agent.js';
 import { threadStatusTools } from './tools/thread-status.js';
 import { threadTitleTools } from './tools/thread-title.js';
+import { previewTools } from './tools/preview.js';
 import { buildOpenbotTools } from './build-tools.js';
 export const OPENBOT_PLUGIN_ID = '@meetopenbot/openbot';
 const modelField = await modelConfigField({
@@ -49,6 +50,7 @@ export const openbotPlugin = defineOpenbotPlugin({
         ...installAgentTools,
         ...threadStatusTools,
         ...threadTitleTools,
+        ...previewTools,
     },
     async *run(args) {
         const { context } = args;

package/dist/runtime.js

@@ -1,10 +1,10 @@
 import { streamText, stepCountIs } from 'ai';
 import { BYOK_ENV_KEYS, isAuthErrorMessage, isCreditsErrorMessage, fetchModelRegistry, getProviderModelOptions, listApiKeyProviders, PROVIDER_API_KEY_LINKS, } from '@meetopenbot/plugin-sdk';
 import { eventsToModelMessages } from './history.js';
-import { addImageParts, eventsHaveImages, pickVisionModel } from './images.js';
+import { addAttachmentParts, chooseTurnCandidates, eventsHaveAttachments, modelSupportsImages, } from './images.js';
 import { buildContext } from './context.js';
 import { OPENBOT_SYSTEM_PROMPT, withGithubGitInstructions, withUserInstructions, } from './system-prompt.js';
-import { AUTO_MODEL_ID, isAutoModel, isRetryableModelError, listAutoModelCandidates, missingByokApiKeyMessage, } from './auto-model.js';
+import { AUTO_MODEL_ID, isAutoModel, isRetryableModelError, missingByokApiKeyMessage, } from './auto-model.js';
 import { reasoningForModel, resolveModel } from './model.js';
 import { errorFromStreamPart } from './stream-error.js';
 import { createAgentOutputBuffer, isOutputBoundaryPart } from './output-buffer.js';
@@ -100,18 +100,39 @@ export async function* runOpenbotTurn(args, options) {
         channelId: state.channelId ?? '',
         threadId,
     });
-    const hasImages = eventsHaveImages(events);
-    let candidates = isAutoModel(configuredModelString)
-        ? listAutoModelCandidates({ authMode })
-        : [configuredModelString];
-    try {
-        if (hasImages && isAutoModel(configuredModelString)) {
-            const vision = pickVisionModel(authMode);
-            if (!vision) {
-                throw new Error('Looking at images needs an OpenAI, Google, or Anthropic API key. Auto\'s usual model cannot see pictures.');
-            }
-            candidates = [vision];
+    const hasAttachments = eventsHaveAttachments(events);
+    const candidates = chooseTurnCandidates({ model: configuredModelString, events, authMode });
+    const readImage = async (name) => {
+        try {
+            const file = await storage.readThreadAttachment({
+                channelId: state.channelId,
+                threadId,
+                name,
+            });
+            return { data: new Uint8Array(file.body), mediaType: file.mimeType };
+        }
+        catch (error) {
+            console.warn('[openbot] could not read attachment', name, error);
+            return null;
+        }
+    };
+    const locateAttachment = async (name) => {
+        if (typeof storage.getThreadAttachmentStat !== 'function')
+            return null;
+        try {
+            const stat = await storage.getThreadAttachmentStat({
+                channelId: state.channelId,
+                threadId,
+                name,
+            });
+            return typeof stat?.abs === 'string' ? stat.abs : null;
+        }
+        catch (error) {
+            console.warn('[openbot] could not locate attachment', name, error);
+            return null;
         }
+    };
+    try {
         const missingKey = missingByokApiKeyMessage(candidates[0], { authMode });
         if (missingKey)
             throw new Error(missingKey);
@@ -128,22 +149,14 @@ export async function* runOpenbotTurn(args, options) {
             const tools = toAiTools(toolDefinitions, args, flushPendingOutput);
             try {
                 let messages = eventsToModelMessages(events);
-                if (hasImages) {
-                    messages = await addImageParts(messages, events, async (name) => {
-                        try {
-                            const file = await storage.readThreadAttachment({
-                                channelId: state.channelId,
-                                threadId,
-                                name,
-                            });
-                            return { data: new Uint8Array(file.body), mediaType: file.mimeType };
-                        }
-                        catch (error) {
-                            console.warn('[openbot] could not read attachment', name, error);
-                            return null;
-                        }
+                if (hasAttachments) {
+                    const includeImages = modelSupportsImages(candidate, { authMode });
+                    messages = await addAttachmentParts(messages, events, {
+                        includeImages,
+                        read: readImage,
+                        locate: locateAttachment,
                     });
-                    console.info('[openbot] attached images for', candidate);
+                    console.info('[openbot] attached files for', candidate, { images: includeImages });
                 }
                 const result = streamText({
                     model: resolveModel(candidate, { authMode }),

package/dist/system-prompt.js

@@ -50,11 +50,19 @@ export const OPENBOT_SYSTEM_PROMPT = [
     "- **Hub-and-spoke**: Specialists cannot talk to each other. You pass context from one to the next.",
     "- **Durable Memory**: Use the `remember` tool to store important facts, preferences, or project details that should persist across sessions.",
     "",
+    "# ATTACHMENTS",
+    "- Files the human attaches are saved on this computer. Their message lists each one with its full path.",
+    "- You see an image only when it is shown to you. You cannot open other files (PDF, documents, spreadsheets, archives, code).",
+    "- Never claim you read a file or saw an image you were not shown.",
+    "- When a coding agent should use a file, `ask_agent` them with the exact full path(s) and ask them to open the file there. Do not paraphrase or shorten the path.",
+    "- If an image was not shown to you and no specialist fits, say the current model cannot view images and suggest picking a vision-capable model.",
+    "",
     "# CLOUD COMPUTER",
     "This runtime is a remote cloud computer. The human is in the browser at openbot.one and cannot open localhost, 127.0.0.1, or any port on this machine.",
     "- Never tell them to visit localhost. They need a public HTTPS URL.",
     "- When you ask a coding agent to run a dev server, preview, or any local process the human should open, tell them to expose it with `cloudflared tunnel --url http://127.0.0.1:<port>` and to add `cloudflared` as a **project devDependency** (not a global install — this image is ephemeral and only workspace files persist).",
-    "- Share the `https://*.trycloudflare.com` URL they print. Quick tunnels change on restart — pass the new one through. If they report a localhost URL, ask them to tunnel it.",
+    "- Share the `https://*.trycloudflare.com` URL they print, and call `open_preview` with it so it opens in the space browser. Quick tunnels change on restart — pass the new one through. If they report a localhost URL, ask them to tunnel it.",
+    "- For any other public page the human should see, call `open_preview` with that http(s) URL too. Never pass localhost, 127.0.0.1, or a port on this machine.",
     "",
     "# SPACE VARIABLES",
     "- Secrets live in OpenBot variables, not in chat. Workspace-wide keys: Settings → Variables. Per-space keys: Space menu → Variables (override workspace keys with the same name for shell commands in that space).",

package/dist/tools/ask-agent.js

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import { randomUUID } from "node:crypto";
 import { loadAgentModelCatalogs, validateAskModel } from "../agent-models.js";
+import { previewEventsForTunnelUrls } from "./preview.js";
 /**
  * `ask_agent` — OpenBot sends a message to another agent in its own harness.
  * Only the orchestrator may ask. Child events stream into this thread so the
@@ -103,6 +104,8 @@ async function runAskedAgent(ctx, args) {
             resolveNext = null;
         }
     });
+    const seenPreviewUrls = new Set();
+    const threadId = ctx.threadId ?? ctx.state.threadId;
     while (!isFinished || eventQueue.length > 0) {
         if (eventQueue.length === 0) {
             await new Promise((r) => {
@@ -110,7 +113,14 @@ async function runAskedAgent(ctx, args) {
             });
         }
         while (eventQueue.length > 0) {
-            await ctx.emit(eventQueue.shift());
+            const nextEvent = eventQueue.shift();
+            await ctx.emit(nextEvent);
+            for (const preview of previewEventsForTunnelUrls(nextEvent, seenPreviewUrls, {
+                agentId: ctx.agentId,
+                threadId,
+            })) {
+                await ctx.emit(preview);
+            }
         }
     }
     await runPromise;

package/dist/tools/preview.js

@@ -0,0 +1,129 @@
+import z from 'zod';
+const LOCAL_HOST = /^(?:localhost|.*\.localhost|0\.0\.0\.0|::1|127(?:\.\d{1,3}){3})$/i;
+function isLocalHostname(hostname) {
+    const name = hostname.toLowerCase().replace(/\.$/, '');
+    return LOCAL_HOST.test(name);
+}
+/** Public http(s) URL the human's browser can open, or null. */
+export function publicPreviewUrl(input) {
+    const trimmed = input.trim();
+    if (!trimmed)
+        return null;
+    const withScheme = /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;
+    let url;
+    try {
+        url = new URL(withScheme);
+    }
+    catch {
+        return null;
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:')
+        return null;
+    if (url.username || url.password)
+        return null;
+    if (isLocalHostname(url.hostname))
+        return null;
+    return url.href;
+}
+const TUNNEL_URL = /https:\/\/[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*\.trycloudflare\.com(?::\d+)?(?:\/[^\s<>"'`)\\\]]*)?/gi;
+const MAX_SCAN_CHARS = 250000;
+function cleanMatch(raw) {
+    return raw.replace(/[.,;:!?)]+$/g, '');
+}
+/** Public Cloudflare quick-tunnel URLs mentioned in specialist output. */
+export function tunnelPreviewUrls(value) {
+    let text;
+    try {
+        text = typeof value === 'string' ? value : JSON.stringify(value) ?? '';
+    }
+    catch {
+        return [];
+    }
+    if (!text)
+        return [];
+    if (text.length > MAX_SCAN_CHARS)
+        text = text.slice(0, MAX_SCAN_CHARS);
+    const seen = new Set();
+    const urls = [];
+    for (const match of text.matchAll(TUNNEL_URL)) {
+        const url = publicPreviewUrl(cleanMatch(match[0]));
+        if (!url || seen.has(url))
+            continue;
+        seen.add(url);
+        urls.push(url);
+    }
+    return urls;
+}
+export function previewWidgetEvent(args) {
+    return {
+        type: 'client:ui:widget',
+        data: {
+            widgetId: `preview:${args.url}`,
+            kind: 'message',
+            title: 'Preview',
+            body: args.url,
+            display: 'expanded',
+            state: 'open',
+            actions: [
+                {
+                    id: 'open_preview',
+                    label: 'Open in browser',
+                    variant: 'primary',
+                    value: { previewUrl: args.url },
+                },
+            ],
+            metadata: { autoOpen: true },
+        },
+        meta: {
+            agentId: args.agentId,
+            ...(args.threadId ? { threadId: args.threadId } : {}),
+        },
+    };
+}
+/** Preview widgets for tunnel URLs in `value` that `seen` has not emitted yet. */
+export function previewEventsForTunnelUrls(value, seen, meta) {
+    const events = [];
+    for (const url of tunnelPreviewUrls(value)) {
+        if (seen.has(url))
+            continue;
+        seen.add(url);
+        events.push(previewWidgetEvent({ url, ...meta }));
+    }
+    return events;
+}
+async function openPreview(rawArgs, ctx) {
+    const channelId = ctx.channelId ?? ctx.state.channelId ?? '';
+    const threadId = ctx.threadId ?? ctx.state.threadId;
+    if (!channelId || !threadId) {
+        const message = 'Missing channelId or threadId for open_preview';
+        return { success: false, error: message, output: message };
+    }
+    const data = (rawArgs ?? {});
+    const rawUrl = typeof data.url === 'string' ? data.url : '';
+    const url = publicPreviewUrl(rawUrl);
+    if (!url) {
+        const message = 'Pass a public http(s) URL. Localhost is not reachable from the browser — tunnel it first.';
+        return { success: false, error: message, output: message };
+    }
+    await ctx.emit(previewWidgetEvent({
+        url,
+        agentId: ctx.agentId,
+        threadId,
+    }));
+    return { success: true, url, output: `Opened ${url} in the space browser.` };
+}
+export const toolDefinitions = {
+    open_preview: {
+        description: 'Open a public http(s) URL in the space browser beside the chat. Call this with a preview or tunnel URL the human should see, including https://*.trycloudflare.com. Never pass localhost, 127.0.0.1, or any address on this machine.',
+        inputSchema: z.object({
+            url: z.string().min(1).describe('Public http(s) URL to open in the space browser.'),
+        }),
+    },
+};
+export const tools = {
+    open_preview: {
+        ...toolDefinitions.open_preview,
+        execute: openPreview,
+    },
+};
+export const previewTools = tools;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meetopenbot/openbot",
-  "version": "1.2.13",
+  "version": "1.2.15",
   "description": "OpenBot coordinator runtime: ask specialists, route work into Spaces, and track todos.",
   "type": "module",
   "main": "./dist/index.js",