@ouro.bot/cli 0.1.0-alpha.250 → 0.1.0-alpha.251

package/changelog.json

@@ -1,6 +1,16 @@
 {
   "_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
   "versions": [
+    {
+      "version": "0.1.0-alpha.251",
+      "changes": [
+        "fix(bluebubbles): images sent via iMessage now reach the model — adds capability-gated image hydration with a MiniMax VLM fallback. Reasoning MiniMax chat models (M2/M2.1/M2.5/M2.7) silently drop OpenAI-style `image_url` content parts, so previously slugger answered image questions from fabricated context. Now, when the active chat model lacks the new `vision: true` capability flag, inbound screenshots are auto-described at ingestion via `/v1/coding_plan/vlm` and the description text replaces the `image_url` part before the turn reaches the model. Vision-capable models (claude-opus/sonnet-4-6, gpt-5.4, MiniMax-Text-01, MiniMax-VL-01) continue to see images natively via pass-through.",
+        "feat(tools): new `describe_image` agent tool registered into the BlueBubbles tool set when the chat model lacks vision. Lets the agent re-interrogate an attachment with a targeted prompt (e.g. 'what's the flight number in the bottom-right?') after ingestion. Backed by a bounded in-memory attachment cache populated during hydration; handler re-downloads bytes and calls the same VLM client.",
+        "fix(bluebubbles): `formatMessageText` now preserves the attachment marker when a message has BOTH text and attachments (B2). Previously the marker was dropped whenever text was present, hiding attachments from the agent's view of the message.",
+        "feat(heart): `ModelCapabilities` gains `vision?: boolean` and `audio?: boolean` flags (B4). Vision rows populated for claude-opus-4-6, claude-sonnet-4-6, claude-opus-4.6, claude-sonnet-4.6, gpt-5.4, MiniMax-Text-01, MiniMax-VL-01. M2.1/M2.5/M2.7 intentionally left unset.",
+        "fix(bluebubbles): `senses.bluebubbles_media_hydrate`, `_repair_start`, `_repair_end`, and `_inbound_logged` now emit at `level: warn` at the call site so they survive the BB sense's default log level without bumping the sense default (B7). `_event_normalized` stays at info. New `senses.bluebubbles_vision_format_unsupported` warn event fires when an inbound attachment is not in png/jpeg/webp."
+      ]
+    },
     {
       "version": "0.1.0-alpha.250",
       "changes": [

package/dist/heart/core.js

@@ -471,7 +471,7 @@ async function runAgent(messages, callbacks, channel, signal, options) {
     }
     catch { /* unsupported */ }
     const toolPreferences = currentContext?.friend?.toolPreferences;
-    const baseTools = options?.tools ?? (0, tools_1.getToolsForChannel)(channel ? (0, channel_1.getChannelCapabilities)(channel) : undefined, toolPreferences && Object.keys(toolPreferences).length > 0 ? toolPreferences : undefined, currentContext, providerRuntime.capabilities, options?.mcpManager);
+    const baseTools = options?.tools ?? (0, tools_1.getToolsForChannel)(channel ? (0, channel_1.getChannelCapabilities)(channel) : undefined, toolPreferences && Object.keys(toolPreferences).length > 0 ? toolPreferences : undefined, currentContext, providerRuntime.capabilities, options?.mcpManager, providerRuntime.model);
     // Augment tool context with reasoning effort controls from provider
     const augmentedToolContext = options?.toolContext
         ? {

package/dist/heart/model-capabilities.js

@@ -8,30 +8,41 @@ exports.MODEL_CAPABILITIES = {
         reasoningEffort: ["low", "medium", "high", "max"],
         thinkingFormat: "anthropic",
         maxOutputTokens: 128000,
+        vision: true,
     },
     "claude-sonnet-4-6": {
         reasoningEffort: ["low", "medium", "high"],
         thinkingFormat: "anthropic",
         maxOutputTokens: 64000,
+        vision: true,
     },
     "claude-opus-4.6": {
         reasoningEffort: ["low", "medium", "high", "max"],
         maxOutputTokens: 128000,
+        vision: true,
     },
     "claude-sonnet-4.6": {
         reasoningEffort: ["low", "medium", "high"],
         maxOutputTokens: 64000,
+        vision: true,
     },
     "gpt-5.4": {
         reasoningEffort: ["low", "medium", "high"],
         phase: true,
         maxOutputTokens: 100000,
+        vision: true,
     },
     "gpt-5.3-codex": {
         reasoningEffort: ["low", "medium", "high"],
         phase: true,
         maxOutputTokens: 100000,
     },
+    "MiniMax-Text-01": {
+        vision: true,
+    },
+    "MiniMax-VL-01": {
+        vision: true,
+    },
 };
 const EMPTY_CAPABILITIES = Object.freeze({});
 function getModelCapabilities(modelId) {

package/dist/heart/providers/minimax-vlm.js

@@ -0,0 +1,167 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.minimaxVlmDescribe = minimaxVlmDescribe;
+const runtime_1 = require("../../nerves/runtime");
+const error_classification_1 = require("./error-classification");
+const DEFAULT_TIMEOUT_MS = 60_000;
+const MODEL_NAME = "minimax-vlm";
+const SUPPORTED_DATA_URL_PATTERN = /^data:image\/(png|jpeg|webp);base64,/i;
+function buildVlmUrl(baseURL) {
+    const stripped = baseURL.replace(/\/v1\/?$/, "").replace(/\/$/, "");
+    return `${stripped}/v1/coding_plan/vlm`;
+}
+function extractMimeType(imageDataUrl) {
+    const match = /^data:([^;,]+)[;,]/.exec(imageDataUrl);
+    return match?.[1]?.toLowerCase();
+}
+function emitStart(params) {
+    (0, runtime_1.emitNervesEvent)({
+        level: "warn",
+        component: "senses",
+        event: "senses.bluebubbles_vlm_describe_start",
+        message: "minimax vlm describe start",
+        meta: {
+            attachmentGuid: params.attachmentGuid,
+            mimeType: params.mimeType,
+            promptLength: params.prompt.length,
+            model: MODEL_NAME,
+            chatModel: params.chatModel,
+        },
+    });
+}
+function emitEnd(params, descriptionLength, latencyMs) {
+    (0, runtime_1.emitNervesEvent)({
+        level: "warn",
+        component: "senses",
+        event: "senses.bluebubbles_vlm_describe_end",
+        message: "minimax vlm describe end",
+        meta: {
+            attachmentGuid: params.attachmentGuid,
+            descriptionLength,
+            latencyMs,
+            model: MODEL_NAME,
+        },
+    });
+}
+function emitError(params, reason, status, traceId) {
+    (0, runtime_1.emitNervesEvent)({
+        level: "warn",
+        component: "senses",
+        event: "senses.bluebubbles_vlm_describe_error",
+        message: "minimax vlm describe error",
+        meta: {
+            attachmentGuid: params.attachmentGuid,
+            mimeType: params.mimeType,
+            model: MODEL_NAME,
+            reason,
+            status,
+            traceId,
+        },
+    });
+}
+function throwAndEmit(params, message, status, traceId) {
+    emitError(params, message, status, traceId);
+    const err = new Error(message);
+    if (status !== undefined)
+        err.status = status;
+    if (traceId)
+        err.traceId = traceId;
+    throw err;
+}
+async function minimaxVlmDescribe(params) {
+    if (!params.apiKey) {
+        // We deliberately do NOT emit _start for param-validation errors — there's
+        // no meaningful "started a request" to pair with. Only the _error fires.
+        emitError(params, "minimax VLM: API key is empty — re-run credential setup or add a minimax key to secrets.json");
+        throw new Error("minimax VLM: API key is empty — re-run credential setup or add a minimax key to secrets.json");
+    }
+    if (!params.prompt) {
+        emitError(params, "minimax VLM: missing prompt — supply a targeted question (e.g. 'what's the flight number in the bottom-right?') and retry");
+        throw new Error("minimax VLM: missing prompt — supply a targeted question (e.g. 'what's the flight number in the bottom-right?') and retry");
+    }
+    if (!params.imageDataUrl) {
+        emitError(params, "minimax VLM: missing image data URL — verify the attachment downloaded OK or ask the user to resend the image");
+        throw new Error("minimax VLM: missing image data URL — verify the attachment downloaded OK or ask the user to resend the image");
+    }
+    if (!SUPPORTED_DATA_URL_PATTERN.test(params.imageDataUrl)) {
+        const mime = extractMimeType(params.imageDataUrl) ?? "unknown";
+        const msg = `minimax VLM: image must be a data:image/(png|jpeg|webp);base64,... URL but got ${mime} — ask the user to resend as a screenshot (png or jpeg)`;
+        emitError({ ...params, mimeType: params.mimeType ?? mime }, msg);
+        throw new Error(msg);
+    }
+    const fetchImpl = params.fetchImpl ?? fetch;
+    const timeoutMs = params.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const url = buildVlmUrl(params.baseURL);
+    emitStart(params);
+    const startTs = Date.now();
+    let response;
+    try {
+        response = await fetchImpl(url, {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${params.apiKey}`,
+                "Content-Type": "application/json",
+                "MM-API-Source": "Ouroboros",
+            },
+            body: JSON.stringify({
+                prompt: params.prompt,
+                image_url: params.imageDataUrl,
+            }),
+            signal: AbortSignal.timeout(timeoutMs),
+        });
+    }
+    catch (error) {
+        const err = (error instanceof Error ? error : new Error(String(error)));
+        if (err.name === "AbortError" || err.name === "TimeoutError") {
+            const seconds = Math.round(timeoutMs / 1000);
+            throwAndEmit(params, `minimax VLM: request timed out after ${seconds}s — retry or ask the user to resend the image`);
+        }
+        if ((0, error_classification_1.isNetworkError)(err)) {
+            throwAndEmit(params, `minimax VLM: network error talking to the vlm endpoint (${err.message}) — retry in a moment`);
+        }
+        throwAndEmit(params, `minimax VLM: unexpected transport error (${err.message}) — retry, and if it persists surface 'image understanding is unavailable' to the user`);
+    }
+    const traceId = response.headers.get("Trace-Id") ?? response.headers.get("trace-id") ?? undefined;
+    const status = response.status;
+    if (status === 401 || status === 403) {
+        throwAndEmit(params, `minimax VLM: rejected credentials (HTTP ${status}) — the minimax key for this agent is invalid, re-run credential setup or rotate the key`, status, traceId);
+    }
+    if (status === 429) {
+        throwAndEmit(params, `minimax VLM: rate limited (HTTP 429) — wait and retry in a moment`, status, traceId);
+    }
+    if (status >= 500) {
+        throwAndEmit(params, `minimax VLM: server error (HTTP ${status}) — retry, and if it persists surface 'image understanding is unavailable' to the user`, status, traceId);
+    }
+    if (!response.ok) {
+        throwAndEmit(params, `minimax VLM: unexpected HTTP ${status} — retry, and if it persists surface 'image understanding is unavailable' to the user`, status, traceId);
+    }
+    let body;
+    try {
+        body = await response.json();
+    }
+    catch (error) {
+        const err = error;
+        throwAndEmit(params, `minimax VLM: malformed response (not JSON: ${err.message}) — this is a provider bug, retry or surface 'image understanding is unavailable' to the user`, status, traceId);
+    }
+    const parsed = body;
+    const providerStatus = parsed?.base_resp?.status_code ?? 0;
+    if (providerStatus !== 0) {
+        const providerMsg = parsed?.base_resp?.status_msg ?? "unknown";
+        throwAndEmit(params, `minimax VLM: provider returned status_code=${providerStatus} (${providerMsg}) — check the minimax account or retry, and if it persists surface 'image understanding is unavailable' to the user`, status, traceId);
+    }
+    const content = parsed?.content;
+    if (typeof content !== "string" || content.length === 0) {
+        throwAndEmit(params, `minimax VLM: returned no description (empty content field) — this is a provider bug, retry or surface 'image understanding is unavailable' to the user`, status, traceId);
+    }
+    const latencyMs = Date.now() - startTs;
+    emitEnd(params, content.length, latencyMs);
+    return content;
+}
+/* v8 ignore start — module-level observability event */
+(0, runtime_1.emitNervesEvent)({
+    component: "engine",
+    event: "engine.minimax_vlm_loaded",
+    message: "minimax vlm client loaded",
+    meta: {},
+});
+/* v8 ignore stop */

package/dist/heart/providers/minimax.js

@@ -3,12 +3,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.MINIMAX_PROVIDER_BASE_URL = void 0;
 exports.classifyMinimaxError = classifyMinimaxError;
 exports.createMinimaxProviderRuntime = createMinimaxProviderRuntime;
 const openai_1 = __importDefault(require("openai"));
 const config_1 = require("../config");
 const runtime_1 = require("../../nerves/runtime");
 const error_classification_1 = require("./error-classification");
+/**
+ * Canonical MiniMax chat-completions base URL. Exported so the BlueBubbles
+ * VLM fallback (which targets a sibling endpoint under the same host) can
+ * derive its URL from one source of truth instead of hard-coding.
+ */
+exports.MINIMAX_PROVIDER_BASE_URL = "https://api.minimaxi.chat/v1";
 function classifyMinimaxError(error) {
     return (0, error_classification_1.classifyHttpError)(error);
 }
@@ -29,7 +36,7 @@ function createMinimaxProviderRuntime(model) {
     (0, model_capabilities_1.getModelCapabilities)(model);
     const client = new openai_1.default({
         apiKey: minimaxConfig.apiKey,
-        baseURL: "https://api.minimaxi.chat/v1",
+        baseURL: exports.MINIMAX_PROVIDER_BASE_URL,
         maxRetries: 0,
     });
     return {

package/dist/mind/prompt.js

@@ -371,7 +371,7 @@ function dateSection() {
     return `current date: ${today}`;
 }
 function toolsSection(channel, options, context) {
-    const channelTools = (0, tools_1.getToolsForChannel)((0, channel_1.getChannelCapabilities)(channel), undefined, context, options?.providerCapabilities);
+    const channelTools = (0, tools_1.getToolsForChannel)((0, channel_1.getChannelCapabilities)(channel), undefined, context, options?.providerCapabilities, undefined, options?.chatModel);
     const activeTools = (options?.toolChoiceRequired ?? true) ? [...channelTools, tools_1.settleTool] : channelTools;
     const list = activeTools
         .map((t) => `- ${t.function.name}: ${t.function.description}`)
@@ -997,6 +997,9 @@ function rhythmStatusSection(preReadHealth) {
  * (edit_file, write_file, shell). Used to gate scrutiny prompts.
  */
 function channelHasCodingTools(channel, providerCapabilities) {
+    // Coding tools are capability/integration-gated, not vision-gated — passing
+    // undefined for chatModel keeps describe_image out of the scan (which is
+    // BB-scoped anyway) without affecting the coding-tool presence check.
     const tools = (0, tools_1.getToolsForChannel)((0, channel_1.getChannelCapabilities)(channel), undefined, undefined, providerCapabilities);
     const codingToolNames = new Set(["edit_file", "write_file", "shell", "coding_spawn"]);
     return tools.some((t) => codingToolNames.has(t.function.name));

package/dist/repertoire/tools-bluebubbles.js

@@ -2,6 +2,22 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.bluebubblesToolDefinitions = void 0;
 const runtime_1 = require("../nerves/runtime");
+const config_1 = require("../heart/config");
+const minimax_1 = require("../heart/providers/minimax");
+const minimax_vlm_1 = require("../heart/providers/minimax-vlm");
+const attachment_cache_1 = require("../senses/bluebubbles/attachment-cache");
+const media_1 = require("../senses/bluebubbles/media");
+// AX-1: the tool description is the only instruction manual a future agent
+// gets. It must teach WHEN to reach for the tool and HOW to write a good
+// prompt. A type-signature-style "Describes an image." is a violation.
+const DESCRIBE_IMAGE_DESCRIPTION = "Use this tool to look at an attachment the user sent, or to re-interrogate an image with a targeted question. " +
+    "The current chat model can't see images natively; this tool runs the image through a vision-language model (VLM) " +
+    "and returns a text description. " +
+    "Prefer targeted prompts (\"what's the flight number in the bottom-right?\") over generic ones (\"describe this image\") — " +
+    "you'll get better answers. " +
+    "Call this when the user sent a screenshot and you need to read text from it, verify a detail, or double-check something " +
+    "the ingestion-time auto-describe summary might have missed. " +
+    "The attachment_guid comes from the attachment marker in the user's most recent message.";
 exports.bluebubblesToolDefinitions = [
     {
         tool: {
@@ -91,4 +107,95 @@ exports.bluebubblesToolDefinitions = [
         },
         summaryKeys: ["target", "threadOriginatorGuid"],
     },
+    {
+        tool: {
+            type: "function",
+            function: {
+                name: "describe_image",
+                description: DESCRIBE_IMAGE_DESCRIPTION,
+                parameters: {
+                    type: "object",
+                    properties: {
+                        attachment_guid: {
+                            type: "string",
+                            description: "The guid of the attachment to describe. Usually pulled from the attachment marker in the user's most recent message.",
+                        },
+                        prompt: {
+                            type: "string",
+                            description: "A targeted question about the image. Prefer specific prompts over generic ones — e.g. 'what's the flight number in the bottom-right?' gives better answers than 'describe this image'.",
+                        },
+                    },
+                    required: ["attachment_guid", "prompt"],
+                },
+            },
+        },
+        handler: async (args) => {
+            const attachmentGuid = typeof args.attachment_guid === "string" ? args.attachment_guid.trim() : "";
+            const prompt = typeof args.prompt === "string" ? args.prompt.trim() : "";
+            if (!attachmentGuid) {
+                (0, runtime_1.emitNervesEvent)({
+                    level: "warn",
+                    component: "tools",
+                    event: "tool.error",
+                    message: "describe_image missing attachment_guid",
+                    meta: {},
+                });
+                return "describe_image: attachment_guid is required — pass the guid from the inbound attachment marker and retry";
+            }
+            if (!prompt) {
+                (0, runtime_1.emitNervesEvent)({
+                    level: "warn",
+                    component: "tools",
+                    event: "tool.error",
+                    message: "describe_image missing prompt",
+                    meta: {},
+                });
+                return "describe_image: prompt is required — supply a targeted question (e.g. 'what's the flight number in the bottom-right?') and retry";
+            }
+            const summary = (0, attachment_cache_1.lookupBlueBubblesAttachment)(attachmentGuid);
+            if (!summary) {
+                (0, runtime_1.emitNervesEvent)({
+                    level: "warn",
+                    component: "tools",
+                    event: "tool.error",
+                    message: "describe_image attachment not found in cache",
+                    meta: { attachmentGuid },
+                });
+                return `describe_image: no attachment with guid ${attachmentGuid} found in recent messages — ask the user to resend the image or verify the guid`;
+            }
+            try {
+                const config = (0, config_1.getBlueBubblesConfig)();
+                const channelConfig = (0, config_1.getBlueBubblesChannelConfig)();
+                const downloaded = await (0, media_1.downloadBlueBubblesAttachment)(summary, config, channelConfig);
+                const mimeType = downloaded.contentType ?? summary.mimeType ?? "image/png";
+                const base64 = downloaded.buffer.toString("base64");
+                const dataUrl = `data:${mimeType};base64,${base64}`;
+                const { apiKey } = (0, config_1.getMinimaxConfig)();
+                if (!apiKey) {
+                    throw new Error("minimax API key not found in secrets.json — re-run credential setup or add a minimax key");
+                }
+                const description = await (0, minimax_vlm_1.minimaxVlmDescribe)({
+                    apiKey,
+                    prompt,
+                    imageDataUrl: dataUrl,
+                    baseURL: minimax_1.MINIMAX_PROVIDER_BASE_URL,
+                    attachmentGuid,
+                    mimeType,
+                });
+                return description;
+            }
+            catch (error) {
+                const reason = error instanceof Error ? error.message : String(error);
+                (0, runtime_1.emitNervesEvent)({
+                    level: "warn",
+                    component: "tools",
+                    event: "tool.error",
+                    message: "describe_image failed",
+                    meta: { attachmentGuid, reason },
+                });
+                return `describe_image failed: ${reason}`;
+            }
+        },
+        summaryKeys: ["attachment_guid", "prompt"],
+    },
 ];

package/dist/repertoire/tools.js

@@ -14,6 +14,7 @@ const tools_github_1 = require("./tools-github");
 const runtime_1 = require("../nerves/runtime");
 const guardrails_1 = require("./guardrails");
 const identity_1 = require("../heart/identity");
+const model_capabilities_1 = require("../heart/model-capabilities");
 const tools_surface_1 = require("./tools-surface");
 const mcp_tools_1 = require("./mcp-tools");
 function safeGetAgentRoot() {
@@ -73,10 +74,17 @@ function filterByCapability(toolList, providerCapabilities) {
 // Teams/integration tools are included only if their integration is in availableIntegrations.
 // When toolPreferences is provided, matching preferences are appended to tool descriptions.
 // When providerCapabilities is provided, tools with requiredCapability are filtered.
-function getToolsForChannel(capabilities, toolPreferences, _context, providerCapabilities, mcpManager) {
+// When chatModel is provided and has `vision: true` in ModelCapabilities, the
+// BlueBubbles-scoped `describe_image` tool is excluded (native image_url
+// pass-through makes the tool redundant). When chatModel is undefined the
+// tool IS included — the default errs toward more tools on an ambiguous caller.
+function getToolsForChannel(capabilities, toolPreferences, _context, providerCapabilities, mcpManager, chatModel) {
     const baseTools = baseToolsForCapabilities();
+    const visionCapable = chatModel ? (0, model_capabilities_1.getModelCapabilities)(chatModel).vision === true : false;
     const bluebubblesTools = capabilities?.channel === "bluebubbles"
-        ? tools_bluebubbles_1.bluebubblesToolDefinitions.map((d) => d.tool)
+        ? tools_bluebubbles_1.bluebubblesToolDefinitions
+            .filter((d) => d.tool.function.name !== "describe_image" || !visionCapable)
+            .map((d) => d.tool)
         : [];
     let result;
     if (!capabilities || capabilities.availableIntegrations.length === 0) {

package/dist/senses/bluebubbles/attachment-cache.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rememberBlueBubblesAttachment = rememberBlueBubblesAttachment;
+exports.lookupBlueBubblesAttachment = lookupBlueBubblesAttachment;
+exports.resetBlueBubblesAttachmentCache = resetBlueBubblesAttachmentCache;
+const runtime_1 = require("../../nerves/runtime");
+/**
+ * Bounded in-memory cache of recently-seen BlueBubbles attachment summaries.
+ *
+ * Populated at attachment-hydration time so the `describe_image` tool can
+ * look up a guid → summary later in the same turn (or a few turns later)
+ * and re-download the bytes on demand. Intentionally NOT persisted — per
+ * planning doc D4, session storage holds only the VLM description text,
+ * never the raw image bytes. The cache is cleared on daemon restart, which
+ * matches the product expectation that "describe_image works on recent
+ * messages in this session".
+ *
+ * Bounded at MAX_CACHED_ATTACHMENTS entries; oldest entries evict first
+ * when the limit is hit.
+ */
+const MAX_CACHED_ATTACHMENTS = 50;
+const cache = new Map();
+function rememberBlueBubblesAttachment(summary) {
+    const guid = summary.guid?.trim();
+    if (!guid)
+        return;
+    // Re-insert to move to end (LRU behavior via Map insertion order).
+    if (cache.has(guid))
+        cache.delete(guid);
+    cache.set(guid, { ...summary });
+    while (cache.size > MAX_CACHED_ATTACHMENTS) {
+        // cache.size > 0 here, so keys().next().value is always defined.
+        const oldestKey = cache.keys().next().value;
+        cache.delete(oldestKey);
+    }
+}
+function lookupBlueBubblesAttachment(guid) {
+    const trimmed = guid?.trim();
+    if (!trimmed)
+        return undefined;
+    return cache.get(trimmed);
+}
+function resetBlueBubblesAttachmentCache() {
+    cache.clear();
+}
+/* v8 ignore start — module-level observability event */
+(0, runtime_1.emitNervesEvent)({
+    component: "senses",
+    event: "senses.bluebubbles_attachment_cache_loaded",
+    message: "bluebubbles attachment cache module loaded",
+    meta: { maxEntries: MAX_CACHED_ATTACHMENTS },
+});
+/* v8 ignore stop */

package/dist/senses/bluebubbles/client.js

@@ -5,6 +5,8 @@ const node_crypto_1 = require("node:crypto");
 const config_1 = require("../../heart/config");
 const identity_1 = require("../../heart/identity");
 const runtime_1 = require("../../nerves/runtime");
+const minimax_1 = require("../../heart/providers/minimax");
+const minimax_vlm_1 = require("../../heart/providers/minimax-vlm");
 const model_1 = require("./model");
 const media_1 = require("./media");
 function buildBlueBubblesApiUrl(baseUrl, endpoint, password) {
@@ -377,6 +379,7 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
                 return event;
             }
             (0, runtime_1.emitNervesEvent)({
+                level: "warn",
                 component: "senses",
                 event: "senses.bluebubbles_repair_start",
                 message: "repairing bluebubbles event by guid",
@@ -440,8 +443,40 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
                 if (hydrated.kind === "message" &&
                     hydrated.balloonBundleId !== "com.apple.messages.URLBalloonProvider" &&
                     hydrated.attachments.length > 0) {
+                    const agentConfig = (0, identity_1.loadAgentConfig)();
+                    const chatModel = agentConfig.humanFacing.model;
+                    const chatProvider = agentConfig.humanFacing.provider;
+                    const vlmDescribe = async (params) => {
+                        if (chatProvider !== "minimax") {
+                            throw new Error("VLM fallback requires a minimax credential for this agent — " +
+                                "configure one or switch to a vision-capable chat model");
+                        }
+                        const { apiKey } = (0, config_1.getMinimaxConfig)();
+                        if (!apiKey) {
+                            throw new Error("VLM fallback: minimax API key not found in secrets.json — " +
+                                "re-run credential setup or ask the user to add a minimax key");
+                        }
+                        return (0, minimax_vlm_1.minimaxVlmDescribe)({
+                            apiKey,
+                            prompt: params.prompt,
+                            imageDataUrl: params.imageDataUrl,
+                            baseURL: minimax_1.MINIMAX_PROVIDER_BASE_URL,
+                            attachmentGuid: params.attachmentGuid,
+                            mimeType: params.mimeType,
+                            chatModel: params.chatModel,
+                        });
+                    };
+                    // VLM prompt context wants the user's raw inbound text — NOT
+                    // the agent-facing rendering, which now carries the attachment
+                    // marker after the B2 fix. Pull it from the source payload.
+                    const rawUserText = typeof data.text === "string"
+                        ? (data.text).trim()
+                        : "";
                     const media = await (0, media_1.hydrateBlueBubblesAttachments)(hydrated.attachments, config, channelConfig, {
-                        preferAudioInput: providerSupportsAudioInput((0, identity_1.loadAgentConfig)().humanFacing.provider),
+                        preferAudioInput: providerSupportsAudioInput(chatProvider),
+                        chatModel,
+                        vlmDescribe,
+                        userText: rawUserText,
                     });
                     const transcriptSuffix = media.transcriptAdditions.map((entry) => `[${entry}]`).join("\n");
                     const noticeSuffix = media.notices.map((entry) => `[${entry}]`).join("\n");
@@ -453,6 +488,7 @@ function createBlueBubblesClient(config = (0, config_1.getBlueBubblesConfig)(),
                     };
                 }
                 (0, runtime_1.emitNervesEvent)({
+                    level: "warn",
                     component: "senses",
                     event: "senses.bluebubbles_repair_end",
                     message: "bluebubbles event repaired",

package/dist/senses/bluebubbles/inbound-log.js

@@ -94,6 +94,7 @@ function recordBlueBubblesInbound(agentName, event, source) {
         return filePath;
     }
     (0, runtime_1.emitNervesEvent)({
+        level: "warn",
         component: "senses",
         event: "senses.bluebubbles_inbound_logged",
         message: "recorded bluebubbles inbound message to sidecar log",

package/dist/senses/bluebubbles/media.js

@@ -33,6 +33,8 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.VLM_TEXT_WRAPPERS = void 0;
+exports.downloadBlueBubblesAttachment = downloadBlueBubblesAttachment;
 exports.hydrateBlueBubblesAttachments = hydrateBlueBubblesAttachments;
 const node_child_process_1 = require("node:child_process");
 const fs = __importStar(require("node:fs/promises"));
@@ -40,6 +42,28 @@ const os = __importStar(require("node:os"));
 const path = __importStar(require("node:path"));
 const runtime_1 = require("../../nerves/runtime");
 const identity_1 = require("../../heart/identity");
+const model_capabilities_1 = require("../../heart/model-capabilities");
+const attachment_cache_1 = require("./attachment-cache");
+// Pin the exact wrapper strings so tests and code read from one source.
+exports.VLM_TEXT_WRAPPERS = {
+    description: (desc) => `[image description: ${desc}]`,
+    unsupported: (mimeType) => `[image attachment not shown: unsupported format ${mimeType}]`,
+    failure: (reason) => `[image description failed: ${reason}]`,
+};
+// VLM supported formats for the /v1/coding_plan/vlm endpoint. The client in
+// minimax-vlm.ts enforces the same set — this helper lets the sense skip the
+// call entirely and emit a clearer `vision_format_unsupported` event instead
+// of letting the VLM client throw.
+const VLM_SUPPORTED_MIME_TYPES = new Set(["image/png", "image/jpeg", "image/webp"]);
+function isSupportedVlmFormat(contentType) {
+    return VLM_SUPPORTED_MIME_TYPES.has(contentType.toLowerCase());
+}
+// D3 prompt template. Included here (not templated out) because the contract
+// is load-bearing — see planning doc section D3 and AX-4.
+function buildVlmPrompt(userText) {
+    const body = userText ?? "";
+    return `User message: "${body}"\n\nDescribe this image in detail, focusing on anything relevant to what the user said above.\nInclude any text visible in the image verbatim.`;
+}
 const MAX_ATTACHMENT_BYTES = 8 * 1024 * 1024;
 const AUDIO_EXTENSIONS = new Set([".mp3", ".wav", ".m4a", ".caf", ".ogg"]);
 const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".heif"]);
@@ -66,9 +90,16 @@ const AUDIO_INPUT_FORMAT_BY_EXTENSION = {
 const WHISPER_CPP_FORMULA = "whisper-cpp";
 const WHISPER_CPP_MODEL_NAME = "ggml-base.en.bin";
 const WHISPER_CPP_MODEL_URL = `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${WHISPER_CPP_MODEL_NAME}`;
-const WHISPER_CPP_TOOLS_DIR = path.join((0, identity_1.getAgentToolsRoot)(), "whisper-cpp");
-const WHISPER_CPP_MODELS_DIR = path.join(WHISPER_CPP_TOOLS_DIR, "models");
-const WHISPER_CPP_MODEL_PATH = path.join(WHISPER_CPP_MODELS_DIR, WHISPER_CPP_MODEL_NAME);
+// Lazy — getAgentToolsRoot() requires identity to be resolved, which isn't
+// true at module-load time in most unit test contexts. Tests that only touch
+// the image/VLM path shouldn't break because the audio path asked for a
+// tools root they don't need.
+function whisperCppPaths() {
+    const toolsDir = path.join((0, identity_1.getAgentToolsRoot)(), "whisper-cpp");
+    const modelsDir = path.join(toolsDir, "models");
+    const modelPath = path.join(modelsDir, WHISPER_CPP_MODEL_NAME);
+    return { toolsDir, modelsDir, modelPath };
+}
 function buildBlueBubblesApiUrl(baseUrl, endpoint, password) {
     const root = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
     const url = new URL(endpoint.replace(/^\//, ""), root);
@@ -170,12 +201,13 @@ async function resolveWhisperCppBinary(timeoutMs) {
     return candidate;
 }
 async function ensureWhisperCppModel(timeoutMs, fetchImpl) {
+    const { modelsDir, modelPath } = whisperCppPaths();
     try {
-        await fs.access(WHISPER_CPP_MODEL_PATH);
-        return WHISPER_CPP_MODEL_PATH;
+        await fs.access(modelPath);
+        return modelPath;
     }
     catch {
-        await fs.mkdir(WHISPER_CPP_MODELS_DIR, { recursive: true });
+        await fs.mkdir(modelsDir, { recursive: true });
         const response = await fetchImpl(WHISPER_CPP_MODEL_URL, {
             method: "GET",
             signal: AbortSignal.timeout(Math.max(timeoutMs, 300_000)),
@@ -183,8 +215,8 @@ async function ensureWhisperCppModel(timeoutMs, fetchImpl) {
         if (!response.ok) {
             throw new Error(`failed to download whisper.cpp model: HTTP ${response.status}`);
         }
-        await fs.writeFile(WHISPER_CPP_MODEL_PATH, Buffer.from(await response.arrayBuffer()));
-        return WHISPER_CPP_MODEL_PATH;
+        await fs.writeFile(modelPath, Buffer.from(await response.arrayBuffer()));
+        return modelPath;
     }
 }
 async function convertAudioForWhisperCpp(sourcePath, outputPath, timeoutMs) {
@@ -236,6 +268,9 @@ async function transcribeAudioWithWhisperCpp(params, modelFetchImpl = fetch) {
         await fs.rm(workDir, { recursive: true, force: true }).catch(() => undefined);
     }
 }
+async function downloadBlueBubblesAttachment(attachment, config, channelConfig, fetchImpl = fetch) {
+    return downloadAttachment(attachment, config, channelConfig, fetchImpl);
+}
 async function downloadAttachment(attachment, config, channelConfig, fetchImpl) {
     const guid = attachment.guid?.trim();
     if (!guid) {
@@ -262,33 +297,114 @@ async function downloadAttachment(attachment, config, channelConfig, fetchImpl)
     };
 }
 async function hydrateBlueBubblesAttachments(attachments, config, channelConfig, deps = {}) {
-    (0, runtime_1.emitNervesEvent)({
-        component: "senses",
-        event: "senses.bluebubbles_media_hydrate",
-        message: "hydrating bluebubbles attachments",
-        meta: {
-            attachmentCount: attachments.length,
-            preferAudioInput: deps.preferAudioInput ?? false,
-        },
-    });
     const fetchImpl = deps.fetchImpl ?? fetch;
     const modelFetchImpl = deps.modelFetchImpl ?? fetch;
     const transcribeAudio = deps.transcribeAudio ?? ((params) => transcribeAudioWithWhisperCpp(params, modelFetchImpl));
     const preferAudioInput = deps.preferAudioInput ?? false;
+    const chatModel = deps.chatModel;
+    const visionCapable = chatModel ? (0, model_capabilities_1.getModelCapabilities)(chatModel).vision === true : true;
     const inputParts = [];
     const transcriptAdditions = [];
     const notices = [];
     for (const attachment of attachments) {
         const name = describeAttachment(attachment);
+        // Remember every attachment we see — the describe_image agent tool looks
+        // up guids against this cache later to re-fetch bytes on demand.
+        (0, attachment_cache_1.rememberBlueBubblesAttachment)(attachment);
         try {
             const downloaded = await downloadAttachment(attachment, config, channelConfig, fetchImpl);
             const base64 = downloaded.buffer.toString("base64");
+            const byteCount = downloaded.buffer.length;
             if (isImageAttachment(attachment, downloaded.contentType)) {
-                inputParts.push({
-                    type: "image_url",
-                    image_url: {
-                        url: `data:${downloaded.contentType ?? "application/octet-stream"};base64,${base64}`,
-                        detail: "auto",
+                const mimeType = downloaded.contentType ?? "application/octet-stream";
+                if (visionCapable) {
+                    inputParts.push({
+                        type: "image_url",
+                        image_url: {
+                            url: `data:${mimeType};base64,${base64}`,
+                            detail: "auto",
+                        },
+                    });
+                    (0, runtime_1.emitNervesEvent)({
+                        level: "warn",
+                        component: "senses",
+                        event: "senses.bluebubbles_media_hydrate",
+                        message: "bluebubbles media hydrate",
+                        meta: {
+                            attachmentGuid: attachment.guid,
+                            mimeType,
+                            byteCount,
+                            hydrationPath: "native-passthrough",
+                        },
+                    });
+                    continue;
+                }
+                // Non-vision chat model — try the VLM describe fallback.
+                if (!isSupportedVlmFormat(mimeType)) {
+                    (0, runtime_1.emitNervesEvent)({
+                        level: "warn",
+                        component: "senses",
+                        event: "senses.bluebubbles_vision_format_unsupported",
+                        message: "bluebubbles vision format unsupported",
+                        meta: {
+                            mimeType,
+                            transferName: attachment.transferName,
+                            attachmentGuid: attachment.guid,
+                            chatModel,
+                        },
+                    });
+                    inputParts.push({
+                        type: "text",
+                        text: exports.VLM_TEXT_WRAPPERS.unsupported(mimeType),
+                    });
+                    (0, runtime_1.emitNervesEvent)({
+                        level: "warn",
+                        component: "senses",
+                        event: "senses.bluebubbles_media_hydrate",
+                        message: "bluebubbles media hydrate",
+                        meta: {
+                            attachmentGuid: attachment.guid,
+                            mimeType,
+                            byteCount,
+                            hydrationPath: "skip-unsupported",
+                        },
+                    });
+                    continue;
+                }
+                const dataUrl = `data:${mimeType};base64,${base64}`;
+                try {
+                    if (!deps.vlmDescribe) {
+                        throw new Error("no VLM describer configured — wire a vlmDescribe dep or configure a vision-capable chat model");
+                    }
+                    const description = await deps.vlmDescribe({
+                        prompt: buildVlmPrompt(deps.userText),
+                        imageDataUrl: dataUrl,
+                        attachmentGuid: attachment.guid,
+                        mimeType,
+                        chatModel,
+                    });
+                    inputParts.push({
+                        type: "text",
+                        text: exports.VLM_TEXT_WRAPPERS.description(description),
+                    });
+                }
+                catch (vlmError) {
+                    const reason = vlmError instanceof Error ? vlmError.message : String(vlmError);
+                    inputParts.push({
+                        type: "text",
+                        text: exports.VLM_TEXT_WRAPPERS.failure(reason),
+                    });
+                }
+                (0, runtime_1.emitNervesEvent)({
+                    level: "warn",
+                    component: "senses",
+                    event: "senses.bluebubbles_media_hydrate",
+                    message: "bluebubbles media hydrate",
+                    meta: {
+                        attachmentGuid: attachment.guid,
+                        mimeType,
+                        byteCount,
+                        hydrationPath: "vlm-describe",
                     },
                 });
                 continue;

package/dist/senses/bluebubbles/model.js

@@ -127,6 +127,13 @@ function formatMessageText(data, attachments) {
         if (balloonBundleId === "com.apple.messages.URLBalloonProvider") {
             return `${text}\n[link preview attached]`;
         }
+        // B2 fix: when text and attachments both exist, append the attachment
+        // marker so downstream senses can see the guid/filename. Previously the
+        // marker was dropped whenever text was non-empty, which hid images from
+        // the agent when the user captioned a screenshot.
+        if (attachments.length > 0) {
+            return `${text}\n${formatAttachmentText(attachments)}`;
+        }
         return text;
     }
     return formatAttachmentText(attachments);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.250",
+  "version": "0.1.0-alpha.251",
   "main": "dist/heart/daemon/ouro-entry.js",
   "bin": {
     "cli": "dist/heart/daemon/ouro-bot-entry.js",