@ouro.bot/cli 0.1.0-alpha.56 → 0.1.0-alpha.560

package/dist/repertoire/tools-attachments.js

@@ -0,0 +1,317 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.attachmentToolDefinitions = void 0;
+const fs = __importStar(require("node:fs/promises"));
+const identity_1 = require("../heart/identity");
+const provider_credentials_1 = require("../heart/provider-credentials");
+const minimax_1 = require("../heart/providers/minimax");
+const minimax_vlm_1 = require("../heart/providers/minimax-vlm");
+const store_1 = require("../heart/attachments/store");
+const materialize_1 = require("../heart/attachments/materialize");
+const runtime_1 = require("../nerves/runtime");
+const tool_results_1 = require("./tool-results");
+const LIST_ATTACHMENTS_DESCRIPTION = "List the most recent attachments the harness preserved across senses. Use this when you need a stable attachment_id, " +
+    "when a repair flow says an attachment could not be found, or when you want to inspect what artifacts are currently on hand " +
+    "before asking the user to resend anything.";
+const MATERIALIZE_ATTACHMENT_DESCRIPTION = "Materialize an attachment into a concrete file path the harness can inspect or hand to other tools. " +
+    "Use variant=original when you want the raw file. Use variant=vision_safe for images when you want the harness to normalize " +
+    "size and format into a VLM-safe image before retrying.";
+const DESCRIBE_IMAGE_DESCRIPTION = "Use this to inspect an image attachment with a targeted question, or to re-interrogate an image after a wrong first answer. " +
+    "The harness materializes the original attachment, normalizes it into a VLM-safe image when needed, and sends it through the vision-language model. " +
+    "Prefer specific prompts like 'what is the confirmation number?' over 'describe this image'. " +
+    "Pass attachment_id when you have it; attachment_guid is accepted as a compatibility alias.";
+function trimArg(value) {
+    return value?.trim() ?? "";
+}
+function resolveAttachmentId(args) {
+    return trimArg(args.attachment_id) || trimArg(args.attachment_guid);
+}
+function parseLimit(raw) {
+    if (!raw)
+        return undefined;
+    const parsed = Number.parseInt(raw, 10);
+    if (!Number.isFinite(parsed) || parsed <= 0)
+        return undefined;
+    return parsed;
+}
+function normalizeThrownReason(error) {
+    const rendered = String(error);
+    return rendered.startsWith("Error: ") ? rendered.slice(7) : rendered;
+}
+function attachmentNotFound(tool, attachmentId) {
+    const friction = {
+        kind: "local_repair",
+        recoverability: "transformable",
+        summary: `Attachment ${attachmentId} is not in the recent attachment store anymore.`,
+        signature: `${tool}:attachment-not-found`,
+        suggested_next_actions: [
+            {
+                kind: "tool",
+                tool: "list_recent_attachments",
+                reason: "Find the current attachment_id for the artifact you want to inspect before retrying.",
+            },
+        ],
+    };
+    return (0, tool_results_1.frictionToolResult)(tool, friction);
+}
+function missingAttachmentId(tool) {
+    return (0, tool_results_1.frictionToolResult)(tool, {
+        kind: "input_error",
+        recoverability: "transformable",
+        summary: "An attachment id is required for this tool call.",
+        signature: `${tool}:missing-attachment-id`,
+        suggested_next_actions: [
+            {
+                kind: "tool",
+                tool: "list_recent_attachments",
+                reason: "Inspect the current recent attachments and retry with attachment_id.",
+            },
+        ],
+    });
+}
+function missingPrompt() {
+    return (0, tool_results_1.frictionToolResult)("describe_image", {
+        kind: "input_error",
+        recoverability: "transformable",
+        summary: "describe_image needs a targeted prompt before retrying.",
+        signature: "describe_image:missing-prompt",
+        suggested_next_actions: [
+            {
+                kind: "message",
+                message: "Retry with a specific question like 'what is the confirmation number?' or 'what date is shown?'",
+            },
+        ],
+    });
+}
+function normalizeMaterializeError(tool, attachmentId, reason) {
+    if (reason.startsWith("Attachment not found:")) {
+        return attachmentNotFound(tool, attachmentId);
+    }
+    if (reason.includes("is not an image")) {
+        return (0, tool_results_1.frictionToolResult)(tool, {
+            kind: "input_error",
+            recoverability: "transformable",
+            summary: `Attachment ${attachmentId} is not an image, so it cannot produce a vision_safe variant.`,
+            signature: `${tool}:not-image`,
+            suggested_next_actions: [
+                {
+                    kind: "tool",
+                    tool: "materialize_attachment",
+                    reason: "Retry with variant=original if you need the raw file instead.",
+                    args: { attachment_id: attachmentId, variant: "original" },
+                },
+            ],
+        });
+    }
+    return (0, tool_results_1.frictionToolResult)(tool, {
+        kind: "local_repair",
+        recoverability: "retryable",
+        summary: reason,
+        signature: `${tool}:materialize-failed`,
+        suggested_next_actions: [
+            {
+                kind: "tool",
+                tool: "materialize_attachment",
+                reason: "Retry materialization once more in case the file path or normalization target changed.",
+                args: { attachment_id: attachmentId, variant: "vision_safe" },
+            },
+        ],
+    });
+}
+async function buildImageDataUrl(filePath, mimeType) {
+    const buffer = await fs.readFile(filePath);
+    const encoded = buffer.toString("base64");
+    const normalizedMime = mimeType?.trim().toLowerCase() || "image/jpeg";
+    return `data:${normalizedMime};base64,${encoded}`;
+}
+exports.attachmentToolDefinitions = [
+    {
+        tool: {
+            type: "function",
+            function: {
+                name: "list_recent_attachments",
+                description: LIST_ATTACHMENTS_DESCRIPTION,
+                parameters: {
+                    type: "object",
+                    properties: {
+                        kind: {
+                            type: "string",
+                            enum: ["image", "audio", "document", "binary", "unknown"],
+                            description: "Optional attachment kind filter.",
+                        },
+                        limit: {
+                            type: "string",
+                            description: "Optional numeric limit for how many attachments to return.",
+                        },
+                    },
+                },
+            },
+        },
+        handler: async (args) => {
+            const agentName = (0, identity_1.getAgentName)();
+            const kind = trimArg(args.kind);
+            const attachments = (0, store_1.listRecentAttachments)(agentName, {
+                kind: kind ? kind : undefined,
+                limit: parseLimit(args.limit),
+            });
+            return (0, tool_results_1.okToolResult)("list_recent_attachments", attachments);
+        },
+        summaryKeys: ["kind", "limit"],
+    },
+    {
+        tool: {
+            type: "function",
+            function: {
+                name: "materialize_attachment",
+                description: MATERIALIZE_ATTACHMENT_DESCRIPTION,
+                parameters: {
+                    type: "object",
+                    properties: {
+                        attachment_id: {
+                            type: "string",
+                            description: "Stable attachment id such as attachment:cli-local-file:... or attachment:bluebubbles:...",
+                        },
+                        attachment_guid: {
+                            type: "string",
+                            description: "Compatibility alias for older callers. Prefer attachment_id.",
+                        },
+                        variant: {
+                            type: "string",
+                            enum: ["original", "vision_safe"],
+                            description: "Which attachment variant to materialize.",
+                        },
+                    },
+                    required: ["variant"],
+                },
+            },
+        },
+        handler: async (args) => {
+            const attachmentId = resolveAttachmentId(args);
+            if (!attachmentId) {
+                return missingAttachmentId("materialize_attachment");
+            }
+            const agentName = (0, identity_1.getAgentName)();
+            const variant = trimArg(args.variant) === "vision_safe" ? "vision_safe" : "original";
+            try {
+                const materialized = await (0, materialize_1.materializeAttachment)(agentName, attachmentId, { variant });
+                return (0, tool_results_1.okToolResult)("materialize_attachment", materialized);
+            }
+            catch (error) {
+                const reason = normalizeThrownReason(error);
+                return normalizeMaterializeError("materialize_attachment", attachmentId, reason);
+            }
+        },
+        summaryKeys: ["attachment_id", "attachment_guid", "variant"],
+    },
+    {
+        tool: {
+            type: "function",
+            function: {
+                name: "describe_image",
+                description: DESCRIBE_IMAGE_DESCRIPTION,
+                parameters: {
+                    type: "object",
+                    properties: {
+                        attachment_id: {
+                            type: "string",
+                            description: "Stable attachment id for the image you want to inspect.",
+                        },
+                        attachment_guid: {
+                            type: "string",
+                            description: "Compatibility alias for older callers. Prefer attachment_id.",
+                        },
+                        prompt: {
+                            type: "string",
+                            description: "A targeted question about the image.",
+                        },
+                    },
+                    required: ["prompt"],
+                },
+            },
+        },
+        handler: async (args) => {
+            const attachmentId = resolveAttachmentId(args);
+            if (!attachmentId) {
+                return missingAttachmentId("describe_image");
+            }
+            const prompt = trimArg(args.prompt);
+            if (!prompt) {
+                return missingPrompt();
+            }
+            const agentName = (0, identity_1.getAgentName)();
+            try {
+                const materialized = await (0, materialize_1.materializeAttachment)(agentName, attachmentId, { variant: "vision_safe" });
+                const imageDataUrl = await buildImageDataUrl(materialized.path, materialized.mimeType);
+                const credential = await (0, provider_credentials_1.readProviderCredentialRecord)(agentName, "minimax");
+                const apiKey = credential.ok ? credential.record.credentials.apiKey : undefined;
+                if (!apiKey) {
+                    return (0, tool_results_1.frictionToolResult)("describe_image", {
+                        kind: "external_blocker",
+                        recoverability: "blocked",
+                        summary: "MiniMax credentials are missing, so image understanding is unavailable until credentials are fixed.",
+                        signature: "describe_image:minimax-missing-key",
+                        suggested_next_actions: [
+                            {
+                                kind: "message",
+                                message: "Run `ouro auth --agent <agent> --provider minimax`, then retry describe_image.",
+                            },
+                        ],
+                    });
+                }
+                const description = await (0, minimax_vlm_1.minimaxVlmDescribe)({
+                    apiKey: String(apiKey),
+                    prompt,
+                    imageDataUrl,
+                    baseURL: minimax_1.MINIMAX_PROVIDER_BASE_URL,
+                    attachmentGuid: attachmentId,
+                    mimeType: materialized.mimeType,
+                });
+                (0, runtime_1.emitNervesEvent)({
+                    component: "tools",
+                    event: "tool.describe_image_success",
+                    message: "describe_image returned a description",
+                    meta: { attachmentId },
+                });
+                return description;
+            }
+            catch (error) {
+                const reason = normalizeThrownReason(error);
+                return normalizeMaterializeError("describe_image", attachmentId, reason);
+            }
+        },
+        summaryKeys: ["attachment_id", "attachment_guid", "prompt"],
+    },
+];