@aigne/anthropic 0.11.16 → 0.12.0

package/CHANGELOG.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## [0.12.0](https://github.com/AIGNE-io/aigne-framework/compare/anthropic-v0.11.17...anthropic-v0.12.0) (2025-09-05)
+
+
+### Features
+
+* add modalities support for chat model ([#454](https://github.com/AIGNE-io/aigne-framework/issues/454)) ([70d1bf6](https://github.com/AIGNE-io/aigne-framework/commit/70d1bf631f4e711235d89c6df8ee210a19179b30))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/core bumped to 1.58.0
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.44
+
+## [0.11.17](https://github.com/AIGNE-io/aigne-framework/compare/anthropic-v0.11.16...anthropic-v0.11.17) (2025-09-01)
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/core bumped to 1.57.5
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.43
+
 ## [0.11.16](https://github.com/AIGNE-io/aigne-framework/compare/anthropic-v0.11.15...anthropic-v0.11.16) (2025-08-30)
 
 

package/lib/cjs/anthropic-chat-model.js

@@ -10,6 +10,7 @@ const logger_js_1 = require("@aigne/core/utils/logger.js");
 const model_utils_js_1 = require("@aigne/core/utils/model-utils.js");
 const stream_utils_js_1 = require("@aigne/core/utils/stream-utils.js");
 const type_utils_js_1 = require("@aigne/core/utils/type-utils.js");
+const index_js_1 = require("@aigne/platform-helpers/nodejs/index.js");
 const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
 const ajv_1 = require("ajv");
 const zod_1 = require("zod");
@@ -117,7 +118,7 @@ class AnthropicChatModel extends core_1.ChatModel {
             top_p: input.modelOptions?.topP ?? this.modelOptions?.topP,
             // TODO: make dynamic based on model https://docs.anthropic.com/en/docs/about-claude/models/all-models
             max_tokens: this.getMaxTokens(model),
-            ...convertMessages(input),
+            ...(await convertMessages(input)),
             ...convertTools({ ...input, disableParallelToolUse }),
         };
         // Claude does not support json_schema response and tool calls in the same request,
@@ -260,7 +261,7 @@ class AnthropicChatModel extends core_1.ChatModel {
     }
 }
 exports.AnthropicChatModel = AnthropicChatModel;
-function convertMessages({ messages, responseFormat, tools }) {
+async function convertMessages({ messages, responseFormat, tools }) {
     const systemMessages = [];
     const msgs = [];
     for (const msg of messages) {
@@ -288,7 +289,7 @@ function convertMessages({ messages, responseFormat, tools }) {
         else if (msg.role === "user") {
             if (!msg.content)
                 throw new Error("User message must have content");
-            msgs.push({ role: "user", content: convertContent(msg.content) });
+            msgs.push({ role: "user", content: await convertContent(msg.content) });
         }
         else if (msg.role === "agent") {
             if (msg.toolCalls?.length) {
@@ -303,7 +304,7 @@ function convertMessages({ messages, responseFormat, tools }) {
                 });
             }
             else if (msg.content) {
-                msgs.push({ role: "assistant", content: convertContent(msg.content) });
+                msgs.push({ role: "assistant", content: await convertContent(msg.content) });
             }
             else {
                 throw new Error("Agent message must have content or toolCalls");
@@ -324,13 +325,33 @@ function convertMessages({ messages, responseFormat, tools }) {
     }
     return { messages: msgs, system };
 }
-function convertContent(content) {
+async function convertContent(content) {
     if (typeof content === "string")
         return content;
     if (Array.isArray(content)) {
-        return content.map((item) => item.type === "image_url"
-            ? { type: "image", source: { type: "url", url: item.url } }
-            : { type: "text", text: item.text });
+        return Promise.all(content.map(async (item) => {
+            if (item.type === "text")
+                return { type: "text", text: item.text };
+            const media_type = core_1.ChatModel.getMimeType(item.mimeType || item.filename || "");
+            switch (item.type) {
+                case "url":
+                    return { type: "image", source: { type: "url", url: item.url } };
+                case "file":
+                    return {
+                        type: "image",
+                        source: { type: "base64", data: item.data, media_type },
+                    };
+                case "local":
+                    return {
+                        type: "image",
+                        source: {
+                            type: "base64",
+                            data: await index_js_1.nodejs.fs.readFile(item.path, "base64"),
+                            media_type,
+                        },
+                    };
+            }
+        }));
     }
     throw new Error("Invalid chat message content");
 }

package/lib/esm/anthropic-chat-model.js

@@ -4,6 +4,7 @@ import { logger } from "@aigne/core/utils/logger.js";
 import { mergeUsage } from "@aigne/core/utils/model-utils.js";
 import { agentResponseStreamToObject } from "@aigne/core/utils/stream-utils.js";
 import { checkArguments, isEmpty, isNonNullable, } from "@aigne/core/utils/type-utils.js";
+import { nodejs } from "@aigne/platform-helpers/nodejs/index.js";
 import Anthropic from "@anthropic-ai/sdk";
 import { Ajv } from "ajv";
 import { z } from "zod";
@@ -111,7 +112,7 @@ export class AnthropicChatModel extends ChatModel {
             top_p: input.modelOptions?.topP ?? this.modelOptions?.topP,
             // TODO: make dynamic based on model https://docs.anthropic.com/en/docs/about-claude/models/all-models
             max_tokens: this.getMaxTokens(model),
-            ...convertMessages(input),
+            ...(await convertMessages(input)),
             ...convertTools({ ...input, disableParallelToolUse }),
         };
         // Claude does not support json_schema response and tool calls in the same request,
@@ -253,7 +254,7 @@ export class AnthropicChatModel extends ChatModel {
         };
     }
 }
-function convertMessages({ messages, responseFormat, tools }) {
+async function convertMessages({ messages, responseFormat, tools }) {
     const systemMessages = [];
     const msgs = [];
     for (const msg of messages) {
@@ -281,7 +282,7 @@ function convertMessages({ messages, responseFormat, tools }) {
         else if (msg.role === "user") {
             if (!msg.content)
                 throw new Error("User message must have content");
-            msgs.push({ role: "user", content: convertContent(msg.content) });
+            msgs.push({ role: "user", content: await convertContent(msg.content) });
         }
         else if (msg.role === "agent") {
             if (msg.toolCalls?.length) {
@@ -296,7 +297,7 @@ function convertMessages({ messages, responseFormat, tools }) {
                 });
             }
             else if (msg.content) {
-                msgs.push({ role: "assistant", content: convertContent(msg.content) });
+                msgs.push({ role: "assistant", content: await convertContent(msg.content) });
             }
             else {
                 throw new Error("Agent message must have content or toolCalls");
@@ -317,13 +318,33 @@ function convertMessages({ messages, responseFormat, tools }) {
     }
     return { messages: msgs, system };
 }
-function convertContent(content) {
+async function convertContent(content) {
     if (typeof content === "string")
         return content;
     if (Array.isArray(content)) {
-        return content.map((item) => item.type === "image_url"
-            ? { type: "image", source: { type: "url", url: item.url } }
-            : { type: "text", text: item.text });
+        return Promise.all(content.map(async (item) => {
+            if (item.type === "text")
+                return { type: "text", text: item.text };
+            const media_type = ChatModel.getMimeType(item.mimeType || item.filename || "");
+            switch (item.type) {
+                case "url":
+                    return { type: "image", source: { type: "url", url: item.url } };
+                case "file":
+                    return {
+                        type: "image",
+                        source: { type: "base64", data: item.data, media_type },
+                    };
+                case "local":
+                    return {
+                        type: "image",
+                        source: {
+                            type: "base64",
+                            data: await nodejs.fs.readFile(item.path, "base64"),
+                            media_type,
+                        },
+                    };
+            }
+        }));
     }
     throw new Error("Invalid chat message content");
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/anthropic",
-  "version": "0.11.16",
+  "version": "0.12.0",
   "description": "AIGNE Anthropic SDK for integrating with Claude AI models",
   "publishConfig": {
     "access": "public"
@@ -38,7 +38,8 @@
     "@anthropic-ai/sdk": "^0.56.0",
     "ajv": "^8.17.1",
     "zod": "^3.25.67",
-    "@aigne/core": "^1.57.4"
+    "@aigne/core": "^1.58.0",
+    "@aigne/platform-helpers": "^0.6.2"
   },
   "devDependencies": {
     "@types/bun": "^1.2.18",
@@ -46,7 +47,7 @@
     "npm-run-all": "^4.1.5",
     "rimraf": "^6.0.1",
     "typescript": "^5.8.3",
-    "@aigne/test-utils": "^0.5.42"
+    "@aigne/test-utils": "^0.5.44"
   },
   "scripts": {
     "lint": "tsc --noEmit",