@chatluna/v1-shared-adapter 1.0.4 → 1.0.5

package/lib/index.cjs

@@ -27,6 +27,7 @@ __export(index_exports, {
   convertMessageToMessageChunk: () => convertMessageToMessageChunk,
   createEmbeddings: () => createEmbeddings,
   createRequestContext: () => createRequestContext,
+  fetchImageUrl: () => fetchImageUrl,
   formatToolToOpenAITool: () => formatToolToOpenAITool,
   formatToolsToOpenAITools: () => formatToolsToOpenAITools,
   getModelMaxContextSize: () => getModelMaxContextSize,
@@ -38,6 +39,7 @@ __export(index_exports, {
   processReasoningContent: () => processReasoningContent,
   processResponse: () => processResponse,
   processStreamResponse: () => processStreamResponse,
+  removeAdditionalProperties: () => removeAdditionalProperties,
   supportImageInput: () => supportImageInput
 });
 module.exports = __toCommonJS(index_exports);
@@ -135,12 +137,13 @@ var import_sse = require("koishi-plugin-chatluna/utils/sse");
 // src/utils.ts
 var import_messages = require("@langchain/core/messages");
 var import_zod_to_json_schema = require("zod-to-json-schema");
-function langchainMessageToOpenAIMessage(messages, model, supportImageInput2, removeSystemMessage) {
+var import_string = require("koishi-plugin-chatluna/utils/string");
+async function langchainMessageToOpenAIMessage(messages, plugin, model, supportImageInput2, removeSystemMessage) {
   const result = [];
   for (const rawMessage of messages) {
     const role = messageTypeToOpenAIRole(rawMessage.getType());
     const msg = {
-      content: rawMessage.content || null,
+      content: rawMessage.content,
       name: role === "assistant" || role === "tool" ? rawMessage.name : void 0,
       role,
       //  function_call: rawMessage.additional_kwargs.function_call,
@@ -180,6 +183,24 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput2, re
           }
         });
       }
+    } else if (Array.isArray(msg.content) && msg.content.length > 0) {
+      msg.content = await Promise.all(
+        msg.content.map(async (content) => {
+          if (!(0, import_string.isMessageContentImageUrl)(content)) return content;
+          try {
+            const url = await fetchImageUrl(plugin, content);
+            return {
+              type: "image_url",
+              image_url: {
+                url,
+                detail: "low"
+              }
+            };
+          } catch {
+            return content;
+          }
+        })
+      );
     }
     result.push(msg);
   }
@@ -226,6 +247,17 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput2, re
   return result;
 }
 __name(langchainMessageToOpenAIMessage, "langchainMessageToOpenAIMessage");
+async function fetchImageUrl(plugin, content) {
+  const url = typeof content.image_url === "string" ? content.image_url : content.image_url.url;
+  if (url.includes("data:image") && url.includes("base64")) {
+    return url;
+  }
+  const ext = url.match(/\.([^.?#]+)(?:[?#]|$)/)?.[1]?.toLowerCase();
+  const imageType = (0, import_string.getImageMimeType)(ext);
+  const buffer = await plugin.fetch(url).then((res) => res.arrayBuffer()).then(Buffer.from);
+  return `data:${imageType};base64,${buffer.toString("base64")}`;
+}
+__name(fetchImageUrl, "fetchImageUrl");
 function messageTypeToOpenAIRole(type) {
   switch (type) {
     case "system":
@@ -412,12 +444,13 @@ __name(convertDeltaToMessageChunk, "convertDeltaToMessageChunk");
 
 // src/requester.ts
 var import_messages2 = require("@langchain/core/messages");
-var import_string = require("koishi-plugin-chatluna/utils/string");
-function buildChatCompletionParams(params, enableGoogleSearch, supportImageInput2) {
+var import_string2 = require("koishi-plugin-chatluna/utils/string");
+async function buildChatCompletionParams(params, plugin, enableGoogleSearch, supportImageInput2) {
   const base = {
     model: params.model,
-    messages: langchainMessageToOpenAIMessage(
+    messages: await langchainMessageToOpenAIMessage(
       params.input,
+      plugin,
       params.model,
       supportImageInput2
     ),
@@ -553,7 +586,7 @@ async function processResponse(requestContext, response) {
     const messageChunk = convertMessageToMessageChunk(choice.message);
     return new import_outputs.ChatGenerationChunk({
       message: messageChunk,
-      text: (0, import_string.getMessageContent)(messageChunk.content),
+      text: (0, import_string2.getMessageContent)(messageChunk.content),
       generationInfo: {
         tokenUsage: data.usage
       }
@@ -577,8 +610,9 @@ async function* completionStream(requestContext, params, completionUrl = "chat/c
   try {
     const response = await modelRequester.post(
       completionUrl,
-      buildChatCompletionParams(
+      await buildChatCompletionParams(
         params,
+        requestContext.plugin,
         enableGoogleSearch ?? false,
         supportImageInput2 ?? true
       ),
@@ -599,8 +633,9 @@ async function* completionStream(requestContext, params, completionUrl = "chat/c
 __name(completionStream, "completionStream");
 async function completion(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput2) {
   const { modelRequester } = requestContext;
-  const chatCompletionParams = buildChatCompletionParams(
+  const chatCompletionParams = await buildChatCompletionParams(
     params,
+    requestContext.plugin,
     enableGoogleSearch ?? false,
     supportImageInput2 ?? true
   );
@@ -672,6 +707,7 @@ __name(createRequestContext, "createRequestContext");
   convertMessageToMessageChunk,
   createEmbeddings,
   createRequestContext,
+  fetchImageUrl,
   formatToolToOpenAITool,
   formatToolsToOpenAITools,
   getModelMaxContextSize,
@@ -683,5 +719,6 @@ __name(createRequestContext, "createRequestContext");
   processReasoningContent,
   processResponse,
   processStreamResponse,
+  removeAdditionalProperties,
   supportImageInput
 });

package/lib/index.mjs

@@ -104,12 +104,16 @@ import {
   ToolMessageChunk
 } from "@langchain/core/messages";
 import { zodToJsonSchema } from "zod-to-json-schema";
-function langchainMessageToOpenAIMessage(messages, model, supportImageInput2, removeSystemMessage) {
+import {
+  getImageMimeType,
+  isMessageContentImageUrl
+} from "koishi-plugin-chatluna/utils/string";
+async function langchainMessageToOpenAIMessage(messages, plugin, model, supportImageInput2, removeSystemMessage) {
   const result = [];
   for (const rawMessage of messages) {
     const role = messageTypeToOpenAIRole(rawMessage.getType());
     const msg = {
-      content: rawMessage.content || null,
+      content: rawMessage.content,
       name: role === "assistant" || role === "tool" ? rawMessage.name : void 0,
       role,
       //  function_call: rawMessage.additional_kwargs.function_call,
@@ -149,6 +153,24 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput2, re
           }
         });
       }
+    } else if (Array.isArray(msg.content) && msg.content.length > 0) {
+      msg.content = await Promise.all(
+        msg.content.map(async (content) => {
+          if (!isMessageContentImageUrl(content)) return content;
+          try {
+            const url = await fetchImageUrl(plugin, content);
+            return {
+              type: "image_url",
+              image_url: {
+                url,
+                detail: "low"
+              }
+            };
+          } catch {
+            return content;
+          }
+        })
+      );
     }
     result.push(msg);
   }
@@ -195,6 +217,17 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput2, re
   return result;
 }
 __name(langchainMessageToOpenAIMessage, "langchainMessageToOpenAIMessage");
+async function fetchImageUrl(plugin, content) {
+  const url = typeof content.image_url === "string" ? content.image_url : content.image_url.url;
+  if (url.includes("data:image") && url.includes("base64")) {
+    return url;
+  }
+  const ext = url.match(/\.([^.?#]+)(?:[?#]|$)/)?.[1]?.toLowerCase();
+  const imageType = getImageMimeType(ext);
+  const buffer = await plugin.fetch(url).then((res) => res.arrayBuffer()).then(Buffer.from);
+  return `data:${imageType};base64,${buffer.toString("base64")}`;
+}
+__name(fetchImageUrl, "fetchImageUrl");
 function messageTypeToOpenAIRole(type) {
   switch (type) {
     case "system":
@@ -382,11 +415,12 @@ __name(convertDeltaToMessageChunk, "convertDeltaToMessageChunk");
 // src/requester.ts
 import { AIMessageChunk as AIMessageChunk2 } from "@langchain/core/messages";
 import { getMessageContent } from "koishi-plugin-chatluna/utils/string";
-function buildChatCompletionParams(params, enableGoogleSearch, supportImageInput2) {
+async function buildChatCompletionParams(params, plugin, enableGoogleSearch, supportImageInput2) {
   const base = {
     model: params.model,
-    messages: langchainMessageToOpenAIMessage(
+    messages: await langchainMessageToOpenAIMessage(
       params.input,
+      plugin,
       params.model,
       supportImageInput2
     ),
@@ -546,8 +580,9 @@ async function* completionStream(requestContext, params, completionUrl = "chat/c
   try {
     const response = await modelRequester.post(
       completionUrl,
-      buildChatCompletionParams(
+      await buildChatCompletionParams(
         params,
+        requestContext.plugin,
         enableGoogleSearch ?? false,
         supportImageInput2 ?? true
       ),
@@ -568,8 +603,9 @@ async function* completionStream(requestContext, params, completionUrl = "chat/c
 __name(completionStream, "completionStream");
 async function completion(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput2) {
   const { modelRequester } = requestContext;
-  const chatCompletionParams = buildChatCompletionParams(
+  const chatCompletionParams = await buildChatCompletionParams(
     params,
+    requestContext.plugin,
     enableGoogleSearch ?? false,
     supportImageInput2 ?? true
   );
@@ -640,6 +676,7 @@ export {
   convertMessageToMessageChunk,
   createEmbeddings,
   createRequestContext,
+  fetchImageUrl,
   formatToolToOpenAITool,
   formatToolsToOpenAITools,
   getModelMaxContextSize,
@@ -651,5 +688,6 @@ export {
   processReasoningContent,
   processResponse,
   processStreamResponse,
+  removeAdditionalProperties,
   supportImageInput
 };

package/lib/requester.d.ts

@@ -12,7 +12,7 @@ interface RequestContext<T extends ClientConfig = ClientConfig, R extends ChatLu
     plugin: ChatLunaPlugin;
     modelRequester: ModelRequester<T, R>;
 }
-export declare function buildChatCompletionParams(params: ModelRequestParams, enableGoogleSearch: boolean, supportImageInput?: boolean): {
+export declare function buildChatCompletionParams(params: ModelRequestParams, plugin: ChatLunaPlugin, enableGoogleSearch: boolean, supportImageInput?: boolean): Promise<{
     model: string;
     messages: import("./types").ChatCompletionResponseMessage[];
     tools: import("./types").ChatCompletionTool[];
@@ -29,7 +29,7 @@ export declare function buildChatCompletionParams(params: ModelRequestParams, en
     stream_options: {
         include_usage: boolean;
     };
-};
+}>;
 export declare function processReasoningContent(delta: {
     reasoning_content?: string;
     content?: string;

package/lib/types.d.ts

@@ -20,18 +20,23 @@ export interface ChatCompletionResponse {
         total_tokens: number;
     };
 }
+export interface ChatCompletionTextPart {
+    type: 'text';
+    text: string;
+}
+export interface ChatCompletionImagePart {
+    type: 'image_url';
+    image_url: string | {
+        url: string;
+        detail?: 'low' | 'high';
+    };
+}
+export type ChatCompletionParts = ChatCompletionTextPart | ChatCompletionImagePart | (Record<string, unknown> & {
+    type: string;
+});
 export interface ChatCompletionResponseMessage {
     role: string;
-    content?: string | ({
-        type: 'text';
-        text: string;
-    } | {
-        type: 'image_url';
-        image_url: {
-            url: string;
-            detail?: 'low' | 'high';
-        };
-    })[];
+    content?: string | ChatCompletionParts[];
     reasoning_content?: string;
     name?: string;
     tool_calls?: ChatCompletionRequestMessageToolCall[];

package/lib/utils.d.ts

@@ -1,9 +1,13 @@
-import { AIMessageChunk, BaseMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, MessageType, SystemMessageChunk, ToolMessageChunk } from '@langchain/core/messages';
+import { AIMessageChunk, BaseMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, MessageContentImageUrl, MessageType, SystemMessageChunk, ToolMessageChunk } from '@langchain/core/messages';
 import { StructuredTool } from '@langchain/core/tools';
+import { JsonSchema7Type } from 'zod-to-json-schema';
 import { ChatCompletionResponseMessage, ChatCompletionResponseMessageRoleEnum, ChatCompletionTool } from './types';
-export declare function langchainMessageToOpenAIMessage(messages: BaseMessage[], model?: string, supportImageInput?: boolean, removeSystemMessage?: boolean): ChatCompletionResponseMessage[];
+import { ChatLunaPlugin } from 'koishi-plugin-chatluna/services/chat';
+export declare function langchainMessageToOpenAIMessage(messages: BaseMessage[], plugin: ChatLunaPlugin, model?: string, supportImageInput?: boolean, removeSystemMessage?: boolean): Promise<ChatCompletionResponseMessage[]>;
+export declare function fetchImageUrl(plugin: ChatLunaPlugin, content: MessageContentImageUrl): Promise<string>;
 export declare function messageTypeToOpenAIRole(type: MessageType): ChatCompletionResponseMessageRoleEnum;
 export declare function formatToolsToOpenAITools(tools: StructuredTool[], includeGoogleSearch: boolean): ChatCompletionTool[];
 export declare function formatToolToOpenAITool(tool: StructuredTool): ChatCompletionTool;
+export declare function removeAdditionalProperties(schema: JsonSchema7Type): JsonSchema7Type;
 export declare function convertMessageToMessageChunk(message: ChatCompletionResponseMessage): HumanMessageChunk | AIMessageChunk | SystemMessageChunk | FunctionMessageChunk | ToolMessageChunk | ChatMessageChunk;
 export declare function convertDeltaToMessageChunk(delta: Record<string, any>, defaultRole?: ChatCompletionResponseMessageRoleEnum): HumanMessageChunk | AIMessageChunk | SystemMessageChunk | FunctionMessageChunk | ToolMessageChunk | ChatMessageChunk;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@chatluna/v1-shared-adapter",
   "description": "chatluna shared adapter",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "main": "lib/index.cjs",
   "module": "lib/index.mjs",
   "typings": "lib/index.d.ts",
@@ -70,6 +70,6 @@
   },
   "peerDependencies": {
     "koishi": "^4.18.7",
-    "koishi-plugin-chatluna": "^1.3.0-alpha.16"
+    "koishi-plugin-chatluna": "^1.3.0-alpha.17"
   }
 }