@chatluna/v1-shared-adapter 1.0.3 → 1.0.5

package/lib/client.d.ts

@@ -2,3 +2,4 @@ import { ModelInfo } from 'koishi-plugin-chatluna/llm-core/platform/types';
 export declare function isEmbeddingModel(modelName: string): boolean;
 export declare function isNonLLMModel(modelName: string): boolean;
 export declare function getModelMaxContextSize(info: ModelInfo): number;
+export declare function supportImageInput(modelName: string): boolean;

package/lib/index.cjs

@@ -27,6 +27,7 @@ __export(index_exports, {
   convertMessageToMessageChunk: () => convertMessageToMessageChunk,
   createEmbeddings: () => createEmbeddings,
   createRequestContext: () => createRequestContext,
+  fetchImageUrl: () => fetchImageUrl,
   formatToolToOpenAITool: () => formatToolToOpenAITool,
   formatToolsToOpenAITools: () => formatToolsToOpenAITools,
   getModelMaxContextSize: () => getModelMaxContextSize,
@@ -37,7 +38,9 @@ __export(index_exports, {
   messageTypeToOpenAIRole: () => messageTypeToOpenAIRole,
   processReasoningContent: () => processReasoningContent,
   processResponse: () => processResponse,
-  processStreamResponse: () => processStreamResponse
+  processStreamResponse: () => processStreamResponse,
+  removeAdditionalProperties: () => removeAdditionalProperties,
+  supportImageInput: () => supportImageInput
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -92,6 +95,39 @@ function getModelMaxContextSize(info) {
   return (0, import_count_tokens.getModelContextSize)("o1-mini");
 }
 __name(getModelMaxContextSize, "getModelMaxContextSize");
+function createGlobMatcher(pattern) {
+  if (!pattern.includes("*")) {
+    return (text) => text.includes(pattern);
+  }
+  const regex = new RegExp("^" + pattern.replace(/\*/g, ".*") + "$");
+  return (text) => regex.test(text);
+}
+__name(createGlobMatcher, "createGlobMatcher");
+var imageModelMatchers = [
+  "vision",
+  "vl",
+  "gpt-4o",
+  "claude",
+  "gemini",
+  "qwen-vl",
+  "omni",
+  "qwen2.5-omni",
+  "qwen-omni",
+  "qvq",
+  "o1",
+  "o3",
+  "o4",
+  "gpt-4.1",
+  "gpt-5",
+  "glm-*v",
+  "step3",
+  "grok-4"
+].map((pattern) => createGlobMatcher(pattern));
+function supportImageInput(modelName) {
+  const lowerModel = modelName.toLowerCase();
+  return imageModelMatchers.some((matcher) => matcher(lowerModel));
+}
+__name(supportImageInput, "supportImageInput");
 
 // src/requester.ts
 var import_outputs = require("@langchain/core/outputs");
@@ -101,12 +137,13 @@ var import_sse = require("koishi-plugin-chatluna/utils/sse");
 // src/utils.ts
 var import_messages = require("@langchain/core/messages");
 var import_zod_to_json_schema = require("zod-to-json-schema");
-function langchainMessageToOpenAIMessage(messages, model, supportImageInput, removeSystemMessage) {
+var import_string = require("koishi-plugin-chatluna/utils/string");
+async function langchainMessageToOpenAIMessage(messages, plugin, model, supportImageInput2, removeSystemMessage) {
   const result = [];
   for (const rawMessage of messages) {
     const role = messageTypeToOpenAIRole(rawMessage.getType());
     const msg = {
-      content: rawMessage.content || null,
+      content: rawMessage.content,
       name: role === "assistant" || role === "tool" ? rawMessage.name : void 0,
       role,
       //  function_call: rawMessage.additional_kwargs.function_call,
@@ -130,7 +167,7 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput, rem
     }
     const images = rawMessage.additional_kwargs.images;
     const lowerModel = model?.toLowerCase() ?? "";
-    if ((lowerModel?.includes("vision") || lowerModel?.includes("gpt-4o") || lowerModel?.includes("claude") || lowerModel?.includes("gemini") || lowerModel?.includes("qwen-vl") || lowerModel?.includes("omni") || lowerModel?.includes("qwen2.5-vl") || lowerModel?.includes("qwen2.5-omni") || lowerModel?.includes("qwen-omni") || lowerModel?.includes("qwen2-vl") || lowerModel?.includes("qvq") || model?.includes("o1") || model?.includes("o4") || model?.includes("o3") || model?.includes("gpt-4.1") || model?.includes("gpt-5") || supportImageInput) && images != null) {
+    if ((lowerModel?.includes("vision") || lowerModel?.includes("gpt-4o") || lowerModel?.includes("claude") || lowerModel?.includes("gemini") || lowerModel?.includes("qwen-vl") || lowerModel?.includes("omni") || lowerModel?.includes("qwen2.5-vl") || lowerModel?.includes("qwen2.5-omni") || lowerModel?.includes("qwen-omni") || lowerModel?.includes("qwen2-vl") || lowerModel?.includes("qvq") || model?.includes("o1") || model?.includes("o4") || model?.includes("o3") || model?.includes("gpt-4.1") || model?.includes("gpt-5") || supportImageInput2) && images != null) {
       msg.content = [
         {
           type: "text",
@@ -146,6 +183,24 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput, rem
           }
         });
       }
+    } else if (Array.isArray(msg.content) && msg.content.length > 0) {
+      msg.content = await Promise.all(
+        msg.content.map(async (content) => {
+          if (!(0, import_string.isMessageContentImageUrl)(content)) return content;
+          try {
+            const url = await fetchImageUrl(plugin, content);
+            return {
+              type: "image_url",
+              image_url: {
+                url,
+                detail: "low"
+              }
+            };
+          } catch {
+            return content;
+          }
+        })
+      );
     }
     result.push(msg);
   }
@@ -192,6 +247,17 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput, rem
   return result;
 }
 __name(langchainMessageToOpenAIMessage, "langchainMessageToOpenAIMessage");
+async function fetchImageUrl(plugin, content) {
+  const url = typeof content.image_url === "string" ? content.image_url : content.image_url.url;
+  if (url.includes("data:image") && url.includes("base64")) {
+    return url;
+  }
+  const ext = url.match(/\.([^.?#]+)(?:[?#]|$)/)?.[1]?.toLowerCase();
+  const imageType = (0, import_string.getImageMimeType)(ext);
+  const buffer = await plugin.fetch(url).then((res) => res.arrayBuffer()).then(Buffer.from);
+  return `data:${imageType};base64,${buffer.toString("base64")}`;
+}
+__name(fetchImageUrl, "fetchImageUrl");
 function messageTypeToOpenAIRole(type) {
   switch (type) {
     case "system":
@@ -378,14 +444,15 @@ __name(convertDeltaToMessageChunk, "convertDeltaToMessageChunk");
 
 // src/requester.ts
 var import_messages2 = require("@langchain/core/messages");
-var import_string = require("koishi-plugin-chatluna/utils/string");
-function buildChatCompletionParams(params, enableGoogleSearch, supportImageInput) {
+var import_string2 = require("koishi-plugin-chatluna/utils/string");
+async function buildChatCompletionParams(params, plugin, enableGoogleSearch, supportImageInput2) {
   const base = {
     model: params.model,
-    messages: langchainMessageToOpenAIMessage(
+    messages: await langchainMessageToOpenAIMessage(
       params.input,
+      plugin,
       params.model,
-      supportImageInput
+      supportImageInput2
     ),
     tools: enableGoogleSearch || params.tools != null ? formatToolsToOpenAITools(
       params.tools ?? [],
@@ -519,7 +586,7 @@ async function processResponse(requestContext, response) {
     const messageChunk = convertMessageToMessageChunk(choice.message);
     return new import_outputs.ChatGenerationChunk({
       message: messageChunk,
-      text: (0, import_string.getMessageContent)(messageChunk.content),
+      text: (0, import_string2.getMessageContent)(messageChunk.content),
       generationInfo: {
         tokenUsage: data.usage
       }
@@ -538,15 +605,16 @@ async function processResponse(requestContext, response) {
   }
 }
 __name(processResponse, "processResponse");
-async function* completionStream(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput) {
+async function* completionStream(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput2) {
   const { modelRequester } = requestContext;
   try {
     const response = await modelRequester.post(
       completionUrl,
-      buildChatCompletionParams(
+      await buildChatCompletionParams(
         params,
+        requestContext.plugin,
         enableGoogleSearch ?? false,
-        supportImageInput ?? true
+        supportImageInput2 ?? true
       ),
       {
         signal: params.signal
@@ -563,12 +631,13 @@ async function* completionStream(requestContext, params, completionUrl = "chat/c
   }
 }
 __name(completionStream, "completionStream");
-async function completion(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput) {
+async function completion(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput2) {
   const { modelRequester } = requestContext;
-  const chatCompletionParams = buildChatCompletionParams(
+  const chatCompletionParams = await buildChatCompletionParams(
     params,
+    requestContext.plugin,
     enableGoogleSearch ?? false,
-    supportImageInput ?? true
+    supportImageInput2 ?? true
   );
   delete chatCompletionParams.stream;
   try {
@@ -638,6 +707,7 @@ __name(createRequestContext, "createRequestContext");
   convertMessageToMessageChunk,
   createEmbeddings,
   createRequestContext,
+  fetchImageUrl,
   formatToolToOpenAITool,
   formatToolsToOpenAITools,
   getModelMaxContextSize,
@@ -648,5 +718,7 @@ __name(createRequestContext, "createRequestContext");
   messageTypeToOpenAIRole,
   processReasoningContent,
   processResponse,
-  processStreamResponse
+  processStreamResponse,
+  removeAdditionalProperties,
+  supportImageInput
 });

package/lib/index.mjs

@@ -52,6 +52,39 @@ function getModelMaxContextSize(info) {
   return getModelContextSize("o1-mini");
 }
 __name(getModelMaxContextSize, "getModelMaxContextSize");
+function createGlobMatcher(pattern) {
+  if (!pattern.includes("*")) {
+    return (text) => text.includes(pattern);
+  }
+  const regex = new RegExp("^" + pattern.replace(/\*/g, ".*") + "$");
+  return (text) => regex.test(text);
+}
+__name(createGlobMatcher, "createGlobMatcher");
+var imageModelMatchers = [
+  "vision",
+  "vl",
+  "gpt-4o",
+  "claude",
+  "gemini",
+  "qwen-vl",
+  "omni",
+  "qwen2.5-omni",
+  "qwen-omni",
+  "qvq",
+  "o1",
+  "o3",
+  "o4",
+  "gpt-4.1",
+  "gpt-5",
+  "glm-*v",
+  "step3",
+  "grok-4"
+].map((pattern) => createGlobMatcher(pattern));
+function supportImageInput(modelName) {
+  const lowerModel = modelName.toLowerCase();
+  return imageModelMatchers.some((matcher) => matcher(lowerModel));
+}
+__name(supportImageInput, "supportImageInput");
 
 // src/requester.ts
 import { ChatGenerationChunk } from "@langchain/core/outputs";
@@ -71,12 +104,16 @@ import {
   ToolMessageChunk
 } from "@langchain/core/messages";
 import { zodToJsonSchema } from "zod-to-json-schema";
-function langchainMessageToOpenAIMessage(messages, model, supportImageInput, removeSystemMessage) {
+import {
+  getImageMimeType,
+  isMessageContentImageUrl
+} from "koishi-plugin-chatluna/utils/string";
+async function langchainMessageToOpenAIMessage(messages, plugin, model, supportImageInput2, removeSystemMessage) {
   const result = [];
   for (const rawMessage of messages) {
     const role = messageTypeToOpenAIRole(rawMessage.getType());
     const msg = {
-      content: rawMessage.content || null,
+      content: rawMessage.content,
       name: role === "assistant" || role === "tool" ? rawMessage.name : void 0,
       role,
       //  function_call: rawMessage.additional_kwargs.function_call,
@@ -100,7 +137,7 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput, rem
     }
     const images = rawMessage.additional_kwargs.images;
     const lowerModel = model?.toLowerCase() ?? "";
-    if ((lowerModel?.includes("vision") || lowerModel?.includes("gpt-4o") || lowerModel?.includes("claude") || lowerModel?.includes("gemini") || lowerModel?.includes("qwen-vl") || lowerModel?.includes("omni") || lowerModel?.includes("qwen2.5-vl") || lowerModel?.includes("qwen2.5-omni") || lowerModel?.includes("qwen-omni") || lowerModel?.includes("qwen2-vl") || lowerModel?.includes("qvq") || model?.includes("o1") || model?.includes("o4") || model?.includes("o3") || model?.includes("gpt-4.1") || model?.includes("gpt-5") || supportImageInput) && images != null) {
+    if ((lowerModel?.includes("vision") || lowerModel?.includes("gpt-4o") || lowerModel?.includes("claude") || lowerModel?.includes("gemini") || lowerModel?.includes("qwen-vl") || lowerModel?.includes("omni") || lowerModel?.includes("qwen2.5-vl") || lowerModel?.includes("qwen2.5-omni") || lowerModel?.includes("qwen-omni") || lowerModel?.includes("qwen2-vl") || lowerModel?.includes("qvq") || model?.includes("o1") || model?.includes("o4") || model?.includes("o3") || model?.includes("gpt-4.1") || model?.includes("gpt-5") || supportImageInput2) && images != null) {
       msg.content = [
         {
           type: "text",
@@ -116,6 +153,24 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput, rem
           }
         });
       }
+    } else if (Array.isArray(msg.content) && msg.content.length > 0) {
+      msg.content = await Promise.all(
+        msg.content.map(async (content) => {
+          if (!isMessageContentImageUrl(content)) return content;
+          try {
+            const url = await fetchImageUrl(plugin, content);
+            return {
+              type: "image_url",
+              image_url: {
+                url,
+                detail: "low"
+              }
+            };
+          } catch {
+            return content;
+          }
+        })
+      );
     }
     result.push(msg);
   }
@@ -162,6 +217,17 @@ function langchainMessageToOpenAIMessage(messages, model, supportImageInput, rem
   return result;
 }
 __name(langchainMessageToOpenAIMessage, "langchainMessageToOpenAIMessage");
+async function fetchImageUrl(plugin, content) {
+  const url = typeof content.image_url === "string" ? content.image_url : content.image_url.url;
+  if (url.includes("data:image") && url.includes("base64")) {
+    return url;
+  }
+  const ext = url.match(/\.([^.?#]+)(?:[?#]|$)/)?.[1]?.toLowerCase();
+  const imageType = getImageMimeType(ext);
+  const buffer = await plugin.fetch(url).then((res) => res.arrayBuffer()).then(Buffer.from);
+  return `data:${imageType};base64,${buffer.toString("base64")}`;
+}
+__name(fetchImageUrl, "fetchImageUrl");
 function messageTypeToOpenAIRole(type) {
   switch (type) {
     case "system":
@@ -349,13 +415,14 @@ __name(convertDeltaToMessageChunk, "convertDeltaToMessageChunk");
 // src/requester.ts
 import { AIMessageChunk as AIMessageChunk2 } from "@langchain/core/messages";
 import { getMessageContent } from "koishi-plugin-chatluna/utils/string";
-function buildChatCompletionParams(params, enableGoogleSearch, supportImageInput) {
+async function buildChatCompletionParams(params, plugin, enableGoogleSearch, supportImageInput2) {
   const base = {
     model: params.model,
-    messages: langchainMessageToOpenAIMessage(
+    messages: await langchainMessageToOpenAIMessage(
       params.input,
+      plugin,
       params.model,
-      supportImageInput
+      supportImageInput2
     ),
     tools: enableGoogleSearch || params.tools != null ? formatToolsToOpenAITools(
       params.tools ?? [],
@@ -508,15 +575,16 @@ async function processResponse(requestContext, response) {
   }
 }
 __name(processResponse, "processResponse");
-async function* completionStream(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput) {
+async function* completionStream(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput2) {
   const { modelRequester } = requestContext;
   try {
     const response = await modelRequester.post(
       completionUrl,
-      buildChatCompletionParams(
+      await buildChatCompletionParams(
         params,
+        requestContext.plugin,
         enableGoogleSearch ?? false,
-        supportImageInput ?? true
+        supportImageInput2 ?? true
       ),
       {
         signal: params.signal
@@ -533,12 +601,13 @@ async function* completionStream(requestContext, params, completionUrl = "chat/c
   }
 }
 __name(completionStream, "completionStream");
-async function completion(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput) {
+async function completion(requestContext, params, completionUrl = "chat/completions", enableGoogleSearch, supportImageInput2) {
   const { modelRequester } = requestContext;
-  const chatCompletionParams = buildChatCompletionParams(
+  const chatCompletionParams = await buildChatCompletionParams(
     params,
+    requestContext.plugin,
     enableGoogleSearch ?? false,
-    supportImageInput ?? true
+    supportImageInput2 ?? true
   );
   delete chatCompletionParams.stream;
   try {
@@ -607,6 +676,7 @@ export {
   convertMessageToMessageChunk,
   createEmbeddings,
   createRequestContext,
+  fetchImageUrl,
   formatToolToOpenAITool,
   formatToolsToOpenAITools,
   getModelMaxContextSize,
@@ -617,5 +687,7 @@ export {
   messageTypeToOpenAIRole,
   processReasoningContent,
   processResponse,
-  processStreamResponse
+  processStreamResponse,
+  removeAdditionalProperties,
+  supportImageInput
 };

package/lib/requester.d.ts

@@ -12,7 +12,7 @@ interface RequestContext<T extends ClientConfig = ClientConfig, R extends ChatLu
     plugin: ChatLunaPlugin;
     modelRequester: ModelRequester<T, R>;
 }
-export declare function buildChatCompletionParams(params: ModelRequestParams, enableGoogleSearch: boolean, supportImageInput?: boolean): {
+export declare function buildChatCompletionParams(params: ModelRequestParams, plugin: ChatLunaPlugin, enableGoogleSearch: boolean, supportImageInput?: boolean): Promise<{
     model: string;
     messages: import("./types").ChatCompletionResponseMessage[];
     tools: import("./types").ChatCompletionTool[];
@@ -29,7 +29,7 @@ export declare function buildChatCompletionParams(params: ModelRequestParams, en
     stream_options: {
         include_usage: boolean;
     };
-};
+}>;
 export declare function processReasoningContent(delta: {
     reasoning_content?: string;
     content?: string;

package/lib/types.d.ts

@@ -20,18 +20,23 @@ export interface ChatCompletionResponse {
         total_tokens: number;
     };
 }
+export interface ChatCompletionTextPart {
+    type: 'text';
+    text: string;
+}
+export interface ChatCompletionImagePart {
+    type: 'image_url';
+    image_url: string | {
+        url: string;
+        detail?: 'low' | 'high';
+    };
+}
+export type ChatCompletionParts = ChatCompletionTextPart | ChatCompletionImagePart | (Record<string, unknown> & {
+    type: string;
+});
 export interface ChatCompletionResponseMessage {
     role: string;
-    content?: string | ({
-        type: 'text';
-        text: string;
-    } | {
-        type: 'image_url';
-        image_url: {
-            url: string;
-            detail?: 'low' | 'high';
-        };
-    })[];
+    content?: string | ChatCompletionParts[];
     reasoning_content?: string;
     name?: string;
     tool_calls?: ChatCompletionRequestMessageToolCall[];

package/lib/utils.d.ts

@@ -1,9 +1,13 @@
-import { AIMessageChunk, BaseMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, MessageType, SystemMessageChunk, ToolMessageChunk } from '@langchain/core/messages';
+import { AIMessageChunk, BaseMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, MessageContentImageUrl, MessageType, SystemMessageChunk, ToolMessageChunk } from '@langchain/core/messages';
 import { StructuredTool } from '@langchain/core/tools';
+import { JsonSchema7Type } from 'zod-to-json-schema';
 import { ChatCompletionResponseMessage, ChatCompletionResponseMessageRoleEnum, ChatCompletionTool } from './types';
-export declare function langchainMessageToOpenAIMessage(messages: BaseMessage[], model?: string, supportImageInput?: boolean, removeSystemMessage?: boolean): ChatCompletionResponseMessage[];
+import { ChatLunaPlugin } from 'koishi-plugin-chatluna/services/chat';
+export declare function langchainMessageToOpenAIMessage(messages: BaseMessage[], plugin: ChatLunaPlugin, model?: string, supportImageInput?: boolean, removeSystemMessage?: boolean): Promise<ChatCompletionResponseMessage[]>;
+export declare function fetchImageUrl(plugin: ChatLunaPlugin, content: MessageContentImageUrl): Promise<string>;
 export declare function messageTypeToOpenAIRole(type: MessageType): ChatCompletionResponseMessageRoleEnum;
 export declare function formatToolsToOpenAITools(tools: StructuredTool[], includeGoogleSearch: boolean): ChatCompletionTool[];
 export declare function formatToolToOpenAITool(tool: StructuredTool): ChatCompletionTool;
-export declare function convertMessageToMessageChunk(message: ChatCompletionResponseMessage): AIMessageChunk | HumanMessageChunk | SystemMessageChunk | FunctionMessageChunk | ToolMessageChunk | ChatMessageChunk;
-export declare function convertDeltaToMessageChunk(delta: Record<string, any>, defaultRole?: ChatCompletionResponseMessageRoleEnum): AIMessageChunk | HumanMessageChunk | SystemMessageChunk | FunctionMessageChunk | ToolMessageChunk | ChatMessageChunk;
+export declare function removeAdditionalProperties(schema: JsonSchema7Type): JsonSchema7Type;
+export declare function convertMessageToMessageChunk(message: ChatCompletionResponseMessage): HumanMessageChunk | AIMessageChunk | SystemMessageChunk | FunctionMessageChunk | ToolMessageChunk | ChatMessageChunk;
+export declare function convertDeltaToMessageChunk(delta: Record<string, any>, defaultRole?: ChatCompletionResponseMessageRoleEnum): HumanMessageChunk | AIMessageChunk | SystemMessageChunk | FunctionMessageChunk | ToolMessageChunk | ChatMessageChunk;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@chatluna/v1-shared-adapter",
   "description": "chatluna shared adapter",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "main": "lib/index.cjs",
   "module": "lib/index.mjs",
   "typings": "lib/index.d.ts",
@@ -70,6 +70,6 @@
   },
   "peerDependencies": {
     "koishi": "^4.18.7",
-    "koishi-plugin-chatluna": "^1.3.0-alpha.15"
+    "koishi-plugin-chatluna": "^1.3.0-alpha.17"
   }
 }