koishi-plugin-chatluna-google-gemini-adapter 1.1.0 → 1.1.2

package/lib/index.cjs

@@ -23,14 +23,14 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/locales/zh-CN.schema.yml
 var require_zh_CN_schema = __commonJS({
   "src/locales/zh-CN.schema.yml"(exports2, module2) {
-    module2.exports = { $inner: [{}, { $desc: "请求选项", apiKeys: { $inner: ["Gemini 的 API Key", "Gemini API 的请求地址"], $desc: "Gemini 的 API Key 和请求地址列表。" } }, { $desc: "模型配置", maxTokens: "输入的最大上下文 Token（16~2097000，必须是 16 的倍数）。注意：仅当您使用的模型最大 Token 为 8000 及以上时，才建议设置超过 2000 token。", temperature: "回复的随机性程度，数值越高，回复越随机（范围：0~2）。", googleSearch: "为模型启用谷歌搜索。", groundingContentDisplay: "是否显示谷歌搜索结果。", searchThreshold: "搜索的置信度阈值，范围：0~1，设置的数值越低，则越倾向于使用谷歌搜索。" }] };
+    module2.exports = { $inner: [{}, { $desc: "请求选项", apiKeys: { $inner: ["Gemini 的 API Key", "Gemini API 的请求地址"], $desc: "Gemini 的 API Key 和请求地址列表。" } }, { $desc: "模型配置", maxTokens: "输入的最大上下文 Token（16~2097000，必须是 16 的倍数）。注意：仅当您使用的模型最大 Token 为 8000 及以上时，才建议设置超过 2000 token。", temperature: "回复的随机性程度，数值越高，回复越随机（范围：0~2）。", googleSearch: "为模型启用谷歌搜索。", groundingContentDisplay: "是否显示谷歌搜索结果。", imageGeneration: "为模型启用图像生成。目前仅支持 `gemini-2.0-flash-exp` 模型。", searchThreshold: "搜索的置信度阈值，范围：0~1，设置的数值越低，则越倾向于使用谷歌搜索。" }] };
   }
 });
 
 // src/locales/en-US.schema.yml
 var require_en_US_schema = __commonJS({
   "src/locales/en-US.schema.yml"(exports2, module2) {
-    module2.exports = { $inner: [{}, { $desc: "API Configuration", apiKeys: { $inner: ["Gemini API Key", "Gemini API Endpoint (optional)"], $desc: "Gemini API access credentials" } }, { $desc: "Model Parameters", maxTokens: "Max output tokens (16-2097000, multiple of 16). >2000 for 8k+ models", temperature: "Sampling temperature (0-2). Higher: more random, Lower: more deterministic", googleSearch: "Enable Google search", groundingContentDisplay: "Enable display of search results", searchThreshold: "Search confidence [threshold](https://ai.google.dev/gemini-api/docs/grounding?lang=rest#dynamic-retrieval) (0-1). Lower: more likely to use Google search" }] };
+    module2.exports = { $inner: [{}, { $desc: "API Configuration", apiKeys: { $inner: ["Gemini API Key", "Gemini API Endpoint (optional)"], $desc: "Gemini API access credentials" } }, { $desc: "Model Parameters", maxTokens: "Max output tokens (16-2097000, multiple of 16). >2000 for 8k+ models", temperature: "Sampling temperature (0-2). Higher: more random, Lower: more deterministic", googleSearch: "Enable Google search", groundingContentDisplay: "Enable display of search results", imageGeneration: "Enable image generation (only for `gemini-2.0-flash-exp` model)", searchThreshold: "Search confidence [threshold](https://ai.google.dev/gemini-api/docs/grounding?lang=rest#dynamic-retrieval) (0-1). Lower: more likely to use Google search" }] };
   }
 });
 
@@ -187,48 +187,6 @@ async function langchainMessageToGeminiMessage(messages, model) {
       ]
     });
   }
-  if (model.includes("vision")) {
-    const textBuffer = [];
-    const last = result.pop();
-    for (let i = 0; i < result.length; i++) {
-      const message = result[i];
-      const text = message.parts[0].text;
-      textBuffer.push(`${message.role}: ${text}`);
-    }
-    const lastParts = last.parts;
-    let lastImagesParts = lastParts.filter(
-      (part) => part.inline_data?.mime_type === "image/jpeg"
-    );
-    if (lastImagesParts.length < 1) {
-      for (let i = result.length - 1; i >= 0; i--) {
-        const message = result[i];
-        const images = message.parts.filter(
-          (part) => part.inline_data?.mime_type === "image/jpeg"
-        );
-        if (images.length > 0) {
-          lastImagesParts = images;
-          break;
-        }
-      }
-    }
-    ;
-    lastParts.filter(
-      (part) => part.text !== void 0 && part.text !== null
-    ).forEach((part) => {
-      textBuffer.push(`${last.role}: ${part.text}`);
-    });
-    return [
-      {
-        role: "user",
-        parts: [
-          {
-            text: textBuffer.join("\n")
-          },
-          ...lastImagesParts
-        ]
-      }
-    ];
-  }
   return result;
 }
 __name(langchainMessageToGeminiMessage, "langchainMessageToGeminiMessage");
@@ -236,6 +194,10 @@ function partAsType(part) {
   return part;
 }
 __name(partAsType, "partAsType");
+function partAsTypeCheck(part, check) {
+  return check(part) ? part : void 0;
+}
+__name(partAsTypeCheck, "partAsTypeCheck");
 function formatToolsToGeminiAITools(tools, config, model) {
   if (tools.length < 1 && !config.googleSearch) {
     return void 0;
@@ -380,7 +342,11 @@ var GeminiRequester = class extends import_api.ModelRequester {
             stopSequences: params.stop,
             temperature: params.temperature,
             maxOutputTokens: params.model.includes("vision") ? void 0 : params.maxTokens,
-            topP: params.topP
+            topP: params.topP,
+            responseModalities: params.model.includes(
+              // TODO: Wait for google release to all models
+              "gemini-2.0-flash-exp"
+            ) && this._pluginConfig.imageGeneration ? ["TEXT", "IMAGE"] : void 0
             // thinkingConfig: { includeThoughts: true }
           },
           tools: params.tools != null || this._pluginConfig.googleSearch ? formatToolsToGeminiAITools(
@@ -414,8 +380,10 @@ var GeminiRequester = class extends import_api.ModelRequester {
           }
           for (const candidate of transformValue.candidates) {
             const parts = candidate.content?.parts;
-            if (parts == null || parts.length < 1) {
+            if ((parts == null || parts.length < 1) && candidate.finishReason !== "STOP") {
               throw new Error(chunk);
+            } else if (candidate.finishReason === "STOP" && parts == null) {
+              continue;
             }
             for (const part of parts) {
               controller.enqueue(part);
@@ -440,14 +408,21 @@ var GeminiRequester = class extends import_api.ModelRequester {
       for await (const chunk of iterable) {
         const messagePart = partAsType(chunk);
         const chatFunctionCallingPart = partAsType(chunk);
+        const imagePart = partAsTypeCheck(
+          chunk,
+          (part) => part["inlineData"] != null
+        );
         if (messagePart.text) {
           if (messagePart.thought) {
             reasoningContent += messagePart.text;
             continue;
           }
           content = messagePart.text;
+        } else if (imagePart) {
+          messagePart.text = `![image](data:${imagePart.inlineData.mime_type};base64,${imagePart.inlineData.data})`;
+          content = messagePart.text;
         }
-        const deltaFunctionCall = chatFunctionCallingPart.functionCall;
+        const deltaFunctionCall = chatFunctionCallingPart?.functionCall;
         if (deltaFunctionCall) {
           let args = deltaFunctionCall.args?.input ?? deltaFunctionCall.args;
           try {
@@ -472,7 +447,10 @@ var GeminiRequester = class extends import_api.ModelRequester {
               name: functionCall.name,
               arguments: functionCall.args,
               args: functionCall.arguments
-            } : void 0
+            } : void 0,
+            images: imagePart ? [
+              `data:${imagePart.inlineData.mime_type};base64,${imagePart.inlineData.data})`
+            ] : void 0
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
           };
           messageChunk.content = content;
@@ -766,6 +744,7 @@ var Config3 = import_koishi.Schema.intersect([
     maxTokens: import_koishi.Schema.number().min(16).max(2097e3).step(16).default(8064),
     temperature: import_koishi.Schema.percent().min(0).max(2).step(0.1).default(0.8),
     googleSearch: import_koishi.Schema.boolean().default(false),
+    imageGeneration: import_koishi.Schema.boolean().default(false),
     groundingContentDisplay: import_koishi.Schema.boolean().default(false),
     searchThreshold: import_koishi.Schema.number().min(0).max(1).step(0.1).default(0.5)
   })

package/lib/index.d.ts

@@ -9,6 +9,7 @@ export interface Config extends ChatLunaPlugin.Config {
     googleSearch: boolean;
     searchThreshold: number;
     groundingContentDisplay: boolean;
+    imageGeneration: boolean;
 }
 export declare const Config: Schema<Config>;
 export declare const inject: string[];

package/lib/index.mjs

@@ -8,14 +8,14 @@ var __commonJS = (cb, mod) => function __require() {
 // src/locales/zh-CN.schema.yml
 var require_zh_CN_schema = __commonJS({
   "src/locales/zh-CN.schema.yml"(exports, module) {
-    module.exports = { $inner: [{}, { $desc: "请求选项", apiKeys: { $inner: ["Gemini 的 API Key", "Gemini API 的请求地址"], $desc: "Gemini 的 API Key 和请求地址列表。" } }, { $desc: "模型配置", maxTokens: "输入的最大上下文 Token（16~2097000，必须是 16 的倍数）。注意：仅当您使用的模型最大 Token 为 8000 及以上时，才建议设置超过 2000 token。", temperature: "回复的随机性程度，数值越高，回复越随机（范围：0~2）。", googleSearch: "为模型启用谷歌搜索。", groundingContentDisplay: "是否显示谷歌搜索结果。", searchThreshold: "搜索的置信度阈值，范围：0~1，设置的数值越低，则越倾向于使用谷歌搜索。" }] };
+    module.exports = { $inner: [{}, { $desc: "请求选项", apiKeys: { $inner: ["Gemini 的 API Key", "Gemini API 的请求地址"], $desc: "Gemini 的 API Key 和请求地址列表。" } }, { $desc: "模型配置", maxTokens: "输入的最大上下文 Token（16~2097000，必须是 16 的倍数）。注意：仅当您使用的模型最大 Token 为 8000 及以上时，才建议设置超过 2000 token。", temperature: "回复的随机性程度，数值越高，回复越随机（范围：0~2）。", googleSearch: "为模型启用谷歌搜索。", groundingContentDisplay: "是否显示谷歌搜索结果。", imageGeneration: "为模型启用图像生成。目前仅支持 `gemini-2.0-flash-exp` 模型。", searchThreshold: "搜索的置信度阈值，范围：0~1，设置的数值越低，则越倾向于使用谷歌搜索。" }] };
   }
 });
 
 // src/locales/en-US.schema.yml
 var require_en_US_schema = __commonJS({
   "src/locales/en-US.schema.yml"(exports, module) {
-    module.exports = { $inner: [{}, { $desc: "API Configuration", apiKeys: { $inner: ["Gemini API Key", "Gemini API Endpoint (optional)"], $desc: "Gemini API access credentials" } }, { $desc: "Model Parameters", maxTokens: "Max output tokens (16-2097000, multiple of 16). >2000 for 8k+ models", temperature: "Sampling temperature (0-2). Higher: more random, Lower: more deterministic", googleSearch: "Enable Google search", groundingContentDisplay: "Enable display of search results", searchThreshold: "Search confidence [threshold](https://ai.google.dev/gemini-api/docs/grounding?lang=rest#dynamic-retrieval) (0-1). Lower: more likely to use Google search" }] };
+    module.exports = { $inner: [{}, { $desc: "API Configuration", apiKeys: { $inner: ["Gemini API Key", "Gemini API Endpoint (optional)"], $desc: "Gemini API access credentials" } }, { $desc: "Model Parameters", maxTokens: "Max output tokens (16-2097000, multiple of 16). >2000 for 8k+ models", temperature: "Sampling temperature (0-2). Higher: more random, Lower: more deterministic", googleSearch: "Enable Google search", groundingContentDisplay: "Enable display of search results", imageGeneration: "Enable image generation (only for `gemini-2.0-flash-exp` model)", searchThreshold: "Search confidence [threshold](https://ai.google.dev/gemini-api/docs/grounding?lang=rest#dynamic-retrieval) (0-1). Lower: more likely to use Google search" }] };
   }
 });
 
@@ -181,48 +181,6 @@ async function langchainMessageToGeminiMessage(messages, model) {
       ]
     });
   }
-  if (model.includes("vision")) {
-    const textBuffer = [];
-    const last = result.pop();
-    for (let i = 0; i < result.length; i++) {
-      const message = result[i];
-      const text = message.parts[0].text;
-      textBuffer.push(`${message.role}: ${text}`);
-    }
-    const lastParts = last.parts;
-    let lastImagesParts = lastParts.filter(
-      (part) => part.inline_data?.mime_type === "image/jpeg"
-    );
-    if (lastImagesParts.length < 1) {
-      for (let i = result.length - 1; i >= 0; i--) {
-        const message = result[i];
-        const images = message.parts.filter(
-          (part) => part.inline_data?.mime_type === "image/jpeg"
-        );
-        if (images.length > 0) {
-          lastImagesParts = images;
-          break;
-        }
-      }
-    }
-    ;
-    lastParts.filter(
-      (part) => part.text !== void 0 && part.text !== null
-    ).forEach((part) => {
-      textBuffer.push(`${last.role}: ${part.text}`);
-    });
-    return [
-      {
-        role: "user",
-        parts: [
-          {
-            text: textBuffer.join("\n")
-          },
-          ...lastImagesParts
-        ]
-      }
-    ];
-  }
   return result;
 }
 __name(langchainMessageToGeminiMessage, "langchainMessageToGeminiMessage");
@@ -230,6 +188,10 @@ function partAsType(part) {
   return part;
 }
 __name(partAsType, "partAsType");
+function partAsTypeCheck(part, check) {
+  return check(part) ? part : void 0;
+}
+__name(partAsTypeCheck, "partAsTypeCheck");
 function formatToolsToGeminiAITools(tools, config, model) {
   if (tools.length < 1 && !config.googleSearch) {
     return void 0;
@@ -374,7 +336,11 @@ var GeminiRequester = class extends ModelRequester {
             stopSequences: params.stop,
             temperature: params.temperature,
             maxOutputTokens: params.model.includes("vision") ? void 0 : params.maxTokens,
-            topP: params.topP
+            topP: params.topP,
+            responseModalities: params.model.includes(
+              // TODO: Wait for google release to all models
+              "gemini-2.0-flash-exp"
+            ) && this._pluginConfig.imageGeneration ? ["TEXT", "IMAGE"] : void 0
             // thinkingConfig: { includeThoughts: true }
           },
           tools: params.tools != null || this._pluginConfig.googleSearch ? formatToolsToGeminiAITools(
@@ -408,8 +374,10 @@ var GeminiRequester = class extends ModelRequester {
           }
           for (const candidate of transformValue.candidates) {
             const parts = candidate.content?.parts;
-            if (parts == null || parts.length < 1) {
+            if ((parts == null || parts.length < 1) && candidate.finishReason !== "STOP") {
               throw new Error(chunk);
+            } else if (candidate.finishReason === "STOP" && parts == null) {
+              continue;
             }
             for (const part of parts) {
               controller.enqueue(part);
@@ -434,14 +402,21 @@ var GeminiRequester = class extends ModelRequester {
       for await (const chunk of iterable) {
         const messagePart = partAsType(chunk);
         const chatFunctionCallingPart = partAsType(chunk);
+        const imagePart = partAsTypeCheck(
+          chunk,
+          (part) => part["inlineData"] != null
+        );
         if (messagePart.text) {
           if (messagePart.thought) {
             reasoningContent += messagePart.text;
             continue;
           }
           content = messagePart.text;
+        } else if (imagePart) {
+          messagePart.text = `![image](data:${imagePart.inlineData.mime_type};base64,${imagePart.inlineData.data})`;
+          content = messagePart.text;
         }
-        const deltaFunctionCall = chatFunctionCallingPart.functionCall;
+        const deltaFunctionCall = chatFunctionCallingPart?.functionCall;
         if (deltaFunctionCall) {
           let args = deltaFunctionCall.args?.input ?? deltaFunctionCall.args;
           try {
@@ -466,7 +441,10 @@ var GeminiRequester = class extends ModelRequester {
               name: functionCall.name,
               arguments: functionCall.args,
               args: functionCall.arguments
-            } : void 0
+            } : void 0,
+            images: imagePart ? [
+              `data:${imagePart.inlineData.mime_type};base64,${imagePart.inlineData.data})`
+            ] : void 0
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
           };
           messageChunk.content = content;
@@ -760,6 +738,7 @@ var Config3 = Schema.intersect([
     maxTokens: Schema.number().min(16).max(2097e3).step(16).default(8064),
     temperature: Schema.percent().min(0).max(2).step(0.1).default(0.8),
     googleSearch: Schema.boolean().default(false),
+    imageGeneration: Schema.boolean().default(false),
     groundingContentDisplay: Schema.boolean().default(false),
     searchThreshold: Schema.number().min(0).max(1).step(0.1).default(0.5)
   })

package/lib/types.d.ts

@@ -2,11 +2,17 @@ export interface ChatCompletionResponseMessage {
     role: string;
     parts?: ChatPart[];
 }
-export type ChatPart = ChatMessagePart | ChatUploadDataPart | ChatFunctionCallingPart | ChatFunctionResponsePart;
+export type ChatPart = ChatMessagePart | ChatInlineDataPart | ChatFunctionCallingPart | ChatFunctionResponsePart | ChatUploadDataPart;
 export type ChatMessagePart = {
     text: string;
     thought?: boolean;
 };
+export type ChatInlineDataPart = {
+    inlineData: {
+        mime_type: string;
+        data?: string;
+    };
+};
 export type ChatUploadDataPart = {
     inline_data: {
         mime_type: string;

package/lib/utils.d.ts

@@ -4,6 +4,7 @@ import { ChatCompletionFunction, ChatCompletionResponseMessage, ChatCompletionRe
 import { Config } from '.';
 export declare function langchainMessageToGeminiMessage(messages: BaseMessage[], model?: string): Promise<ChatCompletionResponseMessage[]>;
 export declare function partAsType<T extends ChatPart>(part: ChatPart): T;
+export declare function partAsTypeCheck<T extends ChatPart>(part: ChatPart, check: (part: ChatPart & unknown) => boolean): T | undefined;
 export declare function formatToolsToGeminiAITools(tools: StructuredTool[], config: Config, model: string): Record<string, any>;
 export declare function formatToolToGeminiAITool(tool: StructuredTool): ChatCompletionFunction;
 export declare function messageTypeToGeminiRole(type: MessageType): ChatCompletionResponseMessageRoleEnum;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "koishi-plugin-chatluna-google-gemini-adapter",
   "description": "google-gemini adapter for chatluna",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "main": "lib/index.cjs",
   "module": "lib/index.mjs",
   "typings": "lib/index.d.ts",
@@ -72,7 +72,7 @@
   },
   "peerDependencies": {
     "koishi": "^4.18.4",
-    "koishi-plugin-chatluna": "^1.1.0"
+    "koishi-plugin-chatluna": "^1.1.2"
   },
   "koishi": {
     "description": {