koishi-plugin-chatluna-google-gemini-adapter 1.0.0-beta.2 → 1.0.0-beta.4

package/lib/client.js

@@ -22,11 +22,15 @@ class GeminiClient extends client_1.PlatformModelAndEmbeddingsClient {
     async refreshModels() {
         try {
             const rawModels = await this._requester.getModels();
+            if (!rawModels.length) {
+                throw new error_1.ChatLunaError(error_1.ChatLunaErrorCode.MODEL_INIT_ERROR, new Error('No model found'));
+            }
             return rawModels
                 .map((model) => model.replace('models/', ''))
                 .map((model) => {
                 return {
                     name: model,
+                    maxTokens: model.includes('vision') ? 12288 : 30720,
                     type: model.includes('embedding')
                         ? types_1.ModelType.embeddings
                         : types_1.ModelType.llm,

package/lib/requester.js

@@ -19,7 +19,7 @@ class GeminiRequester extends api_1.ModelRequester {
     async *completionStream(params) {
         try {
             const response = await this._post(`models/${params.model}:streamGenerateContent`, {
-                contents: (0, utils_1.langchainMessageToGeminiMessage)(params.input, params.model),
+                contents: await (0, utils_1.langchainMessageToGeminiMessage)(params.input, params.model),
                 safetySettings: [
                     {
                         category: 'HARM_CATEGORY_HARASSMENT',
@@ -66,20 +66,28 @@ class GeminiRequester extends api_1.ModelRequester {
                         throw new Error(JSON.stringify(value));
                     }
                     const text = parts[0].text;
+                    _1.logger.debug('text', text);
                     if (text) {
                         await writable.write(text);
                     }
                 }
             };
             await (0, sse_1.sse)(response, async (rawData) => {
+                _1.logger.debug('chunk', rawData);
                 jsonParser.write(rawData);
                 return true;
             });
             let content = '';
-            for await (const chunk of iterable) {
+            let isVisionModel = params.model.includes('vision');
+            for await (let chunk of iterable) {
                 if (chunk === '[DONE]') {
                     return;
                 }
+                // match /w*model:
+                if (isVisionModel && /\s*model:\s*/.test(chunk)) {
+                    isVisionModel = false;
+                    chunk = chunk.replace(/\s*model:\s*/, '');
+                }
                 try {
                     const messageChunk = new schema_1.AIMessageChunk(chunk);
                     messageChunk.content = content + messageChunk.content;
@@ -149,6 +157,10 @@ class GeminiRequester extends api_1.ModelRequester {
             const response = await this._get('models');
             data = await response.text();
             data = JSON.parse(data);
+            if (!data.models || !data.models.length) {
+                throw new Error('error when listing gemini models, Result:' +
+                    JSON.stringify(data));
+            }
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             return data.models
                 .map((model) => model.name)

package/lib/utils.d.ts

@@ -1,5 +1,5 @@
 import { AIMessageChunk, BaseMessage, ChatMessageChunk, HumanMessageChunk, MessageType, SystemMessageChunk } from 'langchain/schema';
 import { ChatCompletionResponseMessage, ChatCompletionResponseMessageRoleEnum } from './types';
-export declare function langchainMessageToGeminiMessage(messages: BaseMessage[], model?: string): ChatCompletionResponseMessage[];
+export declare function langchainMessageToGeminiMessage(messages: BaseMessage[], model?: string): Promise<ChatCompletionResponseMessage[]>;
 export declare function messageTypeToGeminiRole(type: MessageType): ChatCompletionResponseMessageRoleEnum;
 export declare function convertDeltaToMessageChunk(delta: Record<string, any>, defaultRole?: ChatCompletionResponseMessageRoleEnum): HumanMessageChunk | AIMessageChunk | SystemMessageChunk | ChatMessageChunk;

package/lib/utils.js

@@ -2,9 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.convertDeltaToMessageChunk = exports.messageTypeToGeminiRole = exports.langchainMessageToGeminiMessage = void 0;
 const schema_1 = require("langchain/schema");
-function langchainMessageToGeminiMessage(messages, model) {
-    // TODO: image vision
-    const mappedMessage = messages.map((rawMessage) => {
+async function langchainMessageToGeminiMessage(messages, model) {
+    const mappedMessage = await Promise.all(messages.map(async (rawMessage) => {
         const role = messageTypeToGeminiRole(rawMessage._getType());
         const images = rawMessage.additional_kwargs.images;
         const result = {
@@ -20,14 +19,14 @@ function langchainMessageToGeminiMessage(messages, model) {
                 result.parts.push({
                     inline_data: {
                         // base64 image match type
-                        data: image,
+                        data: image.replace(/^data:image\/\w+;base64,/, ''),
                         mime_type: 'image/jpeg'
                     }
                 });
             }
         }
         return result;
-    });
+    }));
     const result = [];
     for (let i = 0; i < mappedMessage.length; i++) {
         const message = mappedMessage[i];
@@ -60,6 +59,46 @@ function langchainMessageToGeminiMessage(messages, model) {
             ]
         });
     }
+    if (model.includes('vision')) {
+        // format prompts
+        const textBuffer = [];
+        const last = result.pop();
+        for (let i = 0; i < result.length; i++) {
+            const message = result[i];
+            const text = message.parts[0].text;
+            textBuffer.push(`${message.role}: ${text}`);
+        }
+        const lastParts = last.parts;
+        let lastImagesParts = lastParts.filter((part) => part.inline_data?.mime_type ===
+            'image/jpeg');
+        if (lastImagesParts.length < 1) {
+            for (let i = result.length - 1; i >= 0; i--) {
+                const message = result[i];
+                const images = message.parts.filter((part) => part.inline_data?.mime_type ===
+                    'image/jpeg');
+                if (images.length > 0) {
+                    lastImagesParts = images;
+                    break;
+                }
+            }
+        }
+        ;
+        lastParts.filter((part) => part.text !== undefined &&
+            part.text !== null).forEach((part) => {
+            textBuffer.push(`${last.role}: ${part.text}`);
+        });
+        return [
+            {
+                role: 'user',
+                parts: [
+                    {
+                        text: textBuffer.join('\n')
+                    },
+                    ...lastImagesParts
+                ]
+            }
+        ];
+    }
     return result;
 }
 exports.langchainMessageToGeminiMessage = langchainMessageToGeminiMessage;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "koishi-plugin-chatluna-google-gemini-adapter",
   "description": "google-gemini adapter for chatluna",
-  "version": "1.0.0-beta.2",
+  "version": "1.0.0-beta.4",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
   "files": [
@@ -43,11 +43,11 @@
   },
   "devDependencies": {
     "atsc": "^1.2.2",
-    "koishi": "^4.16.1"
+    "koishi": "^4.16.3"
   },
   "peerDependencies": {
     "koishi": "^4.16.0",
-    "koishi-plugin-chatluna": "^1.0.0-beta.28"
+    "koishi-plugin-chatluna": "^1.0.0-beta.31"
   },
   "koishi": {
     "description": {