koishi-plugin-chatluna-google-gemini-adapter 1.0.0-beta.3 → 1.0.0-beta.6

package/lib/client.js

@@ -30,10 +30,11 @@ class GeminiClient extends client_1.PlatformModelAndEmbeddingsClient {
                 .map((model) => {
                 return {
                     name: model,
+                    maxTokens: model.includes('vision') ? 12288 : 30720,
                     type: model.includes('embedding')
                         ? types_1.ModelType.embeddings
                         : types_1.ModelType.llm,
-                    functionCall: false,
+                    functionCall: !model.includes('vision'),
                     supportMode: ['all']
                 };
             });

package/lib/index.d.ts

@@ -9,4 +9,4 @@ export interface Config extends ChatLunaPlugin.Config {
 }
 export declare const Config: Schema<Config>;
 export declare const inject: string[];
-export declare const name = "chatluna-gemini-adapter";
+export declare const name = "chatluna-google-gemini-adapter";

package/lib/index.js

@@ -60,4 +60,4 @@ exports.Config = koishi_1.Schema.intersect([
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ]);
 exports.inject = ['chatluna'];
-exports.name = 'chatluna-gemini-adapter';
+exports.name = 'chatluna-google-gemini-adapter';

package/lib/requester.d.ts

@@ -1,6 +1,6 @@
 import { EmbeddingsRequester, EmbeddingsRequestParams, ModelRequester, ModelRequestParams } from 'koishi-plugin-chatluna/lib/llm-core/platform/api';
 import { ClientConfig } from 'koishi-plugin-chatluna/lib/llm-core/platform/config';
-import { ChatGenerationChunk } from 'langchain/schema';
+import { ChatGenerationChunk } from '@langchain/core/outputs';
 export declare class GeminiRequester extends ModelRequester implements EmbeddingsRequester {
     private _config;
     constructor(_config: ClientConfig);

package/lib/requester.js

@@ -2,7 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GeminiRequester = void 0;
 const api_1 = require("koishi-plugin-chatluna/lib/llm-core/platform/api");
-const schema_1 = require("langchain/schema");
+const messages_1 = require("@langchain/core/messages");
+const outputs_1 = require("@langchain/core/outputs");
 const error_1 = require("koishi-plugin-chatluna/lib/utils/error");
 const sse_1 = require("koishi-plugin-chatluna/lib/utils/sse");
 const utils_1 = require("./utils");
@@ -19,23 +20,23 @@ class GeminiRequester extends api_1.ModelRequester {
     async *completionStream(params) {
         try {
             const response = await this._post(`models/${params.model}:streamGenerateContent`, {
-                contents: (0, utils_1.langchainMessageToGeminiMessage)(params.input, params.model),
+                contents: await (0, utils_1.langchainMessageToGeminiMessage)(params.input, params.model),
                 safetySettings: [
                     {
                         category: 'HARM_CATEGORY_HARASSMENT',
-                        threshold: 'BLOCK_ONLY_HIGH'
+                        threshold: 'BLOCK_NONE'
                     },
                     {
                         category: 'HARM_CATEGORY_HATE_SPEECH',
-                        threshold: 'BLOCK_ONLY_HIGH'
+                        threshold: 'BLOCK_NONE'
                     },
                     {
                         category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT',
-                        threshold: 'BLOCK_ONLY_HIGH'
+                        threshold: 'BLOCK_NONE'
                     },
                     {
                         category: 'HARM_CATEGORY_DANGEROUS_CONTENT',
-                        threshold: 'BLOCK_ONLY_HIGH'
+                        threshold: 'BLOCK_NONE'
                     }
                 ],
                 generationConfig: {
@@ -45,7 +46,12 @@ class GeminiRequester extends api_1.ModelRequester {
                         ? undefined
                         : params.maxTokens,
                     topP: params.topP
-                }
+                },
+                tools: !params.model.includes('vision') && params.tools != null
+                    ? {
+                        functionDeclarations: (0, utils_1.formatToolsToGeminiAITools)(params.tools)
+                    }
+                    : undefined
             }, {
                 signal: params.signal
             });
@@ -55,35 +61,79 @@ class GeminiRequester extends api_1.ModelRequester {
             const jsonParser = new json_1.JSONParser();
             const writable = stream.writable.getWriter();
             jsonParser.onEnd = async () => {
-                await writable.write('[DONE]');
+                await writable.close();
             };
             jsonParser.onValue = async ({ value }) => {
                 const transformValue = value;
                 if (transformValue.candidates && transformValue.candidates[0]) {
-                    const parts = transformValue.candidates[0].content
-                        .parts;
+                    const parts = transformValue.candidates[0].content.parts;
                     if (parts.length < 1) {
                         throw new Error(JSON.stringify(value));
                     }
-                    const text = parts[0].text;
-                    if (text) {
-                        await writable.write(text);
+                    for (const part of parts) {
+                        await writable.write(part);
                     }
                 }
             };
             await (0, sse_1.sse)(response, async (rawData) => {
                 jsonParser.write(rawData);
                 return true;
-            });
+            }, 10);
             let content = '';
+            let isVisionModel = params.model.includes('vision');
+            const functionCall = {
+                name: '',
+                args: '',
+                arguments: ''
+            };
             for await (const chunk of iterable) {
-                if (chunk === '[DONE]') {
-                    return;
+                const messagePart = (0, utils_1.partAsType)(chunk);
+                const chatFunctionCallingPart = (0, utils_1.partAsType)(chunk);
+                if (messagePart.text) {
+                    content += messagePart.text;
+                    // match /w*model:
+                    if (isVisionModel && /\s*model:\s*/.test(content)) {
+                        isVisionModel = false;
+                        content = content.replace(/\s*model:\s*/, '');
+                    }
+                }
+                if (chatFunctionCallingPart.functionCall) {
+                    const deltaFunctionCall = chatFunctionCallingPart.functionCall;
+                    if (deltaFunctionCall) {
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        let args = deltaFunctionCall.args?.input ??
+                            deltaFunctionCall.args;
+                        try {
+                            let parsedArgs = JSON.parse(args);
+                            if (typeof parsedArgs !== 'string') {
+                                args = parsedArgs;
+                            }
+                            parsedArgs = JSON.parse(args);
+                            if (typeof parsedArgs !== 'string') {
+                                args = parsedArgs;
+                            }
+                        }
+                        catch (e) { }
+                        functionCall.args = JSON.stringify(args);
+                        functionCall.name =
+                            functionCall.name + (deltaFunctionCall.name ?? '');
+                        functionCall.arguments = deltaFunctionCall.args;
+                    }
                 }
                 try {
-                    const messageChunk = new schema_1.AIMessageChunk(chunk);
-                    messageChunk.content = content + messageChunk.content;
-                    const generationChunk = new schema_1.ChatGenerationChunk({
+                    const messageChunk = new messages_1.AIMessageChunk(content);
+                    messageChunk.additional_kwargs = {
+                        function_call: functionCall.name.length > 0
+                            ? {
+                                name: functionCall.name,
+                                arguments: functionCall.args,
+                                args: functionCall.arguments
+                            }
+                            : undefined
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    };
+                    messageChunk.content = content;
+                    const generationChunk = new outputs_1.ChatGenerationChunk({
                         message: messageChunk,
                         text: messageChunk.content
                     });
@@ -175,7 +225,6 @@ class GeminiRequester extends api_1.ModelRequester {
             }
         }
         const body = JSON.stringify(data);
-        // console.log('POST', requestUrl, body)
         return (0, request_1.chatLunaFetch)(requestUrl, {
             body,
             headers: this._buildHeaders(),

package/lib/types.d.ts

@@ -1,7 +1,8 @@
 export interface ChatCompletionResponseMessage {
     role: string;
-    parts?: (ChatMessagePart | ChatUploadDataPart)[];
+    parts?: ChatPart[];
 }
+export type ChatPart = ChatMessagePart | ChatUploadDataPart | ChatFunctionCallingPart | ChatFunctionResponsePart;
 export type ChatMessagePart = {
     text: string;
 };
@@ -11,6 +12,18 @@ export type ChatUploadDataPart = {
         data?: string;
     };
 };
+export type ChatFunctionCallingPart = {
+    functionCall: {
+        name: string;
+        args?: any;
+    };
+};
+export type ChatFunctionResponsePart = {
+    functionResponse: {
+        name: string;
+        response: any;
+    };
+};
 export interface ChatResponse {
     candidates: {
         content: ChatCompletionResponseMessage;
@@ -28,9 +41,20 @@ export interface ChatResponse {
         }[];
     };
 }
+export interface ChatCompletionFunction {
+    name: string;
+    description?: string;
+    parameters?: {
+        [key: string]: any;
+    };
+}
+export interface ChatCompletionMessageFunctionCall {
+    name: string;
+    args?: any;
+}
 export interface CreateEmbeddingResponse {
     embedding: {
         values: number[];
     };
 }
-export type ChatCompletionResponseMessageRoleEnum = 'system' | 'model' | 'user';
+export type ChatCompletionResponseMessageRoleEnum = 'system' | 'model' | 'user' | 'function';

package/lib/utils.d.ts

@@ -1,5 +1,9 @@
-import { AIMessageChunk, BaseMessage, ChatMessageChunk, HumanMessageChunk, MessageType, SystemMessageChunk } from 'langchain/schema';
-import { ChatCompletionResponseMessage, ChatCompletionResponseMessageRoleEnum } from './types';
-export declare function langchainMessageToGeminiMessage(messages: BaseMessage[], model?: string): ChatCompletionResponseMessage[];
+import { AIMessageChunk, BaseMessage, ChatMessageChunk, HumanMessageChunk, MessageType, SystemMessageChunk } from '@langchain/core/messages';
+import { ChatCompletionFunction, ChatCompletionResponseMessage, ChatCompletionResponseMessageRoleEnum, ChatPart } from './types';
+import { StructuredTool } from '@langchain/core/tools';
+export declare function langchainMessageToGeminiMessage(messages: BaseMessage[], model?: string): Promise<ChatCompletionResponseMessage[]>;
+export declare function partAsType<T extends ChatPart>(part: ChatPart): T;
+export declare function formatToolsToGeminiAITools(tools: StructuredTool[]): ChatCompletionFunction[];
+export declare function formatToolToGeminiAITool(tool: StructuredTool): ChatCompletionFunction;
 export declare function messageTypeToGeminiRole(type: MessageType): ChatCompletionResponseMessageRoleEnum;
-export declare function convertDeltaToMessageChunk(delta: Record<string, any>, defaultRole?: ChatCompletionResponseMessageRoleEnum): HumanMessageChunk | AIMessageChunk | SystemMessageChunk | ChatMessageChunk;
+export declare function convertDeltaToMessageChunk(delta: Record<string, any>, defaultRole?: ChatCompletionResponseMessageRoleEnum): AIMessageChunk | HumanMessageChunk | SystemMessageChunk | ChatMessageChunk;

package/lib/utils.js

@@ -1,11 +1,81 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.convertDeltaToMessageChunk = exports.messageTypeToGeminiRole = exports.langchainMessageToGeminiMessage = void 0;
-const schema_1 = require("langchain/schema");
-function langchainMessageToGeminiMessage(messages, model) {
-    // TODO: image vision
-    const mappedMessage = messages.map((rawMessage) => {
+exports.convertDeltaToMessageChunk = exports.messageTypeToGeminiRole = exports.formatToolToGeminiAITool = exports.formatToolsToGeminiAITools = exports.partAsType = exports.langchainMessageToGeminiMessage = void 0;
+/* eslint-disable @typescript-eslint/no-explicit-any */
+const messages_1 = require("@langchain/core/messages");
+const zod_to_json_schema_1 = require("zod-to-json-schema");
+async function langchainMessageToGeminiMessage(messages, model) {
+    const mappedMessage = await Promise.all(messages.map(async (rawMessage) => {
         const role = messageTypeToGeminiRole(rawMessage._getType());
+        if (role === 'function' ||
+            rawMessage.additional_kwargs?.function_call != null) {
+            return {
+                role: 'function',
+                parts: [
+                    {
+                        functionResponse: rawMessage.additional_kwargs?.function_call !=
+                            null
+                            ? undefined
+                            : {
+                                name: rawMessage.name,
+                                response: {
+                                    name: rawMessage.name,
+                                    content: (() => {
+                                        try {
+                                            const result = JSON.parse(rawMessage.content);
+                                            if (typeof result ===
+                                                'string') {
+                                                return {
+                                                    response: result
+                                                };
+                                            }
+                                            else {
+                                                return result;
+                                            }
+                                        }
+                                        catch (e) {
+                                            return {
+                                                response: rawMessage.content
+                                            };
+                                        }
+                                    })()
+                                }
+                            },
+                        functionCall: rawMessage.additional_kwargs?.function_call !=
+                            null
+                            ? {
+                                name: rawMessage.additional_kwargs
+                                    .function_call.name,
+                                args: (() => {
+                                    try {
+                                        const result = JSON.parse(rawMessage
+                                            .additional_kwargs
+                                            .function_call
+                                            .arguments);
+                                        if (typeof result === 'string') {
+                                            return {
+                                                input: result
+                                            };
+                                        }
+                                        else {
+                                            return result;
+                                        }
+                                    }
+                                    catch (e) {
+                                        return {
+                                            input: rawMessage
+                                                .additional_kwargs
+                                                .function_call
+                                                .arguments
+                                        };
+                                    }
+                                })()
+                            }
+                            : undefined
+                    }
+                ]
+            };
+        }
         const images = rawMessage.additional_kwargs.images;
         const result = {
             role,
@@ -20,14 +90,14 @@ function langchainMessageToGeminiMessage(messages, model) {
                 result.parts.push({
                     inline_data: {
                         // base64 image match type
-                        data: image,
+                        data: image.replace(/^data:image\/\w+;base64,/, ''),
                         mime_type: 'image/jpeg'
                     }
                 });
             }
         }
         return result;
-    });
+    }));
     const result = [];
     for (let i = 0; i < mappedMessage.length; i++) {
         const message = mappedMessage[i];
@@ -50,7 +120,7 @@ function langchainMessageToGeminiMessage(messages, model) {
             parts: [{ text: 'Okay, what do I need to do?' }]
         });
     }
-    if (result[result.length - 1].role === 'assistant') {
+    if (result[result.length - 1].role === 'model') {
         result.push({
             role: 'user',
             parts: [
@@ -60,9 +130,74 @@ function langchainMessageToGeminiMessage(messages, model) {
             ]
         });
     }
+    if (model.includes('vision')) {
+        // format prompts
+        const textBuffer = [];
+        const last = result.pop();
+        for (let i = 0; i < result.length; i++) {
+            const message = result[i];
+            const text = message.parts[0].text;
+            textBuffer.push(`${message.role}: ${text}`);
+        }
+        const lastParts = last.parts;
+        let lastImagesParts = lastParts.filter((part) => part.inline_data?.mime_type ===
+            'image/jpeg');
+        if (lastImagesParts.length < 1) {
+            for (let i = result.length - 1; i >= 0; i--) {
+                const message = result[i];
+                const images = message.parts.filter((part) => part.inline_data?.mime_type ===
+                    'image/jpeg');
+                if (images.length > 0) {
+                    lastImagesParts = images;
+                    break;
+                }
+            }
+        }
+        ;
+        lastParts.filter((part) => part.text !== undefined &&
+            part.text !== null).forEach((part) => {
+            textBuffer.push(`${last.role}: ${part.text}`);
+        });
+        return [
+            {
+                role: 'user',
+                parts: [
+                    {
+                        text: textBuffer.join('\n')
+                    },
+                    ...lastImagesParts
+                ]
+            }
+        ];
+    }
     return result;
 }
 exports.langchainMessageToGeminiMessage = langchainMessageToGeminiMessage;
+function partAsType(part) {
+    return part;
+}
+exports.partAsType = partAsType;
+function formatToolsToGeminiAITools(tools) {
+    if (tools.length < 1) {
+        return undefined;
+    }
+    return tools.map(formatToolToGeminiAITool);
+}
+exports.formatToolsToGeminiAITools = formatToolsToGeminiAITools;
+function formatToolToGeminiAITool(tool) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const parameters = (0, zod_to_json_schema_1.zodToJsonSchema)(tool.schema);
+    // remove unsupported properties
+    delete parameters['$schema'];
+    delete parameters['additionalProperties'];
+    return {
+        name: tool.name,
+        description: tool.description,
+        // any?
+        parameters
+    };
+}
+exports.formatToolToGeminiAITool = formatToolToGeminiAITool;
 function messageTypeToGeminiRole(type) {
     switch (type) {
         case 'system':
@@ -71,6 +206,8 @@ function messageTypeToGeminiRole(type) {
             return 'model';
         case 'human':
             return 'user';
+        case 'function':
+            return 'function';
         default:
             throw new Error(`Unknown message type: ${type}`);
     }
@@ -97,16 +234,16 @@ delta, defaultRole) {
         additional_kwargs = {};
     }
     if (role === 'user') {
-        return new schema_1.HumanMessageChunk({ content });
+        return new messages_1.HumanMessageChunk({ content });
     }
     else if (role === 'assistant') {
-        return new schema_1.AIMessageChunk({ content, additional_kwargs });
+        return new messages_1.AIMessageChunk({ content, additional_kwargs });
     }
     else if (role === 'system') {
-        return new schema_1.SystemMessageChunk({ content });
+        return new messages_1.SystemMessageChunk({ content });
     }
     else {
-        return new schema_1.ChatMessageChunk({ content, role });
+        return new messages_1.ChatMessageChunk({ content, role });
     }
 }
 exports.convertDeltaToMessageChunk = convertDeltaToMessageChunk;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "koishi-plugin-chatluna-google-gemini-adapter",
   "description": "google-gemini adapter for chatluna",
-  "version": "1.0.0-beta.3",
+  "version": "1.0.0-beta.6",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
   "files": [
@@ -38,16 +38,18 @@
     "adapter"
   ],
   "dependencies": {
+    "@langchain/core": "^0.1.10",
     "@streamparser/json": "^0.0.19",
-    "langchain": "^0.0.186"
+    "zod": "^3.22.4",
+    "zod-to-json-schema": "^3.22.3"
   },
   "devDependencies": {
     "atsc": "^1.2.2",
-    "koishi": "^4.16.1"
+    "koishi": "^4.16.3"
   },
   "peerDependencies": {
     "koishi": "^4.16.0",
-    "koishi-plugin-chatluna": "^1.0.0-beta.28"
+    "koishi-plugin-chatluna": "^1.0.0-beta.32"
   },
   "koishi": {
     "description": {