@lobehub/chat 1.138.3 → 1.138.5

package/packages/model-runtime/src/providers/google/generateObject.ts

@@ -1,9 +1,15 @@
-import { GenerateContentConfig, GoogleGenAI, Type as SchemaType } from '@google/genai';
+import {
+  FunctionCallingConfigMode,
+  GenerateContentConfig,
+  GoogleGenAI,
+  Type as SchemaType,
+} from '@google/genai';
 import Debug from 'debug';
 
-import { GenerateObjectOptions } from '../../types';
+import { buildGoogleTool } from '../../core/contextBuilders/google';
+import { ChatCompletionTool, GenerateObjectOptions, GenerateObjectSchema } from '../../types';
 
-const debug = Debug('mode-runtime:google:generateObject');
+const debug = Debug('lobe-mode-runtime:google:generateObject');
 
 enum HarmCategory {
   HARM_CATEGORY_DANGEROUS_CONTENT = 'HARM_CATEGORY_DANGEROUS_CONTENT',
@@ -54,7 +60,7 @@ const convertType = (type: string): SchemaType => {
 /**
  * Convert OpenAI JSON schema to Google Gemini schema format
  */
-export const convertOpenAISchemaToGoogleSchema = (openAISchema: any): any => {
+export const convertOpenAISchemaToGoogleSchema = (openAISchema: GenerateObjectSchema): any => {
   const convertSchema = (schema: any): any => {
     if (!schema) return schema;
 
@@ -92,7 +98,7 @@ export const convertOpenAISchemaToGoogleSchema = (openAISchema: any): any => {
     return converted;
   };
 
-  return convertSchema(openAISchema);
+  return convertSchema(openAISchema.schema);
 };
 
 /**
@@ -104,7 +110,7 @@ export const createGoogleGenerateObject = async (
   payload: {
     contents: any[];
     model: string;
-    schema: any;
+    schema: GenerateObjectSchema;
   },
   options?: GenerateObjectOptions,
 ) => {
@@ -175,3 +181,95 @@ export const createGoogleGenerateObject = async (
     return undefined;
   }
 };
+
+/**
+ * Generate structured output using Google Gemini API with tools calling
+ * @see https://ai.google.dev/gemini-api/docs/function-calling
+ */
+export const createGoogleGenerateObjectWithTools = async (
+  client: GoogleGenAI,
+  payload: {
+    contents: any[];
+    model: string;
+    tools: ChatCompletionTool[];
+  },
+  options?: GenerateObjectOptions,
+) => {
+  const { tools, contents, model } = payload;
+
+  debug('createGoogleGenerateObjectWithTools started', {
+    contentsLength: contents.length,
+    model,
+    toolsCount: tools.length,
+  });
+
+  // Convert tools to Google FunctionDeclaration format
+  const functionDeclarations = tools.map(buildGoogleTool);
+  debug('Tools conversion completed', { functionDeclarations });
+
+  const config: GenerateContentConfig = {
+    abortSignal: options?.signal,
+    // avoid wide sensitive words
+    safetySettings: [
+      {
+        category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+        threshold: getThreshold(model),
+      },
+      {
+        category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+        threshold: getThreshold(model),
+      },
+      {
+        category: HarmCategory.HARM_CATEGORY_HARASSMENT,
+        threshold: getThreshold(model),
+      },
+      {
+        category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+        threshold: getThreshold(model),
+      },
+    ],
+    // Force tool calling with 'any' mode
+    toolConfig: {
+      functionCallingConfig: {
+        mode: FunctionCallingConfigMode.ANY,
+      },
+    },
+    tools: [{ functionDeclarations }],
+  };
+
+  debug('Config prepared', {
+    hasAbortSignal: !!config.abortSignal,
+    hasSafetySettings: !!config.safetySettings,
+    hasTools: !!config.tools,
+    model,
+  });
+
+  const response = await client.models.generateContent({
+    config,
+    contents,
+    model,
+  });
+
+  debug('API response received', {
+    candidatesCount: response.candidates?.length,
+    hasContent: !!response.candidates?.[0]?.content,
+  });
+
+  // Extract function calls from response
+  const candidate = response.candidates?.[0];
+  if (!candidate?.content?.parts) {
+    debug('no content parts in response');
+    return undefined;
+  }
+
+  const functionCalls = candidate.content.parts
+    .filter((part) => part.functionCall)
+    .map((part) => ({
+      arguments: part.functionCall!.args,
+      name: part.functionCall!.name,
+    }));
+
+  debug('extracted function calls', { count: functionCalls.length, functionCalls });
+
+  return functionCalls.length > 0 ? functionCalls : undefined;
+};

package/packages/model-runtime/src/providers/google/index.test.ts

@@ -432,401 +432,6 @@ describe('LobeGoogleAI', () => {
   });
 
   describe('private method', () => {
-    describe('convertContentToGooglePart', () => {
-      it('should handle text type messages', async () => {
-        const result = await instance['convertContentToGooglePart']({
-          type: 'text',
-          text: 'Hello',
-        });
-        expect(result).toEqual({ text: 'Hello' });
-      });
-      it('should handle thinking type messages', async () => {
-        const result = await instance['convertContentToGooglePart']({
-          type: 'thinking',
-          thinking: 'Hello',
-          signature: 'abc',
-        });
-        expect(result).toEqual(undefined);
-      });
-
-      it('should handle base64 type images', async () => {
-        const base64Image =
-          'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==';
-        const result = await instance['convertContentToGooglePart']({
-          type: 'image_url',
-          image_url: { url: base64Image },
-        });
-
-        expect(result).toEqual({
-          inlineData: {
-            data: 'iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==',
-            mimeType: 'image/png',
-          },
-        });
-      });
-
-      it('should handle URL type images', async () => {
-        const imageUrl = 'http://example.com/image.png';
-        const mockBase64 = 'mockBase64Data';
-
-        // Mock the imageUrlToBase64 function
-        vi.spyOn(imageToBase64Module, 'imageUrlToBase64').mockResolvedValueOnce({
-          base64: mockBase64,
-          mimeType: 'image/png',
-        });
-
-        const result = await instance['convertContentToGooglePart']({
-          type: 'image_url',
-          image_url: { url: imageUrl },
-        });
-
-        expect(result).toEqual({
-          inlineData: {
-            data: mockBase64,
-            mimeType: 'image/png',
-          },
-        });
-
-        expect(imageToBase64Module.imageUrlToBase64).toHaveBeenCalledWith(imageUrl);
-      });
-
-      it('should throw TypeError for unsupported image URL types', async () => {
-        const unsupportedImageUrl = 'unsupported://example.com/image.png';
-
-        await expect(
-          instance['convertContentToGooglePart']({
-            type: 'image_url',
-            image_url: { url: unsupportedImageUrl },
-          }),
-        ).rejects.toThrow(TypeError);
-      });
-    });
-
-    describe('buildGoogleMessages', () => {
-      it('get default result with gemini-pro', async () => {
-        const messages: OpenAIChatMessage[] = [{ content: 'Hello', role: 'user' }];
-
-        const contents = await instance['buildGoogleMessages'](messages);
-
-        expect(contents).toHaveLength(1);
-        expect(contents).toEqual([{ parts: [{ text: 'Hello' }], role: 'user' }]);
-      });
-
-      it('should not modify the length if model is gemini-1.5-pro', async () => {
-        const messages: OpenAIChatMessage[] = [
-          { content: 'Hello', role: 'user' },
-          { content: 'Hi', role: 'assistant' },
-        ];
-
-        const contents = await instance['buildGoogleMessages'](messages);
-
-        expect(contents).toHaveLength(2);
-        expect(contents).toEqual([
-          { parts: [{ text: 'Hello' }], role: 'user' },
-          { parts: [{ text: 'Hi' }], role: 'model' },
-        ]);
-      });
-
-      it('should use specified model when images are included in messages', async () => {
-        const messages: OpenAIChatMessage[] = [
-          {
-            content: [
-              { type: 'text', text: 'Hello' },
-              { type: 'image_url', image_url: { url: 'data:image/png;base64,...' } },
-            ],
-            role: 'user',
-          },
-        ];
-
-        // Call the buildGoogleMessages method
-        const contents = await instance['buildGoogleMessages'](messages);
-
-        expect(contents).toHaveLength(1);
-        expect(contents).toEqual([
-          {
-            parts: [{ text: 'Hello' }, { inlineData: { data: '...', mimeType: 'image/png' } }],
-            role: 'user',
-          },
-        ]);
-      });
-
-      it('should correctly convert function response message', async () => {
-        const messages: OpenAIChatMessage[] = [
-          {
-            content: '',
-            role: 'assistant',
-            tool_calls: [
-              {
-                id: 'call_1',
-                function: {
-                  name: 'get_current_weather',
-                  arguments: JSON.stringify({ location: 'London', unit: 'celsius' }),
-                },
-                type: 'function',
-              },
-            ],
-          },
-          {
-            content: '{"success":true,"data":{"temperature":"14°C"}}',
-            name: 'get_current_weather',
-            role: 'tool',
-            tool_call_id: 'call_1',
-          },
-        ];
-
-        const contents = await instance['buildGoogleMessages'](messages);
-        expect(contents).toHaveLength(2);
-        expect(contents).toEqual([
-          {
-            parts: [
-              {
-                functionCall: {
-                  args: { location: 'London', unit: 'celsius' },
-                  name: 'get_current_weather',
-                },
-              },
-            ],
-            role: 'model',
-          },
-          {
-            parts: [
-              {
-                functionResponse: {
-                  name: 'get_current_weather',
-                  response: { result: '{"success":true,"data":{"temperature":"14°C"}}' },
-                },
-              },
-            ],
-            role: 'user',
-          },
-        ]);
-      });
-    });
-
-    describe('buildGoogleTools', () => {
-      it('should return undefined when tools is undefined or empty', () => {
-        expect(instance['buildGoogleTools'](undefined)).toBeUndefined();
-        expect(instance['buildGoogleTools']([])).toBeUndefined();
-      });
-
-      it('should correctly convert ChatCompletionTool to GoogleFunctionCallTool', () => {
-        const tools: OpenAI.ChatCompletionTool[] = [
-          {
-            function: {
-              name: 'testTool',
-              description: 'A test tool',
-              parameters: {
-                type: 'object',
-                properties: {
-                  param1: { type: 'string' },
-                  param2: { type: 'number' },
-                },
-                required: ['param1'],
-              },
-            },
-            type: 'function',
-          },
-        ];
-
-        const googleTools = instance['buildGoogleTools'](tools);
-
-        expect(googleTools).toHaveLength(1);
-        expect((googleTools![0] as Tool).functionDeclarations![0]).toEqual({
-          name: 'testTool',
-          description: 'A test tool',
-          parameters: {
-            type: 'OBJECT',
-            properties: {
-              param1: { type: 'string' },
-              param2: { type: 'number' },
-            },
-            required: ['param1'],
-          },
-        });
-      });
-
-      it('should also add tools when tool_calls exists', () => {
-        const tools: OpenAI.ChatCompletionTool[] = [
-          {
-            function: {
-              name: 'testTool',
-              description: 'A test tool',
-              parameters: {
-                type: 'object',
-                properties: {
-                  param1: { type: 'string' },
-                  param2: { type: 'number' },
-                },
-                required: ['param1'],
-              },
-            },
-            type: 'function',
-          },
-        ];
-
-        const payload: ChatStreamPayload = {
-          messages: [
-            {
-              role: 'user',
-              content: '',
-              tool_calls: [
-                { function: { name: 'some_func', arguments: '' }, id: 'func_1', type: 'function' },
-              ],
-            },
-          ],
-          model: 'gemini-2.5-flash-preview-04-17',
-          temperature: 1,
-        };
-
-        const googleTools = instance['buildGoogleTools'](tools, payload);
-
-        expect(googleTools).toHaveLength(1);
-        expect((googleTools![0] as Tool).functionDeclarations![0]).toEqual({
-          name: 'testTool',
-          description: 'A test tool',
-          parameters: {
-            type: 'OBJECT',
-            properties: {
-              param1: { type: 'string' },
-              param2: { type: 'number' },
-            },
-            required: ['param1'],
-          },
-        });
-      });
-
-      it('should handle googleSearch', () => {
-        const payload: ChatStreamPayload = {
-          messages: [
-            {
-              role: 'user',
-              content: '',
-            },
-          ],
-          model: 'gemini-2.5-flash-preview-04-17',
-          temperature: 1,
-          enabledSearch: true,
-        };
-
-        const googleTools = instance['buildGoogleTools'](undefined, payload);
-
-        expect(googleTools).toHaveLength(1);
-        expect(googleTools![0] as Tool).toEqual({ googleSearch: {} });
-      });
-    });
-
-    describe('convertOAIMessagesToGoogleMessage', () => {
-      it('should correctly convert assistant message', async () => {
-        const message: OpenAIChatMessage = {
-          role: 'assistant',
-          content: 'Hello',
-        };
-
-        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
-
-        expect(converted).toEqual({
-          role: 'model',
-          parts: [{ text: 'Hello' }],
-        });
-      });
-
-      it('should correctly convert user message', async () => {
-        const message: OpenAIChatMessage = {
-          role: 'user',
-          content: 'Hi',
-        };
-
-        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
-
-        expect(converted).toEqual({
-          role: 'user',
-          parts: [{ text: 'Hi' }],
-        });
-      });
-
-      it('should correctly convert message with inline base64 image parts', async () => {
-        const message: OpenAIChatMessage = {
-          role: 'user',
-          content: [
-            { type: 'text', text: 'Check this image:' },
-            { type: 'image_url', image_url: { url: 'data:image/png;base64,...' } },
-          ],
-        };
-
-        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
-
-        expect(converted).toEqual({
-          role: 'user',
-          parts: [
-            { text: 'Check this image:' },
-            { inlineData: { data: '...', mimeType: 'image/png' } },
-          ],
-        });
-      });
-      it.skip('should correctly convert message with image url parts', async () => {
-        const message: OpenAIChatMessage = {
-          role: 'user',
-          content: [
-            { type: 'text', text: 'Check this image:' },
-            { type: 'image_url', image_url: { url: 'https://image-file.com' } },
-          ],
-        };
-
-        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
-
-        expect(converted).toEqual({
-          role: 'user',
-          parts: [
-            { text: 'Check this image:' },
-            { inlineData: { data: '...', mimeType: 'image/png' } },
-          ],
-        });
-      });
-
-      it('should correctly convert function call message', async () => {
-        const message = {
-          role: 'assistant',
-          tool_calls: [
-            {
-              id: 'call_1',
-              function: {
-                name: 'get_current_weather',
-                arguments: JSON.stringify({ location: 'London', unit: 'celsius' }),
-              },
-              type: 'function',
-            },
-          ],
-        } as OpenAIChatMessage;
-
-        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
-        expect(converted).toEqual({
-          role: 'model',
-          parts: [
-            {
-              functionCall: {
-                name: 'get_current_weather',
-                args: { location: 'London', unit: 'celsius' },
-              },
-            },
-          ],
-        });
-      });
-
-      it('should correctly handle empty content', async () => {
-        const message: OpenAIChatMessage = {
-          role: 'user',
-          content: '' as any, // explicitly set as empty string
-        };
-
-        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
-
-        expect(converted).toEqual({
-          role: 'user',
-          parts: [{ text: '' }],
-        });
-      });
-    });
-
     describe('createEnhancedStream', () => {
       it('should handle stream cancellation with data gracefully', async () => {
         const mockStream = (async function* () {