@lobehub/chat 1.138.3 → 1.138.5

package/packages/model-runtime/src/core/contextBuilders/google.ts

@@ -0,0 +1,201 @@
+import {
+  Content,
+  FunctionDeclaration,
+  Tool as GoogleFunctionCallTool,
+  Part,
+  Type as SchemaType,
+} from '@google/genai';
+
+import { ChatCompletionTool, OpenAIChatMessage, UserMessageContentPart } from '../../types';
+import { imageUrlToBase64 } from '../../utils/imageToBase64';
+import { safeParseJSON } from '../../utils/safeParseJSON';
+import { parseDataUri } from '../../utils/uriParser';
+
+/**
+ * Convert OpenAI content part to Google Part format
+ */
+export const buildGooglePart = async (
+  content: UserMessageContentPart,
+): Promise<Part | undefined> => {
+  switch (content.type) {
+    default: {
+      return undefined;
+    }
+
+    case 'text': {
+      return { text: content.text };
+    }
+
+    case 'image_url': {
+      const { mimeType, base64, type } = parseDataUri(content.image_url.url);
+
+      if (type === 'base64') {
+        if (!base64) {
+          throw new TypeError("Image URL doesn't contain base64 data");
+        }
+
+        return {
+          inlineData: { data: base64, mimeType: mimeType || 'image/png' },
+        };
+      }
+
+      if (type === 'url') {
+        const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
+
+        return {
+          inlineData: { data: base64, mimeType },
+        };
+      }
+
+      throw new TypeError(`currently we don't support image url: ${content.image_url.url}`);
+    }
+
+    case 'video_url': {
+      const { mimeType, base64, type } = parseDataUri(content.video_url.url);
+
+      if (type === 'base64') {
+        if (!base64) {
+          throw new TypeError("Video URL doesn't contain base64 data");
+        }
+
+        return {
+          inlineData: { data: base64, mimeType: mimeType || 'video/mp4' },
+        };
+      }
+
+      if (type === 'url') {
+        // For video URLs, we need to fetch and convert to base64
+        // Note: This might need size/duration limits for practical use
+        const response = await fetch(content.video_url.url);
+        const arrayBuffer = await response.arrayBuffer();
+        const base64 = Buffer.from(arrayBuffer).toString('base64');
+        const mimeType = response.headers.get('content-type') || 'video/mp4';
+
+        return {
+          inlineData: { data: base64, mimeType },
+        };
+      }
+
+      throw new TypeError(`currently we don't support video url: ${content.video_url.url}`);
+    }
+  }
+};
+
+/**
+ * Convert OpenAI message to Google Content format
+ */
+export const buildGoogleMessage = async (
+  message: OpenAIChatMessage,
+  toolCallNameMap?: Map<string, string>,
+): Promise<Content> => {
+  const content = message.content as string | UserMessageContentPart[];
+
+  // Handle assistant messages with tool_calls
+  if (!!message.tool_calls) {
+    return {
+      parts: message.tool_calls.map<Part>((tool) => ({
+        functionCall: {
+          args: safeParseJSON(tool.function.arguments)!,
+          name: tool.function.name,
+        },
+      })),
+      role: 'model',
+    };
+  }
+
+  // Convert tool_call result to functionResponse part
+  if (message.role === 'tool' && toolCallNameMap && message.tool_call_id) {
+    const functionName = toolCallNameMap.get(message.tool_call_id);
+    if (functionName) {
+      return {
+        parts: [
+          {
+            functionResponse: {
+              name: functionName,
+              response: { result: message.content },
+            },
+          },
+        ],
+        role: 'user',
+      };
+    }
+  }
+
+  const getParts = async () => {
+    if (typeof content === 'string') return [{ text: content }];
+
+    const parts = await Promise.all(content.map(async (c) => await buildGooglePart(c)));
+    return parts.filter(Boolean) as Part[];
+  };
+
+  return {
+    parts: await getParts(),
+    role: message.role === 'assistant' ? 'model' : 'user',
+  };
+};
+
+/**
+ * Convert messages from the OpenAI format to Google GenAI SDK format
+ */
+export const buildGoogleMessages = async (messages: OpenAIChatMessage[]): Promise<Content[]> => {
+  const toolCallNameMap = new Map<string, string>();
+
+  // Build tool call id to name mapping
+  messages.forEach((message) => {
+    if (message.role === 'assistant' && message.tool_calls) {
+      message.tool_calls.forEach((toolCall) => {
+        if (toolCall.type === 'function') {
+          toolCallNameMap.set(toolCall.id, toolCall.function.name);
+        }
+      });
+    }
+  });
+
+  const pools = messages
+    .filter((message) => message.role !== 'function')
+    .map(async (msg) => await buildGoogleMessage(msg, toolCallNameMap));
+
+  const contents = await Promise.all(pools);
+
+  // Filter out empty messages: contents.parts must not be empty.
+  return contents.filter((content: Content) => content.parts && content.parts.length > 0);
+};
+
+/**
+ * Convert ChatCompletionTool to Google FunctionDeclaration
+ */
+export const buildGoogleTool = (tool: ChatCompletionTool): FunctionDeclaration => {
+  const functionDeclaration = tool.function;
+  const parameters = functionDeclaration.parameters;
+  // refs: https://github.com/lobehub/lobe-chat/pull/5002
+  const properties =
+    parameters?.properties && Object.keys(parameters.properties).length > 0
+      ? parameters.properties
+      : { dummy: { type: 'string' } }; // dummy property to avoid empty object
+
+  return {
+    description: functionDeclaration.description,
+    name: functionDeclaration.name,
+    parameters: {
+      description: parameters?.description,
+      properties: properties,
+      required: parameters?.required,
+      type: SchemaType.OBJECT,
+    },
+  };
+};
+
+/**
+ * Build Google function declarations from ChatCompletionTool array
+ */
+export const buildGoogleTools = (
+  tools: ChatCompletionTool[] | undefined,
+): GoogleFunctionCallTool[] | undefined => {
+  if (!tools || tools.length === 0) return;
+
+  return [
+    {
+      functionDeclarations: tools.map((tool) => buildGoogleTool(tool)),
+    },
+  ];
+};