@lobehub/chat 1.70.11 → 1.71.0

package/locales/fr-FR/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash propose des fonctionnalités et des améliorations de nouvelle génération, y compris une vitesse exceptionnelle, l'utilisation d'outils natifs, la génération multimodale et une fenêtre de contexte de 1M tokens."
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Modèle variant Gemini 2.0 Flash, optimisé pour des objectifs tels que le rapport coût-efficacité et la faible latence."
+  },
   "gemini-2.0-flash-lite": {
     "description": "Une variante du modèle Gemini 2.0 Flash, optimisée pour des objectifs tels que le rapport coût-efficacité et la faible latence."
   },

package/locales/it-IT/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash offre funzionalità e miglioramenti di nuova generazione, tra cui velocità eccezionale, utilizzo di strumenti nativi, generazione multimodale e una finestra di contesto di 1M token."
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Gemini 2.0 Flash è una variante del modello ottimizzata per obiettivi come il rapporto costo-efficacia e la bassa latenza."
+  },
   "gemini-2.0-flash-lite": {
     "description": "Gemini 2.0 Flash è una variante del modello Flash, ottimizzata per obiettivi come il rapporto costo-efficacia e la bassa latenza."
   },

package/locales/ja-JP/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flashは、卓越した速度、ネイティブツールの使用、マルチモーダル生成、1Mトークンのコンテキストウィンドウを含む次世代の機能と改善を提供します。"
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Gemini 2.0 Flash モデルのバリアントで、コスト効率と低遅延などの目標に最適化されています。"
+  },
   "gemini-2.0-flash-lite": {
     "description": "Gemini 2.0 Flashモデルのバリアントで、コスト効率と低遅延などの目標に最適化されています。"
   },

package/locales/ko-KR/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash는 뛰어난 속도, 원주율 도구 사용, 다중 모달 생성 및 1M 토큰 문맥 창을 포함한 차세대 기능과 개선 사항을 제공합니다."
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Gemini 2.0 Flash 모델 변형으로, 비용 효율성과 저지연 등의 목표를 위해 최적화되었습니다."
+  },
   "gemini-2.0-flash-lite": {
     "description": "Gemini 2.0 플래시 모델 변형으로, 비용 효율성과 낮은 지연 시간 등의 목표를 위해 최적화되었습니다."
   },

package/locales/nl-NL/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash biedt next-gen functies en verbeteringen, waaronder uitstekende snelheid, native toolgebruik, multimodale generatie en een contextvenster van 1M tokens."
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Gemini 2.0 Flash modelvariant, geoptimaliseerd voor kosteneffectiviteit en lage latentie."
+  },
   "gemini-2.0-flash-lite": {
     "description": "Gemini 2.0 Flash is een modelvariant die is geoptimaliseerd voor kosteneffectiviteit en lage latentie."
   },

package/locales/pl-PL/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash oferuje funkcje i ulepszenia nowej generacji, w tym doskonałą prędkość, natywne korzystanie z narzędzi, generowanie multimodalne oraz okno kontekstowe o długości 1M tokenów."
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Gemini 2.0 Flash to wariant modelu, zoptymalizowany pod kątem efektywności kosztowej i niskiego opóźnienia."
+  },
   "gemini-2.0-flash-lite": {
     "description": "Gemini 2.0 Flash to wariant modelu, zoptymalizowany pod kątem efektywności kosztowej i niskiego opóźnienia."
   },

package/locales/pt-BR/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash oferece funcionalidades e melhorias de próxima geração, incluindo velocidade excepcional, uso nativo de ferramentas, geração multimodal e uma janela de contexto de 1M tokens."
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Variante do modelo Gemini 2.0 Flash, otimizada para custo-benefício e baixa latência."
+  },
   "gemini-2.0-flash-lite": {
     "description": "Variante do modelo Gemini 2.0 Flash, otimizada para custo-benefício e baixa latência."
   },

package/locales/ru-RU/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash предлагает функции следующего поколения и улучшения, включая выдающуюся скорость, использование встроенных инструментов, многомодальную генерацию и контекстное окно на 1M токенов."
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Модельный вариант Gemini 2.0 Flash, оптимизированный для достижения таких целей, как экономическая эффективность и низкая задержка."
+  },
   "gemini-2.0-flash-lite": {
     "description": "Модельный вариант Gemini 2.0 Flash, оптимизированный для достижения таких целей, как экономическая эффективность и низкая задержка."
   },

package/locales/tr-TR/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash, mükemmel hız, yerel araç kullanımı, çok modlu üretim ve 1M token bağlam penceresi dahil olmak üzere bir sonraki nesil özellikler ve iyileştirmeler sunar."
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Gemini 2.0 Flash modeli varyantı, maliyet etkinliği ve düşük gecikme gibi hedefler için optimize edilmiştir."
+  },
   "gemini-2.0-flash-lite": {
     "description": "Gemini 2.0 Flash model varyantı, maliyet etkinliği ve düşük gecikme gibi hedefler için optimize edilmiştir."
   },

package/locales/vi-VN/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash cung cấp các tính năng và cải tiến thế hệ tiếp theo, bao gồm tốc độ vượt trội, sử dụng công cụ bản địa, tạo đa phương tiện và cửa sổ ngữ cảnh 1M token."
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Biến thể mô hình Gemini 2.0 Flash, được tối ưu hóa cho hiệu quả chi phí và độ trễ thấp."
+  },
   "gemini-2.0-flash-lite": {
     "description": "Biến thể mô hình Gemini 2.0 Flash được tối ưu hóa cho hiệu quả chi phí và độ trễ thấp."
   },

package/locales/zh-CN/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash 提供下一代功能和改进，包括卓越的速度、原生工具使用、多模态生成和1M令牌上下文窗口。"
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Gemini 2.0 Flash 模型变体，针对成本效益和低延迟等目标进行了优化。"
+  },
   "gemini-2.0-flash-lite": {
     "description": "Gemini 2.0 Flash 模型变体，针对成本效益和低延迟等目标进行了优化。"
   },

package/locales/zh-TW/models.json

@@ -833,6 +833,9 @@
   "gemini-2.0-flash-001": {
     "description": "Gemini 2.0 Flash 提供下一代功能和改進，包括卓越的速度、原生工具使用、多模態生成和1M令牌上下文窗口。"
   },
+  "gemini-2.0-flash-exp": {
+    "description": "Gemini 2.0 Flash 模型變體，針對成本效益和低延遲等目標進行了優化。"
+  },
   "gemini-2.0-flash-lite": {
     "description": "Gemini 2.0 Flash 模型變體，針對成本效益和低延遲等目標進行了優化。"
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.70.11",
+  "version": "1.71.0",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",
@@ -35,12 +35,13 @@
     "build-sitemap": "tsx ./scripts/buildSitemapIndex/index.ts",
     "build:analyze": "ANALYZE=true next build",
     "build:docker": "DOCKER=true next build && npm run build-sitemap",
-    "db:generate": "drizzle-kit generate && npm run db:generate-client",
+    "db:generate": "drizzle-kit generate && npm run db:generate-client && npm run workflow:dbml",
     "db:generate-client": "tsx ./scripts/migrateClientDB/compile-migrations.ts",
     "db:migrate": "MIGRATION_DB=1 tsx ./scripts/migrateServerDB/index.ts",
     "db:push": "drizzle-kit push",
     "db:push-test": "NODE_ENV=test drizzle-kit push",
     "db:studio": "drizzle-kit studio",
+    "db:visualize": "dbdocs build docs/developer/database-schema.dbml --project lobe-chat",
     "db:z-pull": "drizzle-kit introspect",
     "dev": "next dev --turbopack -p 3010",
     "docs:i18n": "lobe-i18n md && npm run lint:md && npm run lint:mdx",
@@ -72,6 +73,7 @@
     "workflow:cdn": "tsx ./scripts/cdnWorkflow/index.ts",
     "workflow:changelog": "tsx ./scripts/changelogWorkflow/index.ts",
     "workflow:countCharters": "tsx scripts/countEnWord.ts",
+    "workflow:dbml": "tsx ./scripts/dbmlWorkflow/index.ts",
     "workflow:docs": "tsx ./scripts/docsWorkflow/index.ts",
     "workflow:i18n": "tsx ./scripts/i18nWorkflow/index.ts",
     "workflow:mdx": "tsx ./scripts/mdxWorkflow/index.ts",
@@ -289,8 +291,10 @@
     "commitlint": "^19.6.1",
     "consola": "^3.3.3",
     "crypto-js": "^4.2.0",
+    "dbdocs": "^0.14.3",
     "dotenv": "^16.4.7",
     "dpdm-fast": "^1.0.7",
+    "drizzle-dbml-generator": "^0.10.0",
     "drizzle-kit": "^0.30.1",
     "eslint": "^8.57.1",
     "eslint-plugin-mdx": "^3.1.5",

package/scripts/dbmlWorkflow/index.ts

@@ -0,0 +1,11 @@
+import { pgGenerate } from 'drizzle-dbml-generator';
+import { join } from 'node:path';
+
+import * as schema from '../../src/database/schemas';
+
+const out = join(__dirname,'../../docs/developer/database-schema.dbml');
+const relational = true;
+
+pgGenerate({ out, relational, schema });
+
+console.log('🏁 dbml generated successful!');

package/src/config/aiModels/google.ts

@@ -92,6 +92,23 @@ const googleChatModels: AIChatModelCard[] = [
     releasedAt: '2025-02-05',
     type: 'chat',
   },
+  {
+    abilities: {
+      vision: true,
+    },
+    contextWindowTokens: 32_768,
+    description: 'Gemini 2.0 Flash 模型变体，针对成本效益和低延迟等目标进行了优化。',
+    displayName: 'Gemini 2.0 Flash Exp',
+    enabled: true,
+    id: 'gemini-2.0-flash-exp',
+    maxOutput: 8192,
+    pricing: {
+      input: 0,
+      output: 0,
+    },
+    releasedAt: '2025-02-05',
+    type: 'chat',
+  },
   {
     abilities: {
       vision: true,

package/src/database/client/migrations.json

@@ -316,12 +316,12 @@
   },
   {
     "sql": [
-      "CREATE INDEX \"messages_topic_id_idx\" ON \"messages\" USING btree (\"topic_id\");",
-      "\nCREATE INDEX \"messages_parent_id_idx\" ON \"messages\" USING btree (\"parent_id\");",
-      "\nCREATE INDEX \"messages_quota_id_idx\" ON \"messages\" USING btree (\"quota_id\");"
+      "CREATE INDEX IF NOT EXISTS \"messages_topic_id_idx\" ON \"messages\" USING btree (\"topic_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"messages_parent_id_idx\" ON \"messages\" USING btree (\"parent_id\");",
+      "\nCREATE INDEX IF NOT EXISTS \"messages_quota_id_idx\" ON \"messages\" USING btree (\"quota_id\");\n"
     ],
     "bps": true,
     "folderMillis": 1741844738677,
-    "hash": "f2fc4639bb0b00ca4ed0861d8d3eeaef55810e0e0910f8f3f4277b0912bbe588"
+    "hash": "2a7a98be2e49361391444d6fabf3fb5db0bcb6a65e5540e9c3d426ceeb1f7f3a"
   }
 ]

package/src/database/server/models/message.ts

@@ -21,6 +21,7 @@ import {
   CreateMessageParams,
   MessageItem,
   ModelRankItem,
+  UpdateMessageParams,
 } from '@/types/message';
 import { merge } from '@/utils/merge';
 import { today } from '@/utils/time';
@@ -497,15 +498,25 @@ export class MessageModel {
   };
   // **************** Update *************** //
 
-  update = async (id: string, message: Partial<MessageItem>) => {
-    return this.db
-      .update(messages)
-      .set({
-        ...message,
-        // TODO: need a better way to handle this
-        role: message.role as any,
-      })
-      .where(and(eq(messages.id, id), eq(messages.userId, this.userId)));
+  update = async (id: string, { imageList, ...message }: Partial<UpdateMessageParams>) => {
+    return this.db.transaction(async (trx) => {
+      // 1. insert message files
+      if (imageList && imageList.length > 0) {
+        await trx
+          .insert(messagesFiles)
+          .values(imageList.map((file) => ({ fileId: file.id, messageId: id })));
+      }
+
+      return trx
+        .update(messages)
+        .set({
+          ...message,
+          // TODO: need a better way to handle this
+          // TODO: but I forget why 🤡
+          role: message.role as any,
+        })
+        .where(and(eq(messages.id, id), eq(messages.userId, this.userId)));
+    });
   };
 
   updatePluginState = async (id: string, state: Record<string, any>) => {

package/src/database/server/models/user.test.ts

@@ -0,0 +1,58 @@
+// @vitest-environment node
+import { TRPCError } from '@trpc/server';
+import { describe, expect, it, vi } from 'vitest';
+
+import { UserModel, UserNotFoundError } from '@/database/server/models/user';
+
+describe('UserNotFoundError', () => {
+  it('should extend TRPCError with correct code and message', () => {
+    const error = new UserNotFoundError();
+
+    expect(error).toBeInstanceOf(TRPCError);
+    expect(error.code).toBe('UNAUTHORIZED');
+    expect(error.message).toBe('user not found');
+  });
+});
+
+describe('UserModel', () => {
+  const mockDb = {
+    query: {
+      users: {
+        findFirst: vi.fn(),
+      },
+    },
+  };
+
+  const mockUserId = 'test-user-id';
+  const userModel = new UserModel(mockDb as any, mockUserId);
+
+  describe('getUserRegistrationDuration', () => {
+    it('should return default values when user not found', async () => {
+      mockDb.query.users.findFirst.mockResolvedValue(null);
+
+      const result = await userModel.getUserRegistrationDuration();
+
+      expect(result).toEqual({
+        createdAt: expect.any(String),
+        duration: 1,
+        updatedAt: expect.any(String),
+      });
+    });
+
+    it('should calculate duration correctly for existing user', async () => {
+      const createdAt = new Date('2024-01-01');
+      mockDb.query.users.findFirst.mockResolvedValue({
+        createdAt,
+      });
+
+      const result = await userModel.getUserRegistrationDuration();
+
+      expect(result).toEqual({
+        createdAt: '2024-01-01',
+        duration: expect.any(Number),
+        updatedAt: expect.any(String),
+      });
+      expect(result.duration).toBeGreaterThan(0);
+    });
+  });
+});

package/src/features/AlertBanner/CloudBanner.tsx

@@ -61,7 +61,7 @@ const CloudBanner = memo<{ mobile?: boolean }>(({ mobile }) => {
       <b>{t('alert.cloud.title', { name: LOBE_CHAT_CLOUD })}:</b>
       <span>
         {t(mobile ? 'alert.cloud.descOnMobile' : 'alert.cloud.desc', {
-          credit: new Intl.NumberFormat('en-US').format(450_000),
+          credit: new Intl.NumberFormat('en-US').format(500_000),
           name: LOBE_CHAT_CLOUD,
         })}
       </span>

package/src/features/Conversation/Messages/Assistant/index.tsx

@@ -2,6 +2,7 @@ import { ReactNode, memo } from 'react';
 import { Flexbox } from 'react-layout-kit';
 
 import { LOADING_FLAT } from '@/const/message';
+import ImageFileListViewer from '@/features/Conversation/Messages/User/ImageFileListViewer';
 import { useChatStore } from '@/store/chat';
 import { aiChatSelectors, chatSelectors } from '@/store/chat/selectors';
 import { ChatMessage } from '@/types/message';
@@ -17,7 +18,7 @@ export const AssistantMessage = memo<
   ChatMessage & {
     editableContent: ReactNode;
   }
->(({ id, tools, content, chunksList, search, ...props }) => {
+>(({ id, tools, content, chunksList, search, imageList, ...props }) => {
   const editing = useChatStore(chatSelectors.isMessageEditing(id));
   const generating = useChatStore(chatSelectors.isMessageGenerating(id));
 
@@ -28,6 +29,7 @@ export const AssistantMessage = memo<
   const isIntentUnderstanding = useChatStore(aiChatSelectors.isIntentUnderstanding(id));
 
   const showSearch = !!search && !!search.citations?.length;
+  const showImageItems = !!imageList && imageList.length > 0;
 
   // remove \n to avoid empty content
   // refs: https://github.com/lobehub/lobe-chat/pull/6153
@@ -64,6 +66,7 @@ export const AssistantMessage = memo<
           />
         )
       )}
+      {showImageItems && <ImageFileListViewer items={imageList} />}
       {tools && (
         <Flexbox gap={8}>
           {tools.map((toolCall, index) => (

package/src/features/Conversation/Messages/User/index.tsx

@@ -12,16 +12,16 @@ export const UserMessage = memo<
   ChatMessage & {
     editableContent: ReactNode;
   }
->(({ id, editableContent, content, ...res }) => {
+>(({ id, editableContent, content, imageList, fileList }) => {
   if (content === LOADING_FLAT) return <BubblesLoading />;
 
   return (
     <Flexbox gap={8} id={id}>
       {editableContent}
-      {res.imageList && res.imageList?.length > 0 && <ImageFileListViewer items={res.imageList} />}
-      {res.fileList && res.fileList?.length > 0 && (
+      {imageList && imageList?.length > 0 && <ImageFileListViewer items={imageList} />}
+      {fileList && fileList?.length > 0 && (
         <div style={{ marginTop: 8 }}>
-          <FileListViewer items={res.fileList} />
+          <FileListViewer items={fileList} />
         </div>
       )}
     </Flexbox>

package/src/libs/agent-runtime/google/index.ts

@@ -31,6 +31,9 @@ import { StreamingResponse } from '../utils/response';
 import { GoogleGenerativeAIStream, convertIterableToStream } from '../utils/streams';
 import { parseDataUri } from '../utils/uriParser';
 
+const modelsOffSafetySettings = new Set(['gemini-2.0-flash-exp']);
+const modelsWithModalities = new Set(['gemini-2.0-flash-exp']);
+
 export interface GoogleModelCard {
   displayName: string;
   inputTokenLimit: number;
@@ -50,8 +53,7 @@ enum HarmBlockThreshold {
 }
 
 function getThreshold(model: string): HarmBlockThreshold {
-  const useOFF = ['gemini-2.0-flash-exp'];
-  if (useOFF.includes(model)) {
+  if (modelsOffSafetySettings.has(model)) {
     return 'OFF' as HarmBlockThreshold; // https://discuss.ai.google.dev/t/59352
   }
   return HarmBlockThreshold.BLOCK_NONE;
@@ -94,6 +96,10 @@ export class LobeGoogleAI implements LobeRuntimeAI {
           {
             generationConfig: {
               maxOutputTokens: payload.max_tokens,
+              // @ts-expect-error - Google SDK 0.24.0 doesn't have this property for now with
+              response_modalities: modelsWithModalities.has(model)
+                ? ['Text', 'Image']
+                : undefined,
               temperature: payload.temperature,
               topP: payload.top_p,
             },

package/src/libs/agent-runtime/utils/streams/google-ai.test.ts

@@ -94,4 +94,103 @@ describe('GoogleGenerativeAIStream', () => {
 
     expect(chunks).toEqual([]);
   });
+
+  it('should handle image', async () => {
+    vi.spyOn(uuidModule, 'nanoid').mockReturnValueOnce('1');
+
+    const data = {
+      candidates: [
+        {
+          content: {
+            parts: [{ inlineData: { mimeType: 'image/png', data: 'iVBORw0KGgoAA' } }],
+            role: 'model',
+          },
+          index: 0,
+        },
+      ],
+      usageMetadata: {
+        promptTokenCount: 6,
+        totalTokenCount: 6,
+        promptTokensDetails: [{ modality: 'TEXT', tokenCount: 6 }],
+      },
+      modelVersion: 'gemini-2.0-flash-exp',
+    };
+    const mockGenerateContentResponse = (text: string, functionCalls?: any[]) =>
+      ({
+        text: () => text,
+        functionCall: () => functionCalls?.[0],
+        functionCalls: () => functionCalls,
+      }) as EnhancedGenerateContentResponse;
+
+    const mockGoogleStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue(data);
+
+        controller.close();
+      },
+    });
+
+    const protocolStream = GoogleGenerativeAIStream(mockGoogleStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([
+      // image
+      'id: chat_1\n',
+      'event: base64_image\n',
+      `data: "data:image/png;base64,iVBORw0KGgoAA"\n\n`,
+    ]);
+  });
+
+  it('should handle token count', async () => {
+    vi.spyOn(uuidModule, 'nanoid').mockReturnValueOnce('1');
+
+    const data = {
+      candidates: [{ content: { role: 'model' }, finishReason: 'STOP', index: 0 }],
+      usageMetadata: {
+        promptTokenCount: 266,
+        totalTokenCount: 266,
+        promptTokensDetails: [
+          { modality: 'TEXT', tokenCount: 8 },
+          { modality: 'IMAGE', tokenCount: 258 },
+        ],
+      },
+      modelVersion: 'gemini-2.0-flash-exp',
+    };
+
+    const mockGoogleStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue(data);
+
+        controller.close();
+      },
+    });
+
+    const protocolStream = GoogleGenerativeAIStream(mockGoogleStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([
+      // stop
+      'id: chat_1\n',
+      'event: stop\n',
+      `data: "STOP"\n\n`,
+      // usage
+      'id: chat_1\n',
+      'event: usage\n',
+      `data: {"inputImageTokens":258,"inputTextTokens":8,"totalInputTokens":266,"totalTokens":266}\n\n`,
+    ]);
+  });
 });

package/src/libs/agent-runtime/utils/streams/google-ai.ts

@@ -1,5 +1,6 @@
 import { EnhancedGenerateContentResponse } from '@google/generative-ai';
 
+import { ModelTokensUsage } from '@/types/message';
 import { GroundingSearch } from '@/types/search';
 import { nanoid } from '@/utils/uuid';
 
@@ -18,7 +19,7 @@ const transformGoogleGenerativeAIStream = (
   context: StreamContext,
 ): StreamProtocolChunk | StreamProtocolChunk[] => {
   // maybe need another structure to add support for multiple choices
-  const functionCalls = chunk.functionCalls();
+  const functionCalls = chunk.functionCalls?.();
 
   if (functionCalls) {
     return {
@@ -37,30 +38,75 @@ const transformGoogleGenerativeAIStream = (
       type: 'tool_calls',
     };
   }
-  const text = chunk.text();
 
-  if (chunk.candidates && chunk.candidates[0].groundingMetadata) {
-    const { webSearchQueries, groundingSupports, groundingChunks } =
-      chunk.candidates[0].groundingMetadata;
-    console.log({ groundingChunks, groundingSupports, webSearchQueries });
+  const text = chunk.text?.();
 
-    return [
-      { data: text, id: context.id, type: 'text' },
-      {
-        data: {
-          citations: groundingChunks?.map((chunk) => ({
-            // google 返回的 uri 是经过 google 自己处理过的 url，因此无法展现真实的 favicon
-            // 需要使用 title 作为替换
-            favicon: chunk.web?.title,
-            title: chunk.web?.title,
-            url: chunk.web?.uri,
-          })),
-          searchQueries: webSearchQueries,
-        } as GroundingSearch,
-        id: context.id,
-        type: 'grounding',
-      },
-    ];
+  if (chunk.candidates) {
+    const candidate = chunk.candidates[0];
+
+    // return the grounding
+    if (candidate.groundingMetadata) {
+      const { webSearchQueries, groundingChunks } = candidate.groundingMetadata;
+
+      return [
+        { data: text, id: context.id, type: 'text' },
+        {
+          data: {
+            citations: groundingChunks?.map((chunk) => ({
+              // google 返回的 uri 是经过 google 自己处理过的 url，因此无法展现真实的 favicon
+              // 需要使用 title 作为替换
+              favicon: chunk.web?.title,
+              title: chunk.web?.title,
+              url: chunk.web?.uri,
+            })),
+            searchQueries: webSearchQueries,
+          } as GroundingSearch,
+          id: context.id,
+          type: 'grounding',
+        },
+      ];
+    }
+
+    if (candidate.finishReason) {
+      if (chunk.usageMetadata) {
+        const usage = chunk.usageMetadata;
+        return [
+          { data: candidate.finishReason, id: context?.id, type: 'stop' },
+          {
+            data: {
+              // TODO: Google SDK 0.24.0 don't have promptTokensDetails types
+              inputImageTokens: (usage as any).promptTokensDetails?.find(
+                (i: any) => i.modality === 'IMAGE',
+              )?.tokenCount,
+              inputTextTokens: (usage as any).promptTokensDetails?.find(
+                (i: any) => i.modality === 'TEXT',
+              )?.tokenCount,
+              totalInputTokens: usage.promptTokenCount,
+              totalOutputTokens: usage.candidatesTokenCount,
+              totalTokens: usage.totalTokenCount,
+            } as ModelTokensUsage,
+            id: context?.id,
+            type: 'usage',
+          },
+        ];
+      }
+      return { data: candidate.finishReason, id: context?.id, type: 'stop' };
+    }
+
+    if (!!text?.trim()) return { data: text, id: context?.id, type: 'text' };
+
+    // streaming the image
+    if (Array.isArray(candidate.content.parts) && candidate.content.parts.length > 0) {
+      const part = candidate.content.parts[0];
+
+      if (part && part.inlineData && part.inlineData.data && part.inlineData.mimeType) {
+        return {
+          data: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`,
+          id: context.id,
+          type: 'base64_image',
+        };
+      }
+    }
   }
 
   return {

package/src/libs/agent-runtime/utils/streams/protocol.ts

@@ -32,6 +32,8 @@ export interface StreamProtocolChunk {
   id?: string;
   type: // pure text
   | 'text'
+    // base64 format image
+    | 'base64_image'
     // Tools use
     | 'tool_calls'
     // Model Thinking