@lobehub/chat 1.70.11 → 1.71.0

package/src/services/chat.ts

@@ -438,6 +438,8 @@ class ChatService {
         provider: params.provider!,
       });
 
+      // remove plugins
+      delete params.plugins;
       await this.getChatCompletion(
         { ...params, messages: oaiMessages, tools },
         {
@@ -474,7 +476,7 @@ class ChatService {
     // handle content type for vision model
     // for the models with visual ability, add image url to content
     // refs: https://platform.openai.com/docs/guides/vision/quick-start
-    const getContent = (m: ChatMessage) => {
+    const getUserContent = (m: ChatMessage) => {
       // only if message doesn't have images and files, then return the plain content
       if ((!m.imageList || m.imageList.length === 0) && (!m.fileList || m.fileList.length === 0))
         return m.content;
@@ -490,27 +492,43 @@ class ChatService {
       ] as UserMessageContentPart[];
     };
 
+    const getAssistantContent = (m: ChatMessage) => {
+      // signature is a signal of anthropic thinking mode
+      const shouldIncludeThinking = m.reasoning && !!m.reasoning?.signature;
+
+      if (shouldIncludeThinking) {
+        return [
+          {
+            signature: m.reasoning!.signature,
+            thinking: m.reasoning!.content,
+            type: 'thinking',
+          },
+          { text: m.content, type: 'text' },
+        ] as UserMessageContentPart[];
+      }
+      // only if message doesn't have images and files, then return the plain content
+
+      if (m.imageList && m.imageList.length > 0) {
+        return [
+          !!m.content ? { text: m.content, type: 'text' } : undefined,
+          ...m.imageList.map(
+            (i) => ({ image_url: { detail: 'auto', url: i.url }, type: 'image_url' }) as const,
+          ),
+        ].filter(Boolean) as UserMessageContentPart[];
+      }
+
+      return m.content;
+    };
+
     let postMessages = messages.map((m): OpenAIChatMessage => {
       const supportTools = isCanUseFC(model, provider);
       switch (m.role) {
         case 'user': {
-          return { content: getContent(m), role: m.role };
+          return { content: getUserContent(m), role: m.role };
         }
 
         case 'assistant': {
-          // signature is a signal of anthropic thinking mode
-          const shouldIncludeThinking = m.reasoning && !!m.reasoning?.signature;
-
-          const content = shouldIncludeThinking
-            ? [
-                {
-                  signature: m.reasoning!.signature,
-                  thinking: m.reasoning!.content,
-                  type: 'thinking',
-                } as any,
-                { text: m.content, type: 'text' },
-              ]
-            : m.content;
+          const content = getAssistantContent(m);
 
           if (!supportTools) {
             return { content, role: m.role };

package/src/services/file/client.ts

@@ -11,6 +11,8 @@ export class ClientService extends BaseClientService implements IFileService {
   }
 
   createFile: IFileService['createFile'] = async (file) => {
+    const { isExist } = await this.fileModel.checkHash(file.hash!);
+
     // save to local storage
     // we may want to save to a remote server later
     const res = await this.fileModel.create(
@@ -23,7 +25,7 @@ export class ClientService extends BaseClientService implements IFileService {
         size: file.size,
         url: file.url!,
       },
-      true,
+      !isExist,
     );
 
     // get file to base64 url

package/src/services/message/server.ts

@@ -64,8 +64,8 @@ export class ServerService implements IMessageService {
     return lambdaClient.message.updateMessagePlugin.mutate({ id, value: { arguments: args } });
   };
 
-  updateMessage: IMessageService['updateMessage'] = async (id, message) => {
-    return lambdaClient.message.update.mutate({ id, value: message });
+  updateMessage: IMessageService['updateMessage'] = async (id, value) => {
+    return lambdaClient.message.update.mutate({ id, value });
   };
 
   updateMessageTranslate: IMessageService['updateMessageTranslate'] = async (id, translate) => {

package/src/services/message/type.ts

@@ -8,7 +8,7 @@ import {
   ChatTranslate,
   CreateMessageParams,
   MessageItem,
-  ModelRankItem,
+  ModelRankItem, UpdateMessageParams,
 } from '@/types/message';
 
 /* eslint-disable typescript-sort-keys/interface */
@@ -33,7 +33,7 @@ export interface IMessageService {
   rankModels(): Promise<ModelRankItem[]>;
   getHeatmaps(): Promise<HeatmapsProps['data']>;
   updateMessageError(id: string, error: ChatMessageError): Promise<any>;
-  updateMessage(id: string, message: Partial<MessageItem>): Promise<any>;
+  updateMessage(id: string, message: Partial<UpdateMessageParams>): Promise<any>;
   updateMessageTTS(id: string, tts: Partial<ChatTTS> | false): Promise<any>;
   updateMessageTranslate(id: string, translate: Partial<ChatTranslate> | false): Promise<any>;
   updateMessagePluginState(id: string, value: Record<string, any>): Promise<any>;

package/src/services/upload.ts

@@ -1,14 +1,95 @@
+import dayjs from 'dayjs';
+import { sha256 } from 'js-sha256';
+
 import { fileEnv } from '@/config/file';
+import { isServerMode } from '@/const/version';
+import { parseDataUri } from '@/libs/agent-runtime/utils/uriParser';
 import { edgeClient } from '@/libs/trpc/client';
 import { API_ENDPOINTS } from '@/services/_url';
 import { clientS3Storage } from '@/services/file/ClientS3';
-import { FileMetadata } from '@/types/files';
+import { FileMetadata, UploadBase64ToS3Result } from '@/types/files';
 import { FileUploadState, FileUploadStatus } from '@/types/files/upload';
 import { uuid } from '@/utils/uuid';
 
 export const UPLOAD_NETWORK_ERROR = 'NetWorkError';
 
+interface UploadFileToS3Options {
+  directory?: string;
+  filename?: string;
+  onProgress?: (status: FileUploadStatus, state: FileUploadState) => void;
+}
+
 class UploadService {
+  /**
+   * uniform upload method for both server and client
+   */
+  uploadFileToS3 = async (
+    file: File,
+    options: UploadFileToS3Options = {},
+  ): Promise<FileMetadata> => {
+    const { directory, onProgress } = options;
+
+    if (isServerMode) {
+      return this.uploadWithProgress(file, { directory, onProgress });
+    } else {
+      const fileArrayBuffer = await file.arrayBuffer();
+
+      // 1. check file hash
+      const hash = sha256(fileArrayBuffer);
+
+      return this.uploadToClientS3(hash, file);
+    }
+  };
+
+  uploadBase64ToS3 = async (
+    base64Data: string,
+    options: UploadFileToS3Options = {},
+  ): Promise<UploadBase64ToS3Result> => {
+    // 解析 base64 数据
+    const { base64, mimeType, type } = parseDataUri(base64Data);
+
+    if (!base64 || !mimeType || type !== 'base64') {
+      throw new Error('Invalid base64 data for image');
+    }
+
+    // 将 base64 转换为 Blob
+    const byteCharacters = atob(base64);
+    const byteArrays = [];
+
+    // 分块处理以避免内存问题
+    for (let offset = 0; offset < byteCharacters.length; offset += 1024) {
+      const slice = byteCharacters.slice(offset, offset + 1024);
+
+      const byteNumbers: number[] = Array.from({ length: slice.length });
+      for (let i = 0; i < slice.length; i++) {
+        byteNumbers[i] = slice.charCodeAt(i);
+      }
+
+      const byteArray = new Uint8Array(byteNumbers);
+      byteArrays.push(byteArray);
+    }
+
+    const blob = new Blob(byteArrays, { type: mimeType });
+
+    // 确定文件扩展名
+    const fileExtension = mimeType.split('/')[1] || 'png';
+    const fileName = `${options.filename || `image_${dayjs().format('YYYY-MM-DD-hh-mm-ss')}`}.${fileExtension}`;
+
+    // 创建文件对象
+    const file = new File([blob], fileName, { type: mimeType });
+
+    // 使用统一的上传方法
+    const metadata = await this.uploadFileToS3(file, options);
+    const hash = sha256(await file.arrayBuffer());
+
+    return {
+      fileType: mimeType,
+      hash,
+      metadata,
+      size: file.size,
+    };
+  };
+
   uploadWithProgress = async (
     file: File,
     {

package/src/store/chat/slices/aiChat/actions/generateAIChat.ts

@@ -18,9 +18,11 @@ import { getAiInfraStoreState } from '@/store/aiInfra/store';
 import { chatHelpers } from '@/store/chat/helpers';
 import { ChatStore } from '@/store/chat/store';
 import { messageMapKey } from '@/store/chat/utils/messageMapKey';
+import { getFileStoreState } from '@/store/file/store';
 import { useSessionStore } from '@/store/session';
 import { WebBrowsingManifest } from '@/tools/web-browsing';
 import { ChatMessage, CreateMessageParams, SendMessageParams } from '@/types/message';
+import { ChatImageItem } from '@/types/message/image';
 import { MessageSemanticSearchChunk } from '@/types/rag';
 import { setNamespace } from '@/utils/storeDebug';
 
@@ -533,6 +535,8 @@ export const generateAIChat: StateCreator<
     let thinking = '';
     let thinkingStartAt: number;
     let duration: number;
+    // to upload image
+    const uploadTasks: Map<string, Promise<{ id?: string; url?: string }>> = new Map();
 
     const historySummary = topicSelectors.currentActiveTopicSummary(get());
     await chatService.createAssistantMessageStream({
@@ -569,6 +573,21 @@ export const generateAIChat: StateCreator<
           });
         }
 
+        // 等待所有图片上传完成
+        let finalImages: ChatImageItem[] = [];
+
+        if (uploadTasks.size > 0) {
+          try {
+            // 等待所有上传任务完成
+            const uploadResults = await Promise.all(uploadTasks.values());
+
+            // 使用上传后的 S3 URL 替换原始图像数据
+            finalImages = uploadResults.filter((i) => !!i.url) as ChatImageItem[];
+          } catch (error) {
+            console.error('Error waiting for image uploads:', error);
+          }
+        }
+
         if (toolCalls && toolCalls.length > 0) {
           internal_toggleToolCallingStreaming(messageId, undefined);
         }
@@ -579,6 +598,7 @@ export const generateAIChat: StateCreator<
           reasoning: !!reasoning ? { ...reasoning, duration } : undefined,
           search: !!grounding?.citations ? grounding : undefined,
           metadata: usage,
+          imageList: finalImages.length > 0 ? finalImages : undefined,
         });
       },
       onMessageHandle: async (chunk) => {
@@ -605,6 +625,29 @@ export const generateAIChat: StateCreator<
             break;
           }
 
+          case 'base64_image': {
+            internal_dispatchMessage({
+              id: messageId,
+              type: 'updateMessage',
+              value: {
+                imageList: chunk.images.map((i) => ({ id: i.id, url: i.data, alt: i.id })),
+              },
+            });
+            const image = chunk.image;
+
+            const task = getFileStoreState()
+              .uploadBase64FileWithProgress(image.data)
+              .then((value) => ({
+                id: value?.id,
+                url: value?.url,
+                alt: value?.filename || value?.id,
+              }));
+
+            uploadTasks.set(image.id, task);
+
+            break;
+          }
+
           case 'text': {
             output += chunk.text;
 
@@ -658,10 +701,7 @@ export const generateAIChat: StateCreator<
 
     internal_toggleChatLoading(false, messageId, n('generateMessage(end)') as string);
 
-    return {
-      isFunctionCall,
-      traceId: msgTraceId,
-    };
+    return { isFunctionCall, traceId: msgTraceId };
   },
 
   internal_resendMessage: async (

package/src/store/chat/slices/message/action.ts

@@ -21,6 +21,7 @@ import {
   MessageToolCall,
   ModelReasoning,
 } from '@/types/message';
+import { ChatImageItem } from '@/types/message/image';
 import { GroundingSearch } from '@/types/search';
 import { TraceEventPayloads } from '@/types/trace';
 import { setNamespace } from '@/utils/storeDebug';
@@ -81,6 +82,7 @@ export interface ChatMessageAction {
       reasoning?: ModelReasoning;
       search?: GroundingSearch;
       metadata?: MessageMetadata;
+      imageList?: ChatImageItem[];
       model?: string;
       provider?: string;
     },
@@ -319,6 +321,7 @@ export const chatMessage: StateCreator<
       metadata: extra?.metadata,
       model: extra?.model,
       provider: extra?.provider,
+      imageList: extra?.imageList,
     });
     await refreshMessages();
   },

package/src/store/file/slices/upload/action.ts

@@ -11,21 +11,23 @@ import { FileMetadata, UploadFileItem } from '@/types/files';
 
 import { FileStore } from '../../store';
 
+type OnStatusUpdate = (
+  data:
+    | {
+        id: string;
+        type: 'updateFile';
+        value: Partial<UploadFileItem>;
+      }
+    | {
+        id: string;
+        type: 'removeFile';
+      },
+) => void;
+
 interface UploadWithProgressParams {
   file: File;
   knowledgeBaseId?: string;
-  onStatusUpdate?: (
-    data:
-      | {
-          id: string;
-          type: 'updateFile';
-          value: Partial<UploadFileItem>;
-        }
-      | {
-          id: string;
-          type: 'removeFile';
-        },
-  ) => void;
+  onStatusUpdate?: OnStatusUpdate;
   /**
    * Optional flag to indicate whether to skip the file type check.
    * When set to `true`, any file type checks will be bypassed.
@@ -35,11 +37,19 @@ interface UploadWithProgressParams {
 }
 
 interface UploadWithProgressResult {
+  filename?: string;
   id: string;
   url: string;
 }
 
 export interface FileUploadAction {
+  uploadBase64FileWithProgress: (
+    base64: string,
+    params?: {
+      onStatusUpdate?: OnStatusUpdate;
+    },
+  ) => Promise<UploadWithProgressResult | undefined>;
+
   uploadWithProgress: (
     params: UploadWithProgressParams,
   ) => Promise<UploadWithProgressResult | undefined>;
@@ -51,6 +61,19 @@ export const createFileUploadSlice: StateCreator<
   [],
   FileUploadAction
 > = () => ({
+  uploadBase64FileWithProgress: async (base64) => {
+    const { metadata, fileType, size, hash } = await uploadService.uploadBase64ToS3(base64);
+
+    const res = await fileService.createFile({
+      fileType,
+      hash,
+      metadata,
+      name: metadata.filename,
+      size: size,
+      url: metadata.path,
+    });
+    return { ...res, filename: metadata.filename };
+  },
   uploadWithProgress: async ({ file, onStatusUpdate, knowledgeBaseId, skipCheckFileType }) => {
     const fileArrayBuffer = await file.arrayBuffer();
 
@@ -135,6 +158,6 @@ export const createFileUploadSlice: StateCreator<
       },
     });
 
-    return data;
+    return { ...data, filename: file.name };
   },
 });

package/src/store/file/store.ts

@@ -32,3 +32,5 @@ const createStore: StateCreator<FileStore, [['zustand/devtools', never]]> = (...
 const devtools = createDevtools('file');
 
 export const useFileStore = createWithEqualityFn<FileStore>()(devtools(createStore), shallow);
+
+export const getFileStoreState = () => useFileStore.getState();

package/src/types/files/upload.ts

@@ -95,3 +95,10 @@ export interface CheckFileHashResult {
   size?: number;
   url?: string;
 }
+
+export interface UploadBase64ToS3Result {
+  fileType: string;
+  hash: string;
+  metadata: FileMetadata;
+  size: number;
+}

package/src/types/message/base.ts

@@ -1,3 +1,6 @@
+import { ChatMessageError } from '@/types/message/chat';
+import { ChatImageItem } from '@/types/message/image';
+import { ChatToolPayload, MessageToolCall } from '@/types/message/tools';
 import { GroundingSearch } from '@/types/search';
 
 export interface CitationItem {
@@ -22,12 +25,17 @@ export interface ModelTokensUsage {
    * currently only pplx has citation_tokens
    */
   inputCitationTokens?: number;
+  /**
+   * user prompt image
+   */
+  inputImageTokens?: number;
   /**
    * user prompt input
    */
   inputTextTokens?: number;
   inputWriteCacheTokens?: number;
   outputAudioTokens?: number;
+  outputImageTokens?: number;
   outputReasoningTokens?: number;
   outputTextTokens?: number;
   rejectedPredictionTokens?: number;
@@ -61,7 +69,6 @@ export interface MessageItem {
   search: GroundingSearch | null;
   sessionId: string | null;
   threadId: string | null;
-  // jsonb type
   tools: any | null;
   topicId: string | null;
   // jsonb type
@@ -96,3 +103,17 @@ export interface NewMessage {
   updatedAt?: Date;
   userId: string; // optional because it's generated
 }
+
+export interface UpdateMessageParams {
+  content?: string;
+  error?: ChatMessageError | null;
+  imageList?: ChatImageItem[];
+  metadata?: MessageMetadata;
+  model?: string;
+  provider?: string;
+  reasoning?: ModelReasoning;
+  role?: string;
+  search?: GroundingSearch;
+  toolCalls?: MessageToolCall[];
+  tools?: ChatToolPayload[] | null;
+}

package/src/types/message/chat.ts

@@ -7,6 +7,7 @@ import { MessageSemanticSearchChunk } from '@/types/rag';
 import { GroundingSearch } from '@/types/search';
 
 import { MessageMetadata, MessageRoleType, ModelReasoning } from './base';
+import { ChatImageItem } from './image';
 import { ChatPluginPayload, ChatToolPayload } from './tools';
 import { Translate } from './translate';
 
@@ -37,12 +38,6 @@ export interface ChatFileItem {
   url: string;
 }
 
-export interface ChatImageItem {
-  alt: string;
-  id: string;
-  url: string;
-}
-
 export interface ChatFileChunk {
   fileId: string;
   fileType: string;

package/src/types/message/image.ts

@@ -0,0 +1,11 @@
+export interface ChatImageItem {
+  alt: string;
+  id: string;
+  url: string;
+}
+
+export interface ChatImageChunk {
+  data: string;
+  id: string;
+  isBase64?: boolean;
+}

package/src/types/message/index.ts

@@ -2,6 +2,7 @@ import { UploadFileItem } from '@/types/files';
 
 export * from './base';
 export * from './chat';
+export * from './image';
 export * from './tools';
 
 export interface SendMessageParams {

package/src/utils/fetch/fetchSSE.ts

@@ -12,7 +12,9 @@ import {
   ModelReasoning,
   ModelTokensUsage,
 } from '@/types/message';
+import { ChatImageChunk } from '@/types/message/image';
 import { GroundingSearch } from '@/types/search';
+import { nanoid } from '@/utils/uuid';
 
 import { fetchEventSource } from './fetchEventSource';
 import { getMessageError } from './parseError';
@@ -24,6 +26,7 @@ export type OnFinishHandler = (
   text: string,
   context: {
     grounding?: GroundingSearch;
+    images?: ChatImageChunk[];
     observationId?: string | null;
     reasoning?: ModelReasoning;
     toolCalls?: MessageToolCall[];
@@ -43,6 +46,13 @@ export interface MessageTextChunk {
   type: 'text';
 }
 
+export interface MessageBase64ImageChunk {
+  id: string;
+  image: ChatImageChunk;
+  images: ChatImageChunk[];
+  type: 'base64_image';
+}
+
 export interface MessageReasoningChunk {
   signature?: string;
   text?: string;
@@ -71,7 +81,8 @@ export interface FetchSSEOptions {
       | MessageToolCallsChunk
       | MessageReasoningChunk
       | MessageGroundingChunk
-      | MessageUsageChunk,
+      | MessageUsageChunk
+      | MessageBase64ImageChunk,
   ) => void;
   smoothing?: SmoothingParams | boolean;
 }
@@ -330,6 +341,8 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio
 
   let grounding: GroundingSearch | undefined = undefined;
   let usage: ModelTokensUsage | undefined = undefined;
+  let images: ChatImageChunk[] = [];
+
   await fetchEventSource(url, {
     body: options.body,
     fetch: options?.fetcher,
@@ -389,6 +402,15 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio
           break;
         }
 
+        case 'base64_image': {
+          const id = 'tmp_img_' + nanoid();
+          const item = { data, id, isBase64: true };
+          images.push(item);
+
+          options.onMessageHandle?.({ id, image: item, images, type: 'base64_image' });
+          break;
+        }
+
         case 'text': {
           // skip empty text
           if (!data) break;
@@ -492,6 +514,7 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio
 
       await options?.onFinish?.(output, {
         grounding,
+        images: images.length > 0 ? images : undefined,
         observationId,
         reasoning: !!thinking ? { content: thinking, signature: thinkingSignature } : undefined,
         toolCalls,