@lobehub/chat 1.70.10 → 1.71.0

package/src/store/chat/slices/aiChat/actions/generateAIChat.ts

@@ -18,9 +18,11 @@ import { getAiInfraStoreState } from '@/store/aiInfra/store';
 import { chatHelpers } from '@/store/chat/helpers';
 import { ChatStore } from '@/store/chat/store';
 import { messageMapKey } from '@/store/chat/utils/messageMapKey';
+import { getFileStoreState } from '@/store/file/store';
 import { useSessionStore } from '@/store/session';
 import { WebBrowsingManifest } from '@/tools/web-browsing';
 import { ChatMessage, CreateMessageParams, SendMessageParams } from '@/types/message';
+import { ChatImageItem } from '@/types/message/image';
 import { MessageSemanticSearchChunk } from '@/types/rag';
 import { setNamespace } from '@/utils/storeDebug';
 
@@ -533,6 +535,8 @@ export const generateAIChat: StateCreator<
     let thinking = '';
     let thinkingStartAt: number;
     let duration: number;
+    // to upload image
+    const uploadTasks: Map<string, Promise<{ id?: string; url?: string }>> = new Map();
 
     const historySummary = topicSelectors.currentActiveTopicSummary(get());
     await chatService.createAssistantMessageStream({
@@ -569,6 +573,21 @@ export const generateAIChat: StateCreator<
           });
         }
 
+        // 等待所有图片上传完成
+        let finalImages: ChatImageItem[] = [];
+
+        if (uploadTasks.size > 0) {
+          try {
+            // 等待所有上传任务完成
+            const uploadResults = await Promise.all(uploadTasks.values());
+
+            // 使用上传后的 S3 URL 替换原始图像数据
+            finalImages = uploadResults.filter((i) => !!i.url) as ChatImageItem[];
+          } catch (error) {
+            console.error('Error waiting for image uploads:', error);
+          }
+        }
+
         if (toolCalls && toolCalls.length > 0) {
           internal_toggleToolCallingStreaming(messageId, undefined);
         }
@@ -579,6 +598,7 @@ export const generateAIChat: StateCreator<
           reasoning: !!reasoning ? { ...reasoning, duration } : undefined,
           search: !!grounding?.citations ? grounding : undefined,
           metadata: usage,
+          imageList: finalImages.length > 0 ? finalImages : undefined,
         });
       },
       onMessageHandle: async (chunk) => {
@@ -605,6 +625,29 @@ export const generateAIChat: StateCreator<
             break;
           }
 
+          case 'base64_image': {
+            internal_dispatchMessage({
+              id: messageId,
+              type: 'updateMessage',
+              value: {
+                imageList: chunk.images.map((i) => ({ id: i.id, url: i.data, alt: i.id })),
+              },
+            });
+            const image = chunk.image;
+
+            const task = getFileStoreState()
+              .uploadBase64FileWithProgress(image.data)
+              .then((value) => ({
+                id: value?.id,
+                url: value?.url,
+                alt: value?.filename || value?.id,
+              }));
+
+            uploadTasks.set(image.id, task);
+
+            break;
+          }
+
           case 'text': {
             output += chunk.text;
 
@@ -658,10 +701,7 @@ export const generateAIChat: StateCreator<
 
     internal_toggleChatLoading(false, messageId, n('generateMessage(end)') as string);
 
-    return {
-      isFunctionCall,
-      traceId: msgTraceId,
-    };
+    return { isFunctionCall, traceId: msgTraceId };
   },
 
   internal_resendMessage: async (

package/src/store/chat/slices/message/action.ts

@@ -21,6 +21,7 @@ import {
   MessageToolCall,
   ModelReasoning,
 } from '@/types/message';
+import { ChatImageItem } from '@/types/message/image';
 import { GroundingSearch } from '@/types/search';
 import { TraceEventPayloads } from '@/types/trace';
 import { setNamespace } from '@/utils/storeDebug';
@@ -81,6 +82,7 @@ export interface ChatMessageAction {
       reasoning?: ModelReasoning;
       search?: GroundingSearch;
       metadata?: MessageMetadata;
+      imageList?: ChatImageItem[];
       model?: string;
       provider?: string;
     },
@@ -319,6 +321,7 @@ export const chatMessage: StateCreator<
       metadata: extra?.metadata,
       model: extra?.model,
       provider: extra?.provider,
+      imageList: extra?.imageList,
     });
     await refreshMessages();
   },

package/src/store/file/slices/upload/action.ts

@@ -11,21 +11,23 @@ import { FileMetadata, UploadFileItem } from '@/types/files';
 
 import { FileStore } from '../../store';
 
+type OnStatusUpdate = (
+  data:
+    | {
+        id: string;
+        type: 'updateFile';
+        value: Partial<UploadFileItem>;
+      }
+    | {
+        id: string;
+        type: 'removeFile';
+      },
+) => void;
+
 interface UploadWithProgressParams {
   file: File;
   knowledgeBaseId?: string;
-  onStatusUpdate?: (
-    data:
-      | {
-          id: string;
-          type: 'updateFile';
-          value: Partial<UploadFileItem>;
-        }
-      | {
-          id: string;
-          type: 'removeFile';
-        },
-  ) => void;
+  onStatusUpdate?: OnStatusUpdate;
   /**
    * Optional flag to indicate whether to skip the file type check.
    * When set to `true`, any file type checks will be bypassed.
@@ -35,11 +37,19 @@ interface UploadWithProgressParams {
 }
 
 interface UploadWithProgressResult {
+  filename?: string;
   id: string;
   url: string;
 }
 
 export interface FileUploadAction {
+  uploadBase64FileWithProgress: (
+    base64: string,
+    params?: {
+      onStatusUpdate?: OnStatusUpdate;
+    },
+  ) => Promise<UploadWithProgressResult | undefined>;
+
   uploadWithProgress: (
     params: UploadWithProgressParams,
   ) => Promise<UploadWithProgressResult | undefined>;
@@ -51,6 +61,19 @@ export const createFileUploadSlice: StateCreator<
   [],
   FileUploadAction
 > = () => ({
+  uploadBase64FileWithProgress: async (base64) => {
+    const { metadata, fileType, size, hash } = await uploadService.uploadBase64ToS3(base64);
+
+    const res = await fileService.createFile({
+      fileType,
+      hash,
+      metadata,
+      name: metadata.filename,
+      size: size,
+      url: metadata.path,
+    });
+    return { ...res, filename: metadata.filename };
+  },
   uploadWithProgress: async ({ file, onStatusUpdate, knowledgeBaseId, skipCheckFileType }) => {
     const fileArrayBuffer = await file.arrayBuffer();
 
@@ -135,6 +158,6 @@ export const createFileUploadSlice: StateCreator<
       },
     });
 
-    return data;
+    return { ...data, filename: file.name };
   },
 });

package/src/store/file/store.ts

@@ -32,3 +32,5 @@ const createStore: StateCreator<FileStore, [['zustand/devtools', never]]> = (...
 const devtools = createDevtools('file');
 
 export const useFileStore = createWithEqualityFn<FileStore>()(devtools(createStore), shallow);
+
+export const getFileStoreState = () => useFileStore.getState();

package/src/tools/web-browsing/Render/PageContent/index.tsx

@@ -17,8 +17,8 @@ const PagesContent = memo<PagesContentProps>(({ results, messageId, urls }) => {
   if (!results || results.length === 0) {
     return (
       <Flexbox gap={12} horizontal>
-        {urls.map((url) => (
-          <Loading key={url} url={url} />
+        {urls.map((url, index) => (
+          <Loading key={`${url}_${index}`} url={url} />
         ))}
       </Flexbox>
     );

package/src/tools/web-browsing/Render/Search/SearchResult/SearchResultItem.tsx

@@ -50,7 +50,7 @@ const SearchResultItem = memo<SearchResult>(({ url, title }) => {
   const host = urlObj.hostname;
   return (
     <Link href={url} target={'_blank'}>
-      <Flexbox className={styles.container} gap={2} justify={'space-between'} key={url}>
+      <Flexbox className={styles.container} gap={2} justify={'space-between'}>
         <div className={styles.title}>{title}</div>
         <Flexbox align={'center'} gap={4} horizontal>
           <WebFavicon size={14} title={title} url={url} />

package/src/types/files/upload.ts

@@ -95,3 +95,10 @@ export interface CheckFileHashResult {
   size?: number;
   url?: string;
 }
+
+export interface UploadBase64ToS3Result {
+  fileType: string;
+  hash: string;
+  metadata: FileMetadata;
+  size: number;
+}

package/src/types/message/base.ts

@@ -1,3 +1,6 @@
+import { ChatMessageError } from '@/types/message/chat';
+import { ChatImageItem } from '@/types/message/image';
+import { ChatToolPayload, MessageToolCall } from '@/types/message/tools';
 import { GroundingSearch } from '@/types/search';
 
 export interface CitationItem {
@@ -22,12 +25,17 @@ export interface ModelTokensUsage {
    * currently only pplx has citation_tokens
    */
   inputCitationTokens?: number;
+  /**
+   * user prompt image
+   */
+  inputImageTokens?: number;
   /**
    * user prompt input
    */
   inputTextTokens?: number;
   inputWriteCacheTokens?: number;
   outputAudioTokens?: number;
+  outputImageTokens?: number;
   outputReasoningTokens?: number;
   outputTextTokens?: number;
   rejectedPredictionTokens?: number;
@@ -61,7 +69,6 @@ export interface MessageItem {
   search: GroundingSearch | null;
   sessionId: string | null;
   threadId: string | null;
-  // jsonb type
   tools: any | null;
   topicId: string | null;
   // jsonb type
@@ -96,3 +103,17 @@ export interface NewMessage {
   updatedAt?: Date;
   userId: string; // optional because it's generated
 }
+
+export interface UpdateMessageParams {
+  content?: string;
+  error?: ChatMessageError | null;
+  imageList?: ChatImageItem[];
+  metadata?: MessageMetadata;
+  model?: string;
+  provider?: string;
+  reasoning?: ModelReasoning;
+  role?: string;
+  search?: GroundingSearch;
+  toolCalls?: MessageToolCall[];
+  tools?: ChatToolPayload[] | null;
+}

package/src/types/message/chat.ts

@@ -7,6 +7,7 @@ import { MessageSemanticSearchChunk } from '@/types/rag';
 import { GroundingSearch } from '@/types/search';
 
 import { MessageMetadata, MessageRoleType, ModelReasoning } from './base';
+import { ChatImageItem } from './image';
 import { ChatPluginPayload, ChatToolPayload } from './tools';
 import { Translate } from './translate';
 
@@ -37,12 +38,6 @@ export interface ChatFileItem {
   url: string;
 }
 
-export interface ChatImageItem {
-  alt: string;
-  id: string;
-  url: string;
-}
-
 export interface ChatFileChunk {
   fileId: string;
   fileType: string;

package/src/types/message/image.ts

@@ -0,0 +1,11 @@
+export interface ChatImageItem {
+  alt: string;
+  id: string;
+  url: string;
+}
+
+export interface ChatImageChunk {
+  data: string;
+  id: string;
+  isBase64?: boolean;
+}

package/src/types/message/index.ts

@@ -2,6 +2,7 @@ import { UploadFileItem } from '@/types/files';
 
 export * from './base';
 export * from './chat';
+export * from './image';
 export * from './tools';
 
 export interface SendMessageParams {

package/src/utils/fetch/fetchSSE.ts

@@ -12,7 +12,9 @@ import {
   ModelReasoning,
   ModelTokensUsage,
 } from '@/types/message';
+import { ChatImageChunk } from '@/types/message/image';
 import { GroundingSearch } from '@/types/search';
+import { nanoid } from '@/utils/uuid';
 
 import { fetchEventSource } from './fetchEventSource';
 import { getMessageError } from './parseError';
@@ -24,6 +26,7 @@ export type OnFinishHandler = (
   text: string,
   context: {
     grounding?: GroundingSearch;
+    images?: ChatImageChunk[];
     observationId?: string | null;
     reasoning?: ModelReasoning;
     toolCalls?: MessageToolCall[];
@@ -43,6 +46,13 @@ export interface MessageTextChunk {
   type: 'text';
 }
 
+export interface MessageBase64ImageChunk {
+  id: string;
+  image: ChatImageChunk;
+  images: ChatImageChunk[];
+  type: 'base64_image';
+}
+
 export interface MessageReasoningChunk {
   signature?: string;
   text?: string;
@@ -71,7 +81,8 @@ export interface FetchSSEOptions {
       | MessageToolCallsChunk
       | MessageReasoningChunk
       | MessageGroundingChunk
-      | MessageUsageChunk,
+      | MessageUsageChunk
+      | MessageBase64ImageChunk,
   ) => void;
   smoothing?: SmoothingParams | boolean;
 }
@@ -330,6 +341,8 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio
 
   let grounding: GroundingSearch | undefined = undefined;
   let usage: ModelTokensUsage | undefined = undefined;
+  let images: ChatImageChunk[] = [];
+
   await fetchEventSource(url, {
     body: options.body,
     fetch: options?.fetcher,
@@ -389,6 +402,15 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio
           break;
         }
 
+        case 'base64_image': {
+          const id = 'tmp_img_' + nanoid();
+          const item = { data, id, isBase64: true };
+          images.push(item);
+
+          options.onMessageHandle?.({ id, image: item, images, type: 'base64_image' });
+          break;
+        }
+
         case 'text': {
           // skip empty text
           if (!data) break;
@@ -492,6 +514,7 @@ export const fetchSSE = async (url: string, options: RequestInit & FetchSSEOptio
 
       await options?.onFinish?.(output, {
         grounding,
+        images: images.length > 0 ? images : undefined,
         observationId,
         reasoning: !!thinking ? { content: thinking, signature: thinkingSignature } : undefined,
         toolCalls,