@lobehub/chat 1.11.9 → 1.12.1

package/src/store/file/slices/chat/initialState.ts

@@ -1,13 +1,18 @@
 import { FilePreview } from '@/types/files';
+import { UploadFileItem } from '@/types/files/upload';
 
 export interface ImageFileState {
+  chatUploadFileList: UploadFileItem[];
+  /**
+   * it should be removed after dalle plugin refactor
+   * @deprecated
+   */
   imagesMap: Record<string, FilePreview>;
-  inputFilesList: string[];
   uploadingIds: string[];
 }
 
 export const initialImageFileState: ImageFileState = {
+  chatUploadFileList: [],
   imagesMap: {},
-  inputFilesList: [],
   uploadingIds: [],
 };

package/src/store/file/slices/chat/selectors.test.ts

@@ -1,78 +1,101 @@
-import { beforeEach, describe, expect, it } from 'vitest';
+import { describe, expect, it } from 'vitest';
 
-import { FilesStoreState } from '../../initialState';
-import { filesSelectors } from './selectors';
+import { FilesStoreState, initialState } from '@/store/file/initialState';
+import { UPLOAD_STATUS_SET, UploadFileItem } from '@/types/files/upload';
 
-describe('filesSelectors', () => {
-  let state: FilesStoreState;
+import { fileChatSelectors, filesSelectors } from './selectors';
 
-  beforeEach(() => {
-    // 创建并初始化 state 的模拟实例
-    state = {
-      imagesMap: {
-        '1': {
-          id: '1',
-          name: 'a',
-          fileType: 'image/png',
-          saveMode: 'local',
-          base64Url: 'base64string1',
-          url: 'blob:abc',
-        },
-        '2': {
-          id: '2',
-          name: 'b',
-          fileType: 'image/png',
-          saveMode: 'url',
-          base64Url: 'base64string2',
-          url: 'url2',
-        },
-      },
-      uploadingIds: [],
-      // 假设 '3' 是不存在的 ID
-      inputFilesList: ['1', '2', '3'],
-    };
+describe('filesSelectors', () => {
+  describe('chatUploadFileList', () => {
+    it('should return the chatUploadFileList from state', () => {
+      const state = {
+        ...initialState,
+        chatUploadFileList: [{ id: '1' }] as UploadFileItem[],
+      } as FilesStoreState;
+      expect(filesSelectors.chatUploadFileList(state)).toEqual([{ id: '1' }]);
+    });
   });
 
-  it('getImageDetailByList should return details for the provided list of image IDs', () => {
-    const list = ['1', '2'];
-    const details = filesSelectors.getImageDetailByList(list)(state);
-    expect(details.length).toBe(2);
-    expect(details[0].name).toBe('a');
-    expect(details[1].name).toBe('b');
-  });
+  describe('isImageUploading', () => {
+    it('should return true if there are uploading ids', () => {
+      const state = { uploadingIds: ['1', '2'] } as FilesStoreState;
+      expect(filesSelectors.isImageUploading(state)).toBe(true);
+    });
 
-  it('imageDetailList should return details for the images in the inputFilesList', () => {
-    const details = filesSelectors.imageDetailList(state);
-    // '3' should be filtered due to not exist in map
-    expect(details.length).toBe(2);
+    it('should return false if there are no uploading ids', () => {
+      const state = { uploadingIds: [] as string[] } as FilesStoreState;
+      expect(filesSelectors.isImageUploading(state)).toBe(false);
+    });
   });
+});
 
-  it('getImageUrlOrBase64ById should return the correct URL or Base64 based on saveMode', () => {
-    const localImage = filesSelectors.getImageUrlOrBase64ById('1')(state);
-    expect(localImage).toEqual({ id: '1', url: 'base64string1' });
+describe('fileChatSelectors', () => {
+  describe('chatRawFileList', () => {
+    it('should return a list of raw files', () => {
+      const state = {
+        chatUploadFileList: [
+          { file: { name: 'test1.jpg' } },
+          { file: { name: 'test2.jpg' } },
+        ] as UploadFileItem[],
+      } as FilesStoreState;
 
-    const serverImage = filesSelectors.getImageUrlOrBase64ById('2')(state);
-    expect(serverImage).toEqual({ id: '2', url: 'url2' });
+      expect(fileChatSelectors.chatRawFileList(state)).toEqual([
+        { name: 'test1.jpg' },
+        { name: 'test2.jpg' },
+      ]);
+    });
+  });
 
-    const nonExistentImage = filesSelectors.getImageUrlOrBase64ById('3')(state);
-    expect(nonExistentImage).toBeUndefined();
+  describe('chatUploadFileList', () => {
+    it('should return the chatUploadFileList from state', () => {
+      const state = {
+        chatUploadFileList: [{ id: '1' }] as UploadFileItem[],
+      } as FilesStoreState;
+      expect(fileChatSelectors.chatUploadFileList(state)).toEqual([{ id: '1' }]);
+    });
   });
 
-  it('getImageUrlOrBase64ByList should return the correct list of URLs or Base64 strings', () => {
-    const list = ['1', '2', '3'];
-    const urlsOrBase64s = filesSelectors.getImageUrlOrBase64ByList(list)(state);
-    expect(urlsOrBase64s.length).toBe(2); // '3' 应该被过滤掉，因为它不存在
-    expect(urlsOrBase64s[0].url).toBe('base64string1');
-    expect(urlsOrBase64s[1].url).toBe('url2');
+  describe('chatUploadFileListHasItem', () => {
+    it('should return true if chatUploadFileList has items', () => {
+      const state = { chatUploadFileList: [{ id: '1' }] as UploadFileItem[] } as FilesStoreState;
+      expect(fileChatSelectors.chatUploadFileListHasItem(state)).toBe(true);
+    });
+
+    it('should return false if chatUploadFileList is empty', () => {
+      const state = { chatUploadFileList: [] as UploadFileItem[] } as FilesStoreState;
+      expect(fileChatSelectors.chatUploadFileListHasItem(state)).toBe(false);
+    });
   });
 
-  it('imageUrlOrBase64List should return a list of image URLs or Base64 strings for all images in inputFilesList', () => {
-    const urlsOrBase64s = filesSelectors.imageUrlOrBase64List(state);
-    expect(urlsOrBase64s.length).toBe(2); // '3' 是不存在的 ID，所以应该被过滤掉
+  describe('isUploadingFiles', () => {
+    it('should return true if any file is in uploading status', () => {
+      const state = {
+        chatUploadFileList: [
+          { status: Array.from(UPLOAD_STATUS_SET)[0] },
+          { status: 'completed' },
+        ] as UploadFileItem[],
+      } as FilesStoreState;
+      expect(fileChatSelectors.isUploadingFiles(state)).toBe(true);
+    });
+
+    it('should return true if any file has unfinished embedding tasks', () => {
+      const state = {
+        chatUploadFileList: [
+          { status: 'success', tasks: { finishEmbedding: false } },
+          { status: 'success', tasks: { finishEmbedding: true } },
+        ] as UploadFileItem[],
+      } as FilesStoreState;
+      expect(fileChatSelectors.isUploadingFiles(state)).toBe(true);
+    });
 
-    expect(urlsOrBase64s).toEqual([
-      { id: '1', url: 'base64string1' }, // 本地保存的图像应该使用 base64 URL
-      { id: '2', url: 'url2' }, // 服务器保存的图像应该使用普通 URL
-    ]);
+    it('should return false if no files are uploading or have unfinished tasks', () => {
+      const state: FilesStoreState = {
+        chatUploadFileList: [
+          { status: 'success', tasks: { finishEmbedding: true } },
+          { status: 'success' },
+        ] as UploadFileItem[],
+      } as FilesStoreState;
+      expect(fileChatSelectors.isUploadingFiles(state)).toBe(false);
+    });
   });
 });

package/src/store/file/slices/chat/selectors.ts

@@ -1,40 +1,30 @@
-import { FilesStoreState } from '../../initialState';
-
-const getImageDetailByList = (list: string[]) => (s: FilesStoreState) =>
-  list
-    .map((i) => s.imagesMap[i])
-    .filter(Boolean)
-    .map((i) => ({ ...i, loading: s.uploadingIds.includes(i.id) }));
-
-const imageDetailList = (s: FilesStoreState) => getImageDetailByList(s.inputFilesList)(s);
-
-const getImageUrlOrBase64ById =
-  (id: string) =>
-  (s: FilesStoreState): { id: string; url: string } | undefined => {
-    const preview = s.imagesMap[id];
-
-    if (!preview) return undefined;
-
-    const url = preview.saveMode === 'local' ? (preview.base64Url as string) : preview.url;
+import { UPLOAD_STATUS_SET } from '@/types/files/upload';
 
-    return { id, url: url };
-  };
+import { FilesStoreState } from '../../initialState';
 
-const getImageUrlOrBase64ByList = (idList: string[]) => (s: FilesStoreState) =>
-  idList.map((i) => getImageUrlOrBase64ById(i)(s)).filter(Boolean) as {
-    id: string;
-    url: string;
-  }[];
+const chatUploadFileList = (s: FilesStoreState) => s.chatUploadFileList;
+const isImageUploading = (s: FilesStoreState) => s.uploadingIds.length > 0;
 
-const imageUrlOrBase64List = (s: FilesStoreState) => getImageUrlOrBase64ByList(s.inputFilesList)(s);
+const chatRawFileList = (s: FilesStoreState) => s.chatUploadFileList.map((item) => item.file);
+const chatUploadFileListHasItem = (s: FilesStoreState) => s.chatUploadFileList.length > 0;
 
-const isImageUploading = (s: FilesStoreState) => s.uploadingIds.length > 0;
+const isUploadingFiles = (s: FilesStoreState) =>
+  s.chatUploadFileList.some(
+    (file) =>
+      // is file status in uploading
+      UPLOAD_STATUS_SET.has(file.status) ||
+      // or file has tasks but not finish embedding
+      (file.tasks && !file.tasks?.finishEmbedding),
+  );
 
 export const filesSelectors = {
-  getImageDetailByList,
-  getImageUrlOrBase64ById,
-  getImageUrlOrBase64ByList,
-  imageDetailList,
-  imageUrlOrBase64List,
+  chatUploadFileList,
   isImageUploading,
 };
+
+export const fileChatSelectors = {
+  chatRawFileList,
+  chatUploadFileList,
+  chatUploadFileListHasItem,
+  isUploadingFiles,
+};

package/src/store/file/slices/chunk/action.ts

@@ -0,0 +1,36 @@
+import { StateCreator } from 'zustand/vanilla';
+
+import { ragService } from '@/services/rag';
+
+import { FileStore } from '../../store';
+
+export interface FileChunkAction {
+  closeChunkDrawer: () => void;
+  highlightChunks: (ids: string[]) => void;
+
+  openChunkDrawer: (id: string) => void;
+  semanticSearch: (text: string, fileId: string) => Promise<void>;
+}
+
+export const createFileChunkSlice: StateCreator<
+  FileStore,
+  [['zustand/devtools', never]],
+  [],
+  FileChunkAction
+> = (set) => ({
+  closeChunkDrawer: () => {
+    set({ chunkDetailId: null, isSimilaritySearch: false, similaritySearchChunks: [] });
+  },
+  highlightChunks: (ids) => {
+    set({ highlightChunkIds: ids });
+  },
+  openChunkDrawer: (id) => {
+    set({ chunkDetailId: id });
+  },
+
+  semanticSearch: async (text, fileId) => {
+    set({ isSimilaritySearching: true });
+    const data = await ragService.semanticSearch(text, [fileId]);
+    set({ isSimilaritySearching: false, similaritySearchChunks: data });
+  },
+});

package/src/store/file/slices/chunk/index.ts

@@ -0,0 +1,3 @@
+export * from './action';
+export * from './initialState';
+export * from './selectors';

package/src/store/file/slices/chunk/initialState.ts

@@ -0,0 +1,15 @@
+import { SemanticSearchChunk } from '@/types/chunk';
+
+export interface FileChunkState {
+  chunkDetailId: string | null;
+  highlightChunkIds: string[];
+  isSimilaritySearch?: boolean;
+  isSimilaritySearching?: boolean;
+  similaritySearchChunks?: SemanticSearchChunk[];
+}
+
+export const initialFileChunkState: FileChunkState = {
+  chunkDetailId: null,
+  highlightChunkIds: [],
+  similaritySearchChunks: [],
+};

package/src/store/file/slices/chunk/selectors.ts

@@ -0,0 +1,10 @@
+// import { FileStore } from '../../store';
+import { FilesStoreState } from '@/store/file/initialState';
+
+const showSimilaritySearchResult = (s: FilesStoreState) => s.isSimilaritySearch;
+const enabledChunkFileId = (s: FilesStoreState) => s.chunkDetailId;
+
+export const fileChunkSelectors = {
+  enabledChunkFileId,
+  showSimilaritySearchResult,
+};

package/src/store/file/slices/fileManager/action.ts

@@ -0,0 +1,187 @@
+import { SWRResponse, mutate } from 'swr';
+import { StateCreator } from 'zustand/vanilla';
+
+import { FILE_UPLOAD_BLACKLIST } from '@/const/file';
+import { useClientDataSWR } from '@/libs/swr';
+import { fileService } from '@/services/file';
+import { ServerService } from '@/services/file/server';
+import { ragService } from '@/services/rag';
+import {
+  UploadFileListDispatch,
+  uploadFileListReducer,
+} from '@/store/file/reducers/uploadFileList';
+import { FileListItem, QueryFileListParams } from '@/types/files';
+
+import { FileStore } from '../../store';
+
+const serverFileService = new ServerService();
+
+export interface FileManageAction {
+  dispatchDockFileList: (payload: UploadFileListDispatch) => void;
+  embeddingChunks: (fileIds: string[]) => Promise<void>;
+  parseFilesToChunks: (ids: string[], params?: { skipExist?: boolean }) => Promise<void>;
+  pushDockFileList: (files: File[], knowledgeBaseId?: string) => Promise<void>;
+
+  reParseFile: (id: string) => Promise<void>;
+  refreshFileList: () => Promise<void>;
+  removeAllFiles: () => Promise<void>;
+  removeFileItem: (id: string) => Promise<void>;
+  removeFiles: (ids: string[]) => Promise<void>;
+
+  toggleEmbeddingIds: (ids: string[], loading?: boolean) => void;
+  toggleParsingIds: (ids: string[], loading?: boolean) => void;
+
+  useFetchFileItem: (id?: string) => SWRResponse<FileListItem | undefined>;
+  useFetchFileManage: (params: QueryFileListParams) => SWRResponse<FileListItem[]>;
+}
+
+const FETCH_FILE_LIST_KEY = 'useFetchFileManage';
+
+export const createFileManageSlice: StateCreator<
+  FileStore,
+  [['zustand/devtools', never]],
+  [],
+  FileManageAction
+> = (set, get) => ({
+  dispatchDockFileList: (payload: UploadFileListDispatch) => {
+    const nextValue = uploadFileListReducer(get().dockUploadFileList, payload);
+    if (nextValue === get().dockUploadFileList) return;
+
+    set({ dockUploadFileList: nextValue }, false, `dispatchDockFileList/${payload.type}`);
+  },
+  embeddingChunks: async (fileIds: string[]) => {
+    // toggle file ids
+    get().toggleEmbeddingIds(fileIds);
+
+    // parse files
+    const pools = fileIds.map(async (id) => {
+      try {
+        await ragService.createEmbeddingChunksTask(id);
+      } catch (e) {
+        console.error(e);
+      }
+    });
+
+    await Promise.all(pools);
+    await get().refreshFileList();
+    get().toggleEmbeddingIds(fileIds, false);
+  },
+  parseFilesToChunks: async (ids: string[], params) => {
+    // toggle file ids
+    get().toggleParsingIds(ids);
+
+    // parse files
+    const pools = ids.map(async (id) => {
+      try {
+        await ragService.createParseFileTask(id, params?.skipExist);
+      } catch (e) {
+        console.error(e);
+      }
+    });
+
+    await Promise.all(pools);
+    await get().refreshFileList();
+    get().toggleParsingIds(ids, false);
+  },
+  pushDockFileList: async (rawFiles, knowledgeBaseId) => {
+    const { dispatchDockFileList } = get();
+
+    // 0. skip file in blacklist
+    const files = rawFiles.filter((file) => !FILE_UPLOAD_BLACKLIST.includes(file.name));
+
+    // 1. add files
+    dispatchDockFileList({
+      atStart: true,
+      files: files.map((file) => ({ file, id: file.name, status: 'pending' })),
+      type: 'addFiles',
+    });
+
+    const pools = files.map(async (file) => {
+      await get().uploadWithProgress({
+        file,
+        knowledgeBaseId,
+        onStatusUpdate: dispatchDockFileList,
+      });
+
+      await get().refreshFileList();
+    });
+
+    await Promise.all(pools);
+  },
+
+  reParseFile: async (id) => {
+    // toggle file ids
+    get().toggleParsingIds([id]);
+
+    await ragService.retryParseFile(id);
+
+    await get().refreshFileList();
+
+    get().toggleParsingIds([id], false);
+  },
+  refreshFileList: async () => {
+    await mutate([FETCH_FILE_LIST_KEY, get().queryListParams]);
+  },
+  removeAllFiles: async () => {
+    await fileService.removeAllFiles();
+  },
+  removeFileItem: async (id) => {
+    await fileService.removeFile(id);
+    await get().refreshFileList();
+  },
+
+  removeFiles: async (ids) => {
+    await fileService.removeFiles(ids);
+    await get().refreshFileList();
+  },
+  toggleEmbeddingIds: (ids, loading) => {
+    set((state) => {
+      const nextValue = new Set(state.creatingEmbeddingTaskIds);
+
+      ids.forEach((id) => {
+        if (typeof loading === 'undefined') {
+          if (nextValue.has(id)) nextValue.delete(id);
+          else nextValue.add(id);
+        } else {
+          if (loading) nextValue.add(id);
+          else nextValue.delete(id);
+        }
+      });
+
+      return { creatingEmbeddingTaskIds: Array.from(nextValue.values()) };
+    });
+  },
+  toggleParsingIds: (ids, loading) => {
+    set((state) => {
+      const nextValue = new Set(state.creatingChunkingTaskIds);
+
+      ids.forEach((id) => {
+        if (typeof loading === 'undefined') {
+          if (nextValue.has(id)) nextValue.delete(id);
+          else nextValue.add(id);
+        } else {
+          if (loading) nextValue.add(id);
+          else nextValue.delete(id);
+        }
+      });
+
+      return { creatingChunkingTaskIds: Array.from(nextValue.values()) };
+    });
+  },
+
+  useFetchFileItem: (id) =>
+    useClientDataSWR<FileListItem | undefined>(!id ? null : ['useFetchFileItem', id], () =>
+      serverFileService.getFileItem(id!),
+    ),
+
+  useFetchFileManage: (params) =>
+    useClientDataSWR<FileListItem[]>(
+      [FETCH_FILE_LIST_KEY, params],
+      () => serverFileService.getFiles(params),
+      {
+        onSuccess: (data) => {
+          set({ fileList: data, queryListParams: params });
+        },
+      },
+    ),
+});

package/src/store/file/slices/fileManager/index.ts

@@ -0,0 +1,3 @@
+export * from './action';
+export * from './initialState';
+export * from './selectors';

package/src/store/file/slices/fileManager/initialState.ts

@@ -0,0 +1,18 @@
+import { FileListItem, QueryFileListParams } from '@/types/files';
+import { UploadFileItem } from '@/types/files/upload';
+
+export interface FileManagerState {
+  creatingChunkingTaskIds: string[];
+  creatingEmbeddingTaskIds: string[];
+  dockUploadFileList: UploadFileItem[];
+  fileDetail?: FileListItem;
+  fileList: FileListItem[];
+  queryListParams?: QueryFileListParams;
+}
+
+export const initialFileManagerState: FileManagerState = {
+  creatingChunkingTaskIds: [],
+  creatingEmbeddingTaskIds: [],
+  dockUploadFileList: [],
+  fileList: [],
+};

package/src/store/file/slices/fileManager/selectors.ts

@@ -0,0 +1,58 @@
+// import { FileStore } from '../../store';
+import { FilesStoreState } from '@/store/file/initialState';
+import { FileUploadStatus } from '@/types/files/upload';
+
+const uploadStatusArray = new Set(['uploading', 'pending', 'processing']);
+
+const dockFileList = (s: FilesStoreState) => s.dockUploadFileList;
+const dockRawFileList = (s: FilesStoreState) => s.dockUploadFileList.map((item) => item.file);
+const getFileById = (id?: string | null) => (s: FilesStoreState) => {
+  if (!id) return;
+
+  return s.fileList.find((item) => item.id === id);
+};
+
+const isUploadingFiles = (s: FilesStoreState) =>
+  s.dockUploadFileList.some((file) => uploadStatusArray.has(file.status));
+
+const overviewUploadingStatus = (s: FilesStoreState): FileUploadStatus => {
+  if (s.dockUploadFileList.length === 0) return 'pending';
+  if (s.dockUploadFileList.some((file) => uploadStatusArray.has(file.status))) {
+    return 'uploading';
+  }
+
+  return 'success';
+};
+
+const overviewUploadingProgress = (s: FilesStoreState) => {
+  const uploadFiles = s.dockUploadFileList.filter(
+    (file) => file.status === 'uploading' || file.status === 'pending',
+  );
+
+  if (uploadFiles.length === 0) return 100;
+
+  const totalPercent = uploadFiles.length * 100;
+  const currentPercent = uploadFiles.reduce(
+    (acc, file) => acc + (file.uploadState?.progress || 0),
+    0,
+  );
+
+  return (currentPercent / totalPercent) * 100;
+};
+
+const isCreatingFileParseTask = (id: string) => (s: FilesStoreState) =>
+  s.creatingChunkingTaskIds.includes(id);
+
+const isCreatingChunkEmbeddingTask = (id: string) => (s: FilesStoreState) =>
+  s.creatingEmbeddingTaskIds.includes(id);
+
+export const fileManagerSelectors = {
+  dockFileList,
+  dockRawFileList,
+  getFileById,
+  isCreatingChunkEmbeddingTask,
+  isCreatingFileParseTask,
+  isUploadingFiles,
+  overviewUploadingProgress,
+  overviewUploadingStatus,
+};

package/src/store/file/slices/tts/action.test.ts

@@ -66,7 +66,7 @@ describe('TTSFileAction', () => {
     };
 
     // Mock the fileService.uploadFile to resolve with uploadedFileData
-    vi.spyOn(fileService, 'createFile').mockResolvedValue(uploadedFileData);
+    vi.spyOn(fileService, 'createFile').mockResolvedValue({ id: uploadedFileData.id, url: '' });
 
     let fileId;
     await act(async () => {

package/src/store/file/slices/tts/action.ts

@@ -2,7 +2,7 @@ import useSWR, { SWRResponse } from 'swr';
 import { StateCreator } from 'zustand/vanilla';
 
 import { fileService } from '@/services/file';
-import { uploadService } from '@/services/upload';
+import { legacyUploadService } from '@/services/upload_legacy';
 import { FilePreview } from '@/types/files';
 
 import { FileStore } from '../../store';
@@ -42,7 +42,7 @@ export const createTTSFileSlice: StateCreator<
   },
   uploadTTSFile: async (file) => {
     try {
-      const res = await uploadService.uploadFile({
+      const res = await legacyUploadService.uploadFile({
         createdAt: file.lastModified,
         data: await file.arrayBuffer(),
         fileType: file.type,