@lobehub/chat 1.142.2 → 1.142.3

package/src/store/file/slices/fileManager/action.ts

@@ -1,7 +1,8 @@
+import pMap from 'p-map';
 import { SWRResponse, mutate } from 'swr';
 import { StateCreator } from 'zustand/vanilla';
 
-import { FILE_UPLOAD_BLACKLIST } from '@/const/file';
+import { FILE_UPLOAD_BLACKLIST, MAX_UPLOAD_FILE_COUNT } from '@/const/file';
 import { useClientDataSWR } from '@/libs/swr';
 import { fileService } from '@/services/file';
 import { ServerService } from '@/services/file/server';
@@ -12,6 +13,7 @@ import {
 } from '@/store/file/reducers/uploadFileList';
 import { FileListItem, QueryFileListParams } from '@/types/files';
 import { isChunkingUnsupported } from '@/utils/isChunkingUnsupported';
+import { unzipFile } from '@/utils/unzipFile';
 
 import { FileStore } from '../../store';
 import { fileManagerSelectors } from './selectors';
@@ -89,18 +91,37 @@ export const createFileManageSlice: StateCreator<
   pushDockFileList: async (rawFiles, knowledgeBaseId) => {
     const { dispatchDockFileList } = get();
 
-    // 0. skip file in blacklist
-    const files = rawFiles.filter((file) => !FILE_UPLOAD_BLACKLIST.includes(file.name));
+    // 0. Process ZIP files and extract their contents
+    const filesToUpload: File[] = [];
+    for (const file of rawFiles) {
+      if (file.type === 'application/zip' || file.name.endsWith('.zip')) {
+        try {
+          const extractedFiles = await unzipFile(file);
+          filesToUpload.push(...extractedFiles);
+        } catch (error) {
+          console.error('Failed to extract ZIP file:', error);
+          // If extraction fails, treat it as a regular file
+          filesToUpload.push(file);
+        }
+      } else {
+        filesToUpload.push(file);
+      }
+    }
 
-    // 1. add files
+    // 1. skip file in blacklist
+    const files = filesToUpload.filter((file) => !FILE_UPLOAD_BLACKLIST.includes(file.name));
+
+    // 2. Add all files to dock
     dispatchDockFileList({
       atStart: true,
       files: files.map((file) => ({ file, id: file.name, status: 'pending' })),
       type: 'addFiles',
     });
 
-    const uploadResults = await Promise.all(
-      files.map(async (file) => {
+    // 3. Upload files with concurrency limit using p-map
+    const uploadResults = await pMap(
+      files,
+      async (file) => {
         const result = await get().uploadWithProgress({
           file,
           knowledgeBaseId,
@@ -110,10 +131,11 @@ export const createFileManageSlice: StateCreator<
         await get().refreshFileList();
 
         return { file, fileId: result?.id, fileType: file.type };
-      }),
+      },
+      { concurrency: MAX_UPLOAD_FILE_COUNT },
     );
 
-    // 2. auto-embed files that support chunking
+    // 4. auto-embed files that support chunking
     const fileIdsToEmbed = uploadResults
       .filter(({ fileType, fileId }) => fileId && !isChunkingUnsupported(fileType))
       .map(({ fileId }) => fileId!);

package/src/utils/unzipFile.test.ts

@@ -0,0 +1,128 @@
+import { zip } from 'fflate';
+import { describe, expect, it } from 'vitest';
+
+import { unzipFile } from './unzipFile';
+
+describe('unzipFile', () => {
+  it('should extract files from a ZIP archive', async () => {
+    // Create a mock ZIP file with test data
+    const testFiles = {
+      'test.txt': new TextEncoder().encode('Hello, World!'),
+      'folder/nested.txt': new TextEncoder().encode('Nested file content'),
+    };
+
+    const zipped = await new Promise<Uint8Array>((resolve, reject) => {
+      zip(testFiles, (error, data) => {
+        if (error) reject(error);
+        else resolve(data);
+      });
+    });
+
+    const zipFile = new File([new Uint8Array(zipped)], 'test.zip', { type: 'application/zip' });
+
+    const extractedFiles = await unzipFile(zipFile);
+
+    expect(extractedFiles).toHaveLength(2);
+    expect(extractedFiles[0].name).toBe('test.txt');
+    expect(extractedFiles[1].name).toBe('nested.txt');
+
+    // Verify file contents
+    const content1 = await extractedFiles[0].text();
+    expect(content1).toBe('Hello, World!');
+
+    const content2 = await extractedFiles[1].text();
+    expect(content2).toBe('Nested file content');
+  });
+
+  it('should skip directories in ZIP archive', async () => {
+    const testFiles = {
+      'file.txt': new TextEncoder().encode('File content'),
+      'folder/': new Uint8Array(0), // Directory entry
+    };
+
+    const zipped = await new Promise<Uint8Array>((resolve, reject) => {
+      zip(testFiles, (error, data) => {
+        if (error) reject(error);
+        else resolve(data);
+      });
+    });
+
+    const zipFile = new File([new Uint8Array(zipped)], 'test.zip', { type: 'application/zip' });
+
+    const extractedFiles = await unzipFile(zipFile);
+
+    expect(extractedFiles).toHaveLength(1);
+    expect(extractedFiles[0].name).toBe('file.txt');
+  });
+
+  it('should skip hidden files and __MACOSX directories', async () => {
+    const testFiles = {
+      '.hidden': new TextEncoder().encode('Hidden file'),
+      '__MACOSX/._file.txt': new TextEncoder().encode('Mac metadata'),
+      'visible.txt': new TextEncoder().encode('Visible file'),
+    };
+
+    const zipped = await new Promise<Uint8Array>((resolve, reject) => {
+      zip(testFiles, (error, data) => {
+        if (error) reject(error);
+        else resolve(data);
+      });
+    });
+
+    const zipFile = new File([new Uint8Array(zipped)], 'test.zip', { type: 'application/zip' });
+
+    const extractedFiles = await unzipFile(zipFile);
+
+    expect(extractedFiles).toHaveLength(1);
+    expect(extractedFiles[0].name).toBe('visible.txt');
+  });
+
+  it('should set correct MIME types for extracted files', async () => {
+    const testFiles = {
+      'document.pdf': new TextEncoder().encode('PDF content'),
+      'image.png': new TextEncoder().encode('PNG content'),
+      'code.ts': new TextEncoder().encode('TypeScript code'),
+    };
+
+    const zipped = await new Promise<Uint8Array>((resolve, reject) => {
+      zip(testFiles, (error, data) => {
+        if (error) reject(error);
+        else resolve(data);
+      });
+    });
+
+    const zipFile = new File([new Uint8Array(zipped)], 'test.zip', { type: 'application/zip' });
+
+    const extractedFiles = await unzipFile(zipFile);
+
+    expect(extractedFiles).toHaveLength(3);
+    expect(extractedFiles.find((f) => f.name === 'document.pdf')?.type).toBe('application/pdf');
+    expect(extractedFiles.find((f) => f.name === 'image.png')?.type).toBe('image/png');
+    expect(extractedFiles.find((f) => f.name === 'code.ts')?.type).toBe('text/typescript');
+  });
+
+  it('should handle empty ZIP files', async () => {
+    const testFiles = {};
+
+    const zipped = await new Promise<Uint8Array>((resolve, reject) => {
+      zip(testFiles, (error, data) => {
+        if (error) reject(error);
+        else resolve(data);
+      });
+    });
+
+    const zipFile = new File([new Uint8Array(zipped)], 'empty.zip', { type: 'application/zip' });
+
+    const extractedFiles = await unzipFile(zipFile);
+
+    expect(extractedFiles).toHaveLength(0);
+  });
+
+  it('should reject on invalid ZIP file', async () => {
+    const invalidFile = new File([new Uint8Array([1, 2, 3, 4])], 'invalid.zip', {
+      type: 'application/zip',
+    });
+
+    await expect(unzipFile(invalidFile)).rejects.toThrow();
+  });
+});

package/src/utils/unzipFile.ts

@@ -0,0 +1,122 @@
+import { unzip } from 'fflate';
+
+/**
+ * Determines the MIME type based on file extension
+ */
+const getFileType = (fileName: string): string => {
+  const extension = fileName.split('.').pop()?.toLowerCase() || '';
+
+  const mimeTypes: Record<string, string> = {
+    // Images
+    bmp: 'image/bmp',
+
+    // Code files
+    c: 'text/x-c',
+
+    cpp: 'text/x-c++',
+
+    cs: 'text/x-csharp',
+
+    css: 'text/css',
+
+    // Documents
+    csv: 'text/csv',
+
+    doc: 'application/msword',
+
+    docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+
+    gif: 'image/gif',
+
+    go: 'text/x-go',
+
+    html: 'text/html',
+
+    java: 'text/x-java',
+
+    jpeg: 'image/jpeg',
+
+    jpg: 'image/jpeg',
+
+    js: 'text/javascript',
+
+    json: 'application/json',
+
+    jsx: 'text/javascript',
+
+    md: 'text/markdown',
+
+    pdf: 'application/pdf',
+
+    php: 'application/x-httpd-php',
+
+    png: 'image/png',
+
+    ppt: 'application/vnd.ms-powerpoint',
+    pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+    py: 'text/x-python',
+    rb: 'text/x-ruby',
+    rs: 'text/x-rust',
+    rtf: 'application/rtf',
+    sh: 'application/x-sh',
+    svg: 'image/svg+xml',
+    ts: 'text/typescript',
+    tsx: 'text/typescript',
+    txt: 'text/plain',
+    webp: 'image/webp',
+    xls: 'application/vnd.ms-excel',
+    xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    xml: 'application/xml',
+  };
+
+  return mimeTypes[extension] || 'application/octet-stream';
+};
+
+/**
+ * Extracts files from a ZIP archive
+ * @param zipFile - The ZIP file to extract
+ * @returns Promise that resolves to an array of extracted Files
+ */
+export const unzipFile = async (zipFile: File): Promise<File[]> => {
+  return new Promise((resolve, reject) => {
+    zipFile
+      .arrayBuffer()
+      .then((arrayBuffer) => {
+        const uint8Array = new Uint8Array(arrayBuffer);
+
+        unzip(uint8Array, (error, unzipped) => {
+          if (error) {
+            reject(error);
+            return;
+          }
+
+          const extractedFiles: File[] = [];
+
+          for (const [path, data] of Object.entries(unzipped)) {
+            // Skip directories and hidden files
+            if (path.endsWith('/') || path.includes('__MACOSX') || path.startsWith('.')) {
+              continue;
+            }
+
+            // Get the filename from the path
+            const fileName = path.split('/').pop() || path;
+
+            // Create a File object from the extracted data
+            const blob = new Blob([new Uint8Array(data)], {
+              type: getFileType(fileName),
+            });
+            const file = new File([blob], fileName, {
+              type: getFileType(fileName),
+            });
+
+            extractedFiles.push(file);
+          }
+
+          resolve(extractedFiles);
+        });
+      })
+      .catch(() => {
+        reject(new Error('Failed to read ZIP file'));
+      });
+  });
+};

package/vitest.config.mts

@@ -15,6 +15,7 @@ export default defineConfig({
       '@/const/locale': resolve(__dirname, './src/const/locale'),
       // TODO: after refactor the errorResponse, we can remove it
       '@/utils/errorResponse': resolve(__dirname, './src/utils/errorResponse'),
+      '@/utils/unzipFile': resolve(__dirname, './src/utils/unzipFile'),
       '@/utils': resolve(__dirname, './packages/utils/src'),
       '@/types': resolve(__dirname, './packages/types/src'),
       '@/const': resolve(__dirname, './packages/const/src'),