@lobehub/chat 1.142.2 → 1.142.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +8 -8
- package/README.zh-CN.md +8 -8
- package/changelog/v1.json +18 -0
- package/docs/development/database-schema.dbml +1 -0
- package/locales/ar/chat.json +4 -4
- package/locales/ar/file.json +1 -0
- package/locales/ar/models.json +1 -1
- package/locales/bg-BG/chat.json +4 -4
- package/locales/bg-BG/file.json +1 -0
- package/locales/bg-BG/models.json +1 -1
- package/locales/de-DE/chat.json +4 -4
- package/locales/de-DE/file.json +1 -0
- package/locales/de-DE/models.json +1 -1
- package/locales/en-US/chat.json +4 -4
- package/locales/en-US/file.json +1 -0
- package/locales/en-US/models.json +1 -1
- package/locales/es-ES/chat.json +4 -4
- package/locales/es-ES/file.json +1 -0
- package/locales/es-ES/models.json +1 -1
- package/locales/fa-IR/chat.json +4 -4
- package/locales/fa-IR/file.json +1 -0
- package/locales/fa-IR/models.json +1 -1
- package/locales/fr-FR/chat.json +4 -4
- package/locales/fr-FR/file.json +1 -0
- package/locales/fr-FR/models.json +1 -1
- package/locales/it-IT/chat.json +4 -4
- package/locales/it-IT/file.json +1 -0
- package/locales/ja-JP/chat.json +4 -4
- package/locales/ja-JP/file.json +1 -0
- package/locales/ja-JP/models.json +1 -1
- package/locales/ko-KR/chat.json +4 -4
- package/locales/ko-KR/file.json +1 -0
- package/locales/ko-KR/models.json +1 -1
- package/locales/nl-NL/chat.json +4 -4
- package/locales/nl-NL/file.json +1 -0
- package/locales/nl-NL/models.json +1 -1
- package/locales/pl-PL/chat.json +4 -4
- package/locales/pl-PL/file.json +1 -0
- package/locales/pl-PL/models.json +1 -1
- package/locales/pt-BR/chat.json +4 -4
- package/locales/pt-BR/file.json +1 -0
- package/locales/ru-RU/chat.json +4 -4
- package/locales/ru-RU/file.json +1 -0
- package/locales/ru-RU/models.json +1 -1
- package/locales/tr-TR/chat.json +4 -4
- package/locales/tr-TR/file.json +1 -0
- package/locales/tr-TR/models.json +1 -1
- package/locales/vi-VN/chat.json +4 -4
- package/locales/vi-VN/file.json +1 -0
- package/locales/vi-VN/models.json +1 -1
- package/locales/zh-CN/chat.json +4 -4
- package/locales/zh-CN/file.json +1 -0
- package/locales/zh-TW/chat.json +4 -4
- package/locales/zh-TW/file.json +1 -0
- package/locales/zh-TW/models.json +1 -1
- package/package.json +3 -2
- package/packages/const/src/file.ts +2 -0
- package/packages/database/migrations/0039_add_editor_data.sql +1 -0
- package/packages/database/migrations/meta/0039_snapshot.json +7586 -0
- package/packages/database/migrations/meta/_journal.json +7 -0
- package/packages/database/src/core/migrations.json +6 -0
- package/packages/database/src/schemas/document.ts +2 -0
- package/packages/database/src/utils/__tests__/groupMessages.test.ts +989 -0
- package/packages/database/src/utils/groupMessages.ts +359 -0
- package/packages/memory-extract/.env.example +3 -0
- package/packages/memory-extract/package.json +21 -0
- package/packages/memory-extract/vitest.config.mts +10 -0
- package/packages/model-runtime/src/core/streams/protocol.ts +3 -3
- package/packages/model-runtime/src/types/chat.ts +2 -2
- package/packages/obervability-otel/package.json +7 -7
- package/packages/types/src/message/common/base.ts +0 -1
- package/packages/types/src/message/common/metadata.ts +5 -5
- package/packages/types/src/message/common/tools.ts +17 -0
- package/packages/types/src/message/db/item.ts +23 -17
- package/packages/types/src/message/ui/chat.ts +22 -66
- package/packages/types/src/message/ui/index.ts +1 -0
- package/packages/types/src/message/ui/params.ts +65 -0
- package/packages/types/src/tool/builtin.ts +34 -0
- package/packages/types/src/tool/intervention.ts +39 -0
- package/packages/utils/src/fetch/fetchSSE.ts +4 -4
- package/renovate.json +14 -2
- package/src/app/[variants]/(main)/settings/provider/features/ModelList/CreateNewModelModal/index.tsx +7 -0
- package/src/app/[variants]/(main)/settings/provider/features/ModelList/ModelConfigModal/index.tsx +7 -0
- package/src/app/[variants]/(main)/settings/provider/features/ProviderConfig/Checker.tsx +7 -2
- package/src/features/ChatInput/ActionBar/components/ActionDropdown.tsx +21 -1
- package/src/features/ChatItem/components/Title.tsx +4 -3
- package/src/features/Conversation/Messages/Assistant/MessageContent.tsx +3 -16
- package/src/features/Conversation/Messages/Assistant/index.tsx +3 -3
- package/src/features/Conversation/{utils.test.ts → utils/markdown.test.ts} +1 -1
- package/src/features/Conversation/{utils.ts → utils/markdown.ts} +1 -1
- package/src/features/FileManager/FileList/FileListItem/index.tsx +3 -2
- package/src/features/FileManager/FileList/MasonryFileItem/MasonryItemWrapper.tsx +1 -1
- package/src/features/FileManager/FileList/MasonryFileItem/index.tsx +2 -4
- package/src/features/FileManager/FileList/MasonrySkeleton.tsx +11 -5
- package/src/features/FileManager/FileList/ToolBar/MultiSelectActions.tsx +1 -1
- package/src/features/FileManager/FileList/index.tsx +51 -9
- package/src/features/ModelSwitchPanel/index.tsx +1 -0
- package/src/locales/default/chat.ts +4 -4
- package/src/locales/default/file.ts +1 -0
- package/src/store/file/slices/fileManager/action.test.ts +136 -1
- package/src/store/file/slices/fileManager/action.ts +30 -8
- package/src/tools/web-browsing/Render/PageContent/index.tsx +2 -2
- package/src/tools/web-browsing/Render/index.tsx +5 -4
- package/src/utils/unzipFile.test.ts +128 -0
- package/src/utils/unzipFile.ts +122 -0
- package/vitest.config.mts +1 -0
|
@@ -51,10 +51,15 @@ const FileList = memo<FileListProps>(({ knowledgeBaseId, category }) => {
|
|
|
51
51
|
|
|
52
52
|
const [selectFileIds, setSelectedFileIds] = useState<string[]>([]);
|
|
53
53
|
const [viewConfig, setViewConfig] = useState({ showFilesInKnowledgeBase: false });
|
|
54
|
+
const [lastSelectedIndex, setLastSelectedIndex] = useState<number | null>(null);
|
|
55
|
+
const [isTransitioning, setIsTransitioning] = useState(false);
|
|
54
56
|
|
|
55
57
|
const viewMode = useGlobalStore((s) => s.status.fileManagerViewMode || 'list') as ViewMode;
|
|
56
58
|
const updateSystemStatus = useGlobalStore((s) => s.updateSystemStatus);
|
|
57
|
-
const setViewMode = (mode: ViewMode) =>
|
|
59
|
+
const setViewMode = (mode: ViewMode) => {
|
|
60
|
+
setIsTransitioning(true);
|
|
61
|
+
updateSystemStatus({ fileManagerViewMode: mode });
|
|
62
|
+
};
|
|
58
63
|
|
|
59
64
|
const [columnCount, setColumnCount] = useState(4);
|
|
60
65
|
|
|
@@ -105,6 +110,19 @@ const FileList = memo<FileListProps>(({ knowledgeBaseId, category }) => {
|
|
|
105
110
|
...viewConfig,
|
|
106
111
|
});
|
|
107
112
|
|
|
113
|
+
// Handle view transition with a brief delay to show skeleton
|
|
114
|
+
React.useEffect(() => {
|
|
115
|
+
if (isTransitioning && data) {
|
|
116
|
+
// Use requestAnimationFrame to ensure smooth transition
|
|
117
|
+
requestAnimationFrame(() => {
|
|
118
|
+
const timer = setTimeout(() => {
|
|
119
|
+
setIsTransitioning(false);
|
|
120
|
+
}, 100);
|
|
121
|
+
return () => clearTimeout(timer);
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
}, [isTransitioning, viewMode, data]);
|
|
125
|
+
|
|
108
126
|
useCheckTaskStatus(data);
|
|
109
127
|
|
|
110
128
|
// Clean up selected files that no longer exist in the data
|
|
@@ -118,6 +136,13 @@ const FileList = memo<FileListProps>(({ knowledgeBaseId, category }) => {
|
|
|
118
136
|
}
|
|
119
137
|
}, [data]);
|
|
120
138
|
|
|
139
|
+
// Reset lastSelectedIndex when selection is cleared
|
|
140
|
+
React.useEffect(() => {
|
|
141
|
+
if (selectFileIds.length === 0) {
|
|
142
|
+
setLastSelectedIndex(null);
|
|
143
|
+
}
|
|
144
|
+
}, [selectFileIds.length]);
|
|
145
|
+
|
|
121
146
|
// Memoize context object to avoid recreating on every render
|
|
122
147
|
const masonryContext = useMemo(
|
|
123
148
|
() => ({
|
|
@@ -161,7 +186,7 @@ const FileList = memo<FileListProps>(({ knowledgeBaseId, category }) => {
|
|
|
161
186
|
</Flexbox>
|
|
162
187
|
)}
|
|
163
188
|
</Flexbox>
|
|
164
|
-
{isLoading ? (
|
|
189
|
+
{isLoading || isTransitioning ? (
|
|
165
190
|
viewMode === 'masonry' ? (
|
|
166
191
|
<MasonrySkeleton columnCount={columnCount} />
|
|
167
192
|
) : (
|
|
@@ -184,13 +209,30 @@ const FileList = memo<FileListProps>(({ knowledgeBaseId, category }) => {
|
|
|
184
209
|
index={index}
|
|
185
210
|
key={item.id}
|
|
186
211
|
knowledgeBaseId={knowledgeBaseId}
|
|
187
|
-
onSelectedChange={(id, checked) => {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
212
|
+
onSelectedChange={(id, checked, shiftKey, clickedIndex) => {
|
|
213
|
+
if (shiftKey && lastSelectedIndex !== null && selectFileIds.length > 0 && data) {
|
|
214
|
+
// Range selection with shift key
|
|
215
|
+
const start = Math.min(lastSelectedIndex, clickedIndex);
|
|
216
|
+
const end = Math.max(lastSelectedIndex, clickedIndex);
|
|
217
|
+
const rangeIds = data.slice(start, end + 1).map((item) => item.id);
|
|
218
|
+
|
|
219
|
+
setSelectedFileIds((prev) => {
|
|
220
|
+
// Create a Set for efficient lookup
|
|
221
|
+
const prevSet = new Set(prev);
|
|
222
|
+
// Add all items in range
|
|
223
|
+
rangeIds.forEach((rangeId) => prevSet.add(rangeId));
|
|
224
|
+
return Array.from(prevSet);
|
|
225
|
+
});
|
|
226
|
+
} else {
|
|
227
|
+
// Normal selection
|
|
228
|
+
setSelectedFileIds((prev) => {
|
|
229
|
+
if (checked) {
|
|
230
|
+
return [...prev, id];
|
|
231
|
+
}
|
|
232
|
+
return prev.filter((item) => item !== id);
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
setLastSelectedIndex(clickedIndex);
|
|
194
236
|
}}
|
|
195
237
|
selected={selectFileIds.includes(item.id)}
|
|
196
238
|
{...item}
|
|
@@ -175,9 +175,9 @@ export default {
|
|
|
175
175
|
addMember: '添加成员',
|
|
176
176
|
allMembers: '全体成员',
|
|
177
177
|
createGroup: '创建 Agent 团队',
|
|
178
|
-
noAvailableAgents: '
|
|
179
|
-
noSelectedAgents: '
|
|
180
|
-
searchAgents: '
|
|
178
|
+
noAvailableAgents: '没有可邀请的 Agent',
|
|
179
|
+
noSelectedAgents: '还未选择 Agent',
|
|
180
|
+
searchAgents: '搜索 Agent...',
|
|
181
181
|
setInitialMembers: '选择团队成员',
|
|
182
182
|
},
|
|
183
183
|
|
|
@@ -247,7 +247,7 @@ export default {
|
|
|
247
247
|
jumpToMessage: '跳转至第 {{index}} 条消息',
|
|
248
248
|
nextMessage: '下一条消息',
|
|
249
249
|
previousMessage: '上一条消息',
|
|
250
|
-
senderAssistant: '
|
|
250
|
+
senderAssistant: 'Agent',
|
|
251
251
|
senderUser: '你',
|
|
252
252
|
},
|
|
253
253
|
|
|
@@ -2,17 +2,50 @@ import { act, renderHook, waitFor } from '@testing-library/react';
|
|
|
2
2
|
import { mutate } from 'swr';
|
|
3
3
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { message } from '@/components/AntdStaticMethods';
|
|
6
|
+
import { FILE_UPLOAD_BLACKLIST, MAX_UPLOAD_FILE_COUNT } from '@/const/file';
|
|
6
7
|
import { lambdaClient } from '@/libs/trpc/client';
|
|
7
8
|
import { fileService } from '@/services/file';
|
|
8
9
|
import { ragService } from '@/services/rag';
|
|
9
10
|
import { FileListItem } from '@/types/files';
|
|
10
11
|
import { UploadFileItem } from '@/types/files/upload';
|
|
12
|
+
import { unzipFile } from '@/utils/unzipFile';
|
|
11
13
|
|
|
12
14
|
import { useFileStore as useStore } from '../../store';
|
|
13
15
|
|
|
14
16
|
vi.mock('zustand/traditional');
|
|
15
17
|
|
|
18
|
+
// Mock i18next translation function
|
|
19
|
+
vi.mock('i18next', () => ({
|
|
20
|
+
t: (key: string, options?: any) => {
|
|
21
|
+
// Return a mock translation string that includes the options for verification
|
|
22
|
+
if (key === 'uploadDock.fileQueueInfo' && options?.count !== undefined) {
|
|
23
|
+
return `Uploading ${options.count} files, ${options.remaining} queued`;
|
|
24
|
+
}
|
|
25
|
+
return key;
|
|
26
|
+
},
|
|
27
|
+
}));
|
|
28
|
+
|
|
29
|
+
// Mock message
|
|
30
|
+
vi.mock('@/components/AntdStaticMethods', () => ({
|
|
31
|
+
message: {
|
|
32
|
+
info: vi.fn(),
|
|
33
|
+
warning: vi.fn(),
|
|
34
|
+
},
|
|
35
|
+
}));
|
|
36
|
+
|
|
37
|
+
// Mock unzipFile
|
|
38
|
+
vi.mock('@/utils/unzipFile', () => ({
|
|
39
|
+
unzipFile: vi.fn(),
|
|
40
|
+
}));
|
|
41
|
+
|
|
42
|
+
// Mock p-map to run sequentially for easier testing
|
|
43
|
+
vi.mock('p-map', () => ({
|
|
44
|
+
default: vi.fn(async (items, mapper) => {
|
|
45
|
+
return Promise.all(items.map(mapper));
|
|
46
|
+
}),
|
|
47
|
+
}));
|
|
48
|
+
|
|
16
49
|
// Mock SWR
|
|
17
50
|
vi.mock('swr', async () => {
|
|
18
51
|
const actual = await vi.importActual('swr');
|
|
@@ -398,6 +431,108 @@ describe('FileManagerActions', () => {
|
|
|
398
431
|
// Should not auto-parse when upload returns undefined
|
|
399
432
|
expect(parseSpy).not.toHaveBeenCalled();
|
|
400
433
|
});
|
|
434
|
+
|
|
435
|
+
it('should enforce file count limit and queue excess files', async () => {
|
|
436
|
+
const { result } = renderHook(() => useStore());
|
|
437
|
+
|
|
438
|
+
// Create more files than the limit
|
|
439
|
+
const totalFiles = MAX_UPLOAD_FILE_COUNT + 5;
|
|
440
|
+
const files = Array.from(
|
|
441
|
+
{ length: totalFiles },
|
|
442
|
+
(_, i) => new File(['content'], `file-${i}.txt`, { type: 'text/plain' }),
|
|
443
|
+
);
|
|
444
|
+
|
|
445
|
+
vi.spyOn(result.current, 'uploadWithProgress').mockResolvedValue({
|
|
446
|
+
id: 'file-1',
|
|
447
|
+
url: 'http://example.com/file-1',
|
|
448
|
+
});
|
|
449
|
+
vi.spyOn(result.current, 'refreshFileList').mockResolvedValue();
|
|
450
|
+
vi.spyOn(result.current, 'parseFilesToChunks').mockResolvedValue();
|
|
451
|
+
const dispatchSpy = vi.spyOn(result.current, 'dispatchDockFileList');
|
|
452
|
+
|
|
453
|
+
await act(async () => {
|
|
454
|
+
await result.current.pushDockFileList(files);
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
// Should add all files to dock (not just first MAX_UPLOAD_FILE_COUNT)
|
|
458
|
+
expect(dispatchSpy).toHaveBeenCalledWith({
|
|
459
|
+
atStart: true,
|
|
460
|
+
files: expect.arrayContaining([
|
|
461
|
+
expect.objectContaining({ file: expect.any(File), status: 'pending' }),
|
|
462
|
+
]),
|
|
463
|
+
type: 'addFiles',
|
|
464
|
+
});
|
|
465
|
+
|
|
466
|
+
// Verify all files were dispatched
|
|
467
|
+
const dispatchCall = dispatchSpy.mock.calls.find((call) => call[0].type === 'addFiles');
|
|
468
|
+
expect(dispatchCall?.[0]).toHaveProperty('files');
|
|
469
|
+
if (dispatchCall && 'files' in dispatchCall[0]) {
|
|
470
|
+
expect(dispatchCall[0].files).toHaveLength(totalFiles);
|
|
471
|
+
}
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
it('should extract ZIP files and upload contents', async () => {
|
|
475
|
+
const { result } = renderHook(() => useStore());
|
|
476
|
+
|
|
477
|
+
const zipFile = new File(['zip content'], 'archive.zip', { type: 'application/zip' });
|
|
478
|
+
const extractedFiles = [
|
|
479
|
+
new File(['file1'], 'file1.txt', { type: 'text/plain' }),
|
|
480
|
+
new File(['file2'], 'file2.txt', { type: 'text/plain' }),
|
|
481
|
+
];
|
|
482
|
+
|
|
483
|
+
vi.mocked(unzipFile).mockResolvedValue(extractedFiles);
|
|
484
|
+
vi.spyOn(result.current, 'uploadWithProgress').mockResolvedValue({
|
|
485
|
+
id: 'file-1',
|
|
486
|
+
url: 'http://example.com/file-1',
|
|
487
|
+
});
|
|
488
|
+
vi.spyOn(result.current, 'refreshFileList').mockResolvedValue();
|
|
489
|
+
vi.spyOn(result.current, 'parseFilesToChunks').mockResolvedValue();
|
|
490
|
+
const dispatchSpy = vi.spyOn(result.current, 'dispatchDockFileList');
|
|
491
|
+
|
|
492
|
+
await act(async () => {
|
|
493
|
+
await result.current.pushDockFileList([zipFile]);
|
|
494
|
+
});
|
|
495
|
+
|
|
496
|
+
// Should extract ZIP file
|
|
497
|
+
expect(unzipFile).toHaveBeenCalledWith(zipFile);
|
|
498
|
+
|
|
499
|
+
// Should upload extracted files
|
|
500
|
+
expect(dispatchSpy).toHaveBeenCalledWith({
|
|
501
|
+
atStart: true,
|
|
502
|
+
files: extractedFiles.map((file) => ({ file, id: file.name, status: 'pending' })),
|
|
503
|
+
type: 'addFiles',
|
|
504
|
+
});
|
|
505
|
+
});
|
|
506
|
+
|
|
507
|
+
it('should handle ZIP extraction errors gracefully', async () => {
|
|
508
|
+
const { result } = renderHook(() => useStore());
|
|
509
|
+
|
|
510
|
+
const zipFile = new File(['zip content'], 'archive.zip', { type: 'application/zip' });
|
|
511
|
+
|
|
512
|
+
vi.mocked(unzipFile).mockRejectedValue(new Error('Extraction failed'));
|
|
513
|
+
const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
514
|
+
vi.spyOn(result.current, 'uploadWithProgress').mockResolvedValue({
|
|
515
|
+
id: 'file-1',
|
|
516
|
+
url: 'http://example.com/file-1',
|
|
517
|
+
});
|
|
518
|
+
vi.spyOn(result.current, 'refreshFileList').mockResolvedValue();
|
|
519
|
+
vi.spyOn(result.current, 'parseFilesToChunks').mockResolvedValue();
|
|
520
|
+
const dispatchSpy = vi.spyOn(result.current, 'dispatchDockFileList');
|
|
521
|
+
|
|
522
|
+
await act(async () => {
|
|
523
|
+
await result.current.pushDockFileList([zipFile]);
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
// Should log error
|
|
527
|
+
expect(consoleErrorSpy).toHaveBeenCalled();
|
|
528
|
+
|
|
529
|
+
// Should fallback to uploading the ZIP file itself
|
|
530
|
+
expect(dispatchSpy).toHaveBeenCalledWith({
|
|
531
|
+
atStart: true,
|
|
532
|
+
files: [{ file: zipFile, id: zipFile.name, status: 'pending' }],
|
|
533
|
+
type: 'addFiles',
|
|
534
|
+
});
|
|
535
|
+
});
|
|
401
536
|
});
|
|
402
537
|
|
|
403
538
|
describe('reEmbeddingChunks', () => {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import pMap from 'p-map';
|
|
1
2
|
import { SWRResponse, mutate } from 'swr';
|
|
2
3
|
import { StateCreator } from 'zustand/vanilla';
|
|
3
4
|
|
|
4
|
-
import { FILE_UPLOAD_BLACKLIST } from '@/const/file';
|
|
5
|
+
import { FILE_UPLOAD_BLACKLIST, MAX_UPLOAD_FILE_COUNT } from '@/const/file';
|
|
5
6
|
import { useClientDataSWR } from '@/libs/swr';
|
|
6
7
|
import { fileService } from '@/services/file';
|
|
7
8
|
import { ServerService } from '@/services/file/server';
|
|
@@ -12,6 +13,7 @@ import {
|
|
|
12
13
|
} from '@/store/file/reducers/uploadFileList';
|
|
13
14
|
import { FileListItem, QueryFileListParams } from '@/types/files';
|
|
14
15
|
import { isChunkingUnsupported } from '@/utils/isChunkingUnsupported';
|
|
16
|
+
import { unzipFile } from '@/utils/unzipFile';
|
|
15
17
|
|
|
16
18
|
import { FileStore } from '../../store';
|
|
17
19
|
import { fileManagerSelectors } from './selectors';
|
|
@@ -89,18 +91,37 @@ export const createFileManageSlice: StateCreator<
|
|
|
89
91
|
pushDockFileList: async (rawFiles, knowledgeBaseId) => {
|
|
90
92
|
const { dispatchDockFileList } = get();
|
|
91
93
|
|
|
92
|
-
// 0.
|
|
93
|
-
const
|
|
94
|
+
// 0. Process ZIP files and extract their contents
|
|
95
|
+
const filesToUpload: File[] = [];
|
|
96
|
+
for (const file of rawFiles) {
|
|
97
|
+
if (file.type === 'application/zip' || file.name.endsWith('.zip')) {
|
|
98
|
+
try {
|
|
99
|
+
const extractedFiles = await unzipFile(file);
|
|
100
|
+
filesToUpload.push(...extractedFiles);
|
|
101
|
+
} catch (error) {
|
|
102
|
+
console.error('Failed to extract ZIP file:', error);
|
|
103
|
+
// If extraction fails, treat it as a regular file
|
|
104
|
+
filesToUpload.push(file);
|
|
105
|
+
}
|
|
106
|
+
} else {
|
|
107
|
+
filesToUpload.push(file);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
94
110
|
|
|
95
|
-
// 1.
|
|
111
|
+
// 1. skip file in blacklist
|
|
112
|
+
const files = filesToUpload.filter((file) => !FILE_UPLOAD_BLACKLIST.includes(file.name));
|
|
113
|
+
|
|
114
|
+
// 2. Add all files to dock
|
|
96
115
|
dispatchDockFileList({
|
|
97
116
|
atStart: true,
|
|
98
117
|
files: files.map((file) => ({ file, id: file.name, status: 'pending' })),
|
|
99
118
|
type: 'addFiles',
|
|
100
119
|
});
|
|
101
120
|
|
|
102
|
-
|
|
103
|
-
|
|
121
|
+
// 3. Upload files with concurrency limit using p-map
|
|
122
|
+
const uploadResults = await pMap(
|
|
123
|
+
files,
|
|
124
|
+
async (file) => {
|
|
104
125
|
const result = await get().uploadWithProgress({
|
|
105
126
|
file,
|
|
106
127
|
knowledgeBaseId,
|
|
@@ -110,10 +131,11 @@ export const createFileManageSlice: StateCreator<
|
|
|
110
131
|
await get().refreshFileList();
|
|
111
132
|
|
|
112
133
|
return { file, fileId: result?.id, fileType: file.type };
|
|
113
|
-
}
|
|
134
|
+
},
|
|
135
|
+
{ concurrency: MAX_UPLOAD_FILE_COUNT },
|
|
114
136
|
);
|
|
115
137
|
|
|
116
|
-
//
|
|
138
|
+
// 4. auto-embed files that support chunking
|
|
117
139
|
const fileIdsToEmbed = uploadResults
|
|
118
140
|
.filter(({ fileType, fileId }) => fileId && !isChunkingUnsupported(fileType))
|
|
119
141
|
.map(({ fileId }) => fileId!);
|
|
@@ -12,10 +12,10 @@ import Result from './Result';
|
|
|
12
12
|
interface PagesContentProps {
|
|
13
13
|
messageId: string;
|
|
14
14
|
results?: CrawlPluginState['results'];
|
|
15
|
-
urls
|
|
15
|
+
urls?: string[];
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
const PagesContent = memo<PagesContentProps>(({ results, messageId, urls }) => {
|
|
18
|
+
const PagesContent = memo<PagesContentProps>(({ results, messageId, urls = [] }) => {
|
|
19
19
|
const isMobile = useIsMobile();
|
|
20
20
|
|
|
21
21
|
if (!results || results.length === 0) {
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
|
+
BuiltinRenderProps,
|
|
2
3
|
CrawlMultiPagesQuery,
|
|
3
4
|
CrawlPluginState,
|
|
4
5
|
CrawlSinglePageQuery,
|
|
5
6
|
SearchContent,
|
|
6
7
|
SearchQuery,
|
|
7
8
|
UniformSearchResponse,
|
|
8
|
-
|
|
9
|
+
} from '@lobechat/types';
|
|
9
10
|
import { memo } from 'react';
|
|
10
11
|
|
|
11
12
|
import { WebBrowsingApiName } from '@/tools/web-browsing';
|
|
@@ -22,7 +23,7 @@ const WebBrowsing = memo<BuiltinRenderProps<SearchContent[]>>(
|
|
|
22
23
|
<Search
|
|
23
24
|
messageId={messageId}
|
|
24
25
|
pluginError={pluginError}
|
|
25
|
-
searchQuery={args as SearchQuery}
|
|
26
|
+
searchQuery={(args as SearchQuery) || {}}
|
|
26
27
|
searchResponse={pluginState as UniformSearchResponse}
|
|
27
28
|
/>
|
|
28
29
|
);
|
|
@@ -33,7 +34,7 @@ const WebBrowsing = memo<BuiltinRenderProps<SearchContent[]>>(
|
|
|
33
34
|
<PageContent
|
|
34
35
|
messageId={messageId}
|
|
35
36
|
results={(pluginState as CrawlPluginState)?.results}
|
|
36
|
-
urls={[(args as CrawlSinglePageQuery)
|
|
37
|
+
urls={[(args as CrawlSinglePageQuery)?.url]}
|
|
37
38
|
/>
|
|
38
39
|
);
|
|
39
40
|
}
|
|
@@ -43,7 +44,7 @@ const WebBrowsing = memo<BuiltinRenderProps<SearchContent[]>>(
|
|
|
43
44
|
<PageContent
|
|
44
45
|
messageId={messageId}
|
|
45
46
|
results={(pluginState as CrawlPluginState)?.results}
|
|
46
|
-
urls={(args as CrawlMultiPagesQuery)
|
|
47
|
+
urls={(args as CrawlMultiPagesQuery)?.urls}
|
|
47
48
|
/>
|
|
48
49
|
);
|
|
49
50
|
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { zip } from 'fflate';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
|
|
4
|
+
import { unzipFile } from './unzipFile';
|
|
5
|
+
|
|
6
|
+
describe('unzipFile', () => {
|
|
7
|
+
it('should extract files from a ZIP archive', async () => {
|
|
8
|
+
// Create a mock ZIP file with test data
|
|
9
|
+
const testFiles = {
|
|
10
|
+
'test.txt': new TextEncoder().encode('Hello, World!'),
|
|
11
|
+
'folder/nested.txt': new TextEncoder().encode('Nested file content'),
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
const zipped = await new Promise<Uint8Array>((resolve, reject) => {
|
|
15
|
+
zip(testFiles, (error, data) => {
|
|
16
|
+
if (error) reject(error);
|
|
17
|
+
else resolve(data);
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
const zipFile = new File([new Uint8Array(zipped)], 'test.zip', { type: 'application/zip' });
|
|
22
|
+
|
|
23
|
+
const extractedFiles = await unzipFile(zipFile);
|
|
24
|
+
|
|
25
|
+
expect(extractedFiles).toHaveLength(2);
|
|
26
|
+
expect(extractedFiles[0].name).toBe('test.txt');
|
|
27
|
+
expect(extractedFiles[1].name).toBe('nested.txt');
|
|
28
|
+
|
|
29
|
+
// Verify file contents
|
|
30
|
+
const content1 = await extractedFiles[0].text();
|
|
31
|
+
expect(content1).toBe('Hello, World!');
|
|
32
|
+
|
|
33
|
+
const content2 = await extractedFiles[1].text();
|
|
34
|
+
expect(content2).toBe('Nested file content');
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('should skip directories in ZIP archive', async () => {
|
|
38
|
+
const testFiles = {
|
|
39
|
+
'file.txt': new TextEncoder().encode('File content'),
|
|
40
|
+
'folder/': new Uint8Array(0), // Directory entry
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const zipped = await new Promise<Uint8Array>((resolve, reject) => {
|
|
44
|
+
zip(testFiles, (error, data) => {
|
|
45
|
+
if (error) reject(error);
|
|
46
|
+
else resolve(data);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
const zipFile = new File([new Uint8Array(zipped)], 'test.zip', { type: 'application/zip' });
|
|
51
|
+
|
|
52
|
+
const extractedFiles = await unzipFile(zipFile);
|
|
53
|
+
|
|
54
|
+
expect(extractedFiles).toHaveLength(1);
|
|
55
|
+
expect(extractedFiles[0].name).toBe('file.txt');
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it('should skip hidden files and __MACOSX directories', async () => {
|
|
59
|
+
const testFiles = {
|
|
60
|
+
'.hidden': new TextEncoder().encode('Hidden file'),
|
|
61
|
+
'__MACOSX/._file.txt': new TextEncoder().encode('Mac metadata'),
|
|
62
|
+
'visible.txt': new TextEncoder().encode('Visible file'),
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const zipped = await new Promise<Uint8Array>((resolve, reject) => {
|
|
66
|
+
zip(testFiles, (error, data) => {
|
|
67
|
+
if (error) reject(error);
|
|
68
|
+
else resolve(data);
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
const zipFile = new File([new Uint8Array(zipped)], 'test.zip', { type: 'application/zip' });
|
|
73
|
+
|
|
74
|
+
const extractedFiles = await unzipFile(zipFile);
|
|
75
|
+
|
|
76
|
+
expect(extractedFiles).toHaveLength(1);
|
|
77
|
+
expect(extractedFiles[0].name).toBe('visible.txt');
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('should set correct MIME types for extracted files', async () => {
|
|
81
|
+
const testFiles = {
|
|
82
|
+
'document.pdf': new TextEncoder().encode('PDF content'),
|
|
83
|
+
'image.png': new TextEncoder().encode('PNG content'),
|
|
84
|
+
'code.ts': new TextEncoder().encode('TypeScript code'),
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const zipped = await new Promise<Uint8Array>((resolve, reject) => {
|
|
88
|
+
zip(testFiles, (error, data) => {
|
|
89
|
+
if (error) reject(error);
|
|
90
|
+
else resolve(data);
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const zipFile = new File([new Uint8Array(zipped)], 'test.zip', { type: 'application/zip' });
|
|
95
|
+
|
|
96
|
+
const extractedFiles = await unzipFile(zipFile);
|
|
97
|
+
|
|
98
|
+
expect(extractedFiles).toHaveLength(3);
|
|
99
|
+
expect(extractedFiles.find((f) => f.name === 'document.pdf')?.type).toBe('application/pdf');
|
|
100
|
+
expect(extractedFiles.find((f) => f.name === 'image.png')?.type).toBe('image/png');
|
|
101
|
+
expect(extractedFiles.find((f) => f.name === 'code.ts')?.type).toBe('text/typescript');
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it('should handle empty ZIP files', async () => {
|
|
105
|
+
const testFiles = {};
|
|
106
|
+
|
|
107
|
+
const zipped = await new Promise<Uint8Array>((resolve, reject) => {
|
|
108
|
+
zip(testFiles, (error, data) => {
|
|
109
|
+
if (error) reject(error);
|
|
110
|
+
else resolve(data);
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
const zipFile = new File([new Uint8Array(zipped)], 'empty.zip', { type: 'application/zip' });
|
|
115
|
+
|
|
116
|
+
const extractedFiles = await unzipFile(zipFile);
|
|
117
|
+
|
|
118
|
+
expect(extractedFiles).toHaveLength(0);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it('should reject on invalid ZIP file', async () => {
|
|
122
|
+
const invalidFile = new File([new Uint8Array([1, 2, 3, 4])], 'invalid.zip', {
|
|
123
|
+
type: 'application/zip',
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
await expect(unzipFile(invalidFile)).rejects.toThrow();
|
|
127
|
+
});
|
|
128
|
+
});
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { unzip } from 'fflate';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Determines the MIME type based on file extension
|
|
5
|
+
*/
|
|
6
|
+
const getFileType = (fileName: string): string => {
|
|
7
|
+
const extension = fileName.split('.').pop()?.toLowerCase() || '';
|
|
8
|
+
|
|
9
|
+
const mimeTypes: Record<string, string> = {
|
|
10
|
+
// Images
|
|
11
|
+
bmp: 'image/bmp',
|
|
12
|
+
|
|
13
|
+
// Code files
|
|
14
|
+
c: 'text/x-c',
|
|
15
|
+
|
|
16
|
+
cpp: 'text/x-c++',
|
|
17
|
+
|
|
18
|
+
cs: 'text/x-csharp',
|
|
19
|
+
|
|
20
|
+
css: 'text/css',
|
|
21
|
+
|
|
22
|
+
// Documents
|
|
23
|
+
csv: 'text/csv',
|
|
24
|
+
|
|
25
|
+
doc: 'application/msword',
|
|
26
|
+
|
|
27
|
+
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
28
|
+
|
|
29
|
+
gif: 'image/gif',
|
|
30
|
+
|
|
31
|
+
go: 'text/x-go',
|
|
32
|
+
|
|
33
|
+
html: 'text/html',
|
|
34
|
+
|
|
35
|
+
java: 'text/x-java',
|
|
36
|
+
|
|
37
|
+
jpeg: 'image/jpeg',
|
|
38
|
+
|
|
39
|
+
jpg: 'image/jpeg',
|
|
40
|
+
|
|
41
|
+
js: 'text/javascript',
|
|
42
|
+
|
|
43
|
+
json: 'application/json',
|
|
44
|
+
|
|
45
|
+
jsx: 'text/javascript',
|
|
46
|
+
|
|
47
|
+
md: 'text/markdown',
|
|
48
|
+
|
|
49
|
+
pdf: 'application/pdf',
|
|
50
|
+
|
|
51
|
+
php: 'application/x-httpd-php',
|
|
52
|
+
|
|
53
|
+
png: 'image/png',
|
|
54
|
+
|
|
55
|
+
ppt: 'application/vnd.ms-powerpoint',
|
|
56
|
+
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
57
|
+
py: 'text/x-python',
|
|
58
|
+
rb: 'text/x-ruby',
|
|
59
|
+
rs: 'text/x-rust',
|
|
60
|
+
rtf: 'application/rtf',
|
|
61
|
+
sh: 'application/x-sh',
|
|
62
|
+
svg: 'image/svg+xml',
|
|
63
|
+
ts: 'text/typescript',
|
|
64
|
+
tsx: 'text/typescript',
|
|
65
|
+
txt: 'text/plain',
|
|
66
|
+
webp: 'image/webp',
|
|
67
|
+
xls: 'application/vnd.ms-excel',
|
|
68
|
+
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
69
|
+
xml: 'application/xml',
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
return mimeTypes[extension] || 'application/octet-stream';
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Extracts files from a ZIP archive
|
|
77
|
+
* @param zipFile - The ZIP file to extract
|
|
78
|
+
* @returns Promise that resolves to an array of extracted Files
|
|
79
|
+
*/
|
|
80
|
+
export const unzipFile = async (zipFile: File): Promise<File[]> => {
|
|
81
|
+
return new Promise((resolve, reject) => {
|
|
82
|
+
zipFile
|
|
83
|
+
.arrayBuffer()
|
|
84
|
+
.then((arrayBuffer) => {
|
|
85
|
+
const uint8Array = new Uint8Array(arrayBuffer);
|
|
86
|
+
|
|
87
|
+
unzip(uint8Array, (error, unzipped) => {
|
|
88
|
+
if (error) {
|
|
89
|
+
reject(error);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const extractedFiles: File[] = [];
|
|
94
|
+
|
|
95
|
+
for (const [path, data] of Object.entries(unzipped)) {
|
|
96
|
+
// Skip directories and hidden files
|
|
97
|
+
if (path.endsWith('/') || path.includes('__MACOSX') || path.startsWith('.')) {
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Get the filename from the path
|
|
102
|
+
const fileName = path.split('/').pop() || path;
|
|
103
|
+
|
|
104
|
+
// Create a File object from the extracted data
|
|
105
|
+
const blob = new Blob([new Uint8Array(data)], {
|
|
106
|
+
type: getFileType(fileName),
|
|
107
|
+
});
|
|
108
|
+
const file = new File([blob], fileName, {
|
|
109
|
+
type: getFileType(fileName),
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
extractedFiles.push(file);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
resolve(extractedFiles);
|
|
116
|
+
});
|
|
117
|
+
})
|
|
118
|
+
.catch(() => {
|
|
119
|
+
reject(new Error('Failed to read ZIP file'));
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
};
|