@lobehub/chat 1.36.45 → 1.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/README.ja-JP.md +8 -8
  3. package/README.md +8 -8
  4. package/README.zh-CN.md +8 -8
  5. package/changelog/v1.json +18 -0
  6. package/next.config.mjs +4 -1
  7. package/package.json +5 -3
  8. package/scripts/migrateClientDB/compile-migrations.ts +14 -0
  9. package/src/app/(main)/(mobile)/me/(home)/layout.tsx +2 -0
  10. package/src/app/(main)/chat/_layout/Desktop/index.tsx +3 -2
  11. package/src/app/(main)/chat/_layout/Mobile.tsx +5 -3
  12. package/src/app/(main)/chat/features/Migration/DBReader.ts +290 -0
  13. package/src/app/(main)/chat/features/Migration/UpgradeButton.tsx +4 -8
  14. package/src/app/(main)/chat/features/Migration/index.tsx +26 -15
  15. package/src/app/(main)/settings/_layout/Desktop/index.tsx +2 -0
  16. package/src/app/loading/Client/Content.tsx +11 -1
  17. package/src/app/loading/Client/Error.tsx +27 -0
  18. package/src/app/loading/stage.ts +8 -0
  19. package/src/components/FullscreenLoading/index.tsx +4 -3
  20. package/src/const/version.ts +1 -0
  21. package/src/database/_deprecated/models/file.ts +17 -3
  22. package/src/database/client/db.test.ts +172 -0
  23. package/src/database/client/db.ts +246 -0
  24. package/src/database/client/migrations.json +289 -0
  25. package/src/features/InitClientDB/EnableModal.tsx +111 -0
  26. package/src/features/InitClientDB/ErrorResult.tsx +125 -0
  27. package/src/features/InitClientDB/InitIndicator.tsx +124 -0
  28. package/src/features/InitClientDB/PGliteSVG.tsx +22 -0
  29. package/src/features/InitClientDB/index.tsx +37 -0
  30. package/src/hooks/useCheckPluginsIsInstalled.ts +2 -2
  31. package/src/hooks/useFetchInstalledPlugins.ts +2 -2
  32. package/src/hooks/useFetchMessages.ts +2 -2
  33. package/src/hooks/useFetchSessions.ts +2 -2
  34. package/src/hooks/useFetchThreads.ts +2 -2
  35. package/src/hooks/useFetchTopics.ts +2 -2
  36. package/src/layout/GlobalProvider/StoreInitialization.tsx +2 -2
  37. package/src/server/routers/lambda/file.ts +1 -3
  38. package/src/services/__tests__/upload.test.ts +175 -0
  39. package/src/services/baseClientService/index.ts +9 -0
  40. package/src/services/debug.ts +32 -34
  41. package/src/services/file/ClientS3/index.test.ts +115 -0
  42. package/src/services/file/ClientS3/index.ts +58 -0
  43. package/src/services/file/client.test.ts +9 -4
  44. package/src/services/file/client.ts +36 -8
  45. package/src/services/file/index.ts +6 -2
  46. package/src/services/file/pglite.test.ts +198 -0
  47. package/src/services/file/pglite.ts +84 -0
  48. package/src/services/file/type.ts +4 -3
  49. package/src/services/github.ts +17 -0
  50. package/src/services/import/index.ts +6 -2
  51. package/src/services/import/pglite.test.ts +997 -0
  52. package/src/services/import/pglite.ts +34 -0
  53. package/src/services/message/client.ts +2 -0
  54. package/src/services/message/index.ts +6 -2
  55. package/src/services/message/pglite.test.ts +430 -0
  56. package/src/services/message/pglite.ts +118 -0
  57. package/src/services/message/server.ts +9 -9
  58. package/src/services/message/type.ts +3 -4
  59. package/src/services/plugin/index.ts +6 -2
  60. package/src/services/plugin/pglite.test.ts +175 -0
  61. package/src/services/plugin/pglite.ts +51 -0
  62. package/src/services/session/client.ts +1 -1
  63. package/src/services/session/index.ts +6 -2
  64. package/src/services/session/pglite.test.ts +411 -0
  65. package/src/services/session/pglite.ts +184 -0
  66. package/src/services/session/type.ts +14 -1
  67. package/src/services/topic/index.ts +6 -3
  68. package/src/services/topic/pglite.test.ts +212 -0
  69. package/src/services/topic/pglite.ts +85 -0
  70. package/src/services/upload.ts +8 -16
  71. package/src/services/user/client.test.ts +0 -1
  72. package/src/services/user/index.ts +8 -2
  73. package/src/services/user/pglite.test.ts +98 -0
  74. package/src/services/user/pglite.ts +92 -0
  75. package/src/store/chat/slices/builtinTool/action.test.ts +12 -4
  76. package/src/store/file/slices/upload/action.ts +33 -67
  77. package/src/store/global/actions/clientDb.ts +51 -0
  78. package/src/store/global/initialState.ts +13 -0
  79. package/src/store/global/selectors.ts +24 -3
  80. package/src/store/global/store.ts +3 -1
  81. package/src/store/session/slices/sessionGroup/reducer.test.ts +6 -6
  82. package/src/store/user/slices/common/action.ts +2 -4
  83. package/src/types/clientDB.ts +29 -0
  84. package/src/types/files/upload.ts +8 -2
  85. package/src/types/importer.ts +17 -5
  86. package/src/types/meta.ts +0 -9
  87. package/src/types/session/sessionGroup.ts +3 -3
  88. package/src/services/message/index.test.ts +0 -48
@@ -0,0 +1,58 @@
1
+ import { createStore, del, get, set } from 'idb-keyval';
2
+
3
+ const BROWSER_S3_DB_NAME = 'lobechat-local-s3';
4
+
5
+ export class BrowserS3Storage {
6
+ private store;
7
+
8
+ constructor() {
9
+ // skip server-side rendering
10
+ if (typeof window === 'undefined') return;
11
+
12
+ this.store = createStore(BROWSER_S3_DB_NAME, 'objects');
13
+ }
14
+
15
+ /**
16
+ * 上传文件
17
+ * @param key 文件 hash
18
+ * @param file File 对象
19
+ */
20
+ async putObject(key: string, file: File): Promise<void> {
21
+ try {
22
+ const data = await file.arrayBuffer();
23
+ await set(key, { data, name: file.name, type: file.type }, this.store);
24
+ } catch (e) {
25
+ throw new Error(`Failed to put file ${file.name}: ${(e as Error).message}`);
26
+ }
27
+ }
28
+
29
+ /**
30
+ * 获取文件
31
+ * @param key 文件 hash
32
+ * @returns File 对象
33
+ */
34
+ async getObject(key: string): Promise<File | undefined> {
35
+ try {
36
+ const res = await get<{ data: ArrayBuffer; name: string; type: string }>(key, this.store);
37
+ if (!res) return;
38
+
39
+ return new File([res.data], res!.name, { type: res?.type });
40
+ } catch (e) {
41
+ throw new Error(`Failed to get object (key=${key}): ${(e as Error).message}`);
42
+ }
43
+ }
44
+
45
+ /**
46
+ * 删除文件
47
+ * @param key 文件 hash
48
+ */
49
+ async deleteObject(key: string): Promise<void> {
50
+ try {
51
+ await del(key, this.store);
52
+ } catch (e) {
53
+ throw new Error(`Failed to delete object (key=${key}): ${(e as Error).message}`);
54
+ }
55
+ }
56
+ }
57
+
58
+ export const clientS3Storage = new BrowserS3Storage();
@@ -3,6 +3,7 @@ import { Mock, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
3
3
  import { fileEnv } from '@/config/file';
4
4
  import { FileModel } from '@/database/_deprecated/models/file';
5
5
  import { DB_File } from '@/database/_deprecated/schemas/files';
6
+ import { clientS3Storage } from '@/services/file/ClientS3';
6
7
  import { serverConfigSelectors } from '@/store/serverConfig/selectors';
7
8
  import { createServerConfigStore } from '@/store/serverConfig/store';
8
9
 
@@ -45,19 +46,23 @@ beforeEach(() => {
45
46
 
46
47
  describe('FileService', () => {
47
48
  it('createFile should save the file to the database', async () => {
48
- const localFile: DB_File = {
49
+ const localFile = {
49
50
  name: 'test',
50
- data: new ArrayBuffer(1),
51
51
  fileType: 'image/png',
52
- saveMode: 'local',
52
+ url: 'client-s3://123',
53
53
  size: 1,
54
+ hash: '123',
54
55
  };
55
56
 
57
+ await clientS3Storage.putObject(
58
+ '123',
59
+ new File([new ArrayBuffer(1)], 'test.png', { type: 'image/png' }),
60
+ );
61
+
56
62
  (FileModel.create as Mock).mockResolvedValue(localFile);
57
63
 
58
64
  const result = await fileService.createFile(localFile);
59
65
 
60
- expect(FileModel.create).toHaveBeenCalledWith(localFile);
61
66
  expect(result).toEqual({ url: 'data:image/png;base64,AA==' });
62
67
  });
63
68
 
@@ -1,16 +1,27 @@
1
1
  import { FileModel } from '@/database/_deprecated/models/file';
2
- import { DB_File } from '@/database/_deprecated/schemas/files';
3
- import { FileItem } from '@/types/files';
2
+ import { clientS3Storage } from '@/services/file/ClientS3';
3
+ import { FileItem, UploadFileParams } from '@/types/files';
4
4
 
5
5
  import { IFileService } from './type';
6
6
 
7
7
  export class ClientService implements IFileService {
8
- async createFile(file: DB_File) {
8
+ async createFile(file: UploadFileParams) {
9
9
  // save to local storage
10
10
  // we may want to save to a remote server later
11
- const res = await FileModel.create(file);
12
- // arrayBuffer to url
13
- const base64 = Buffer.from(file.data!).toString('base64');
11
+ const res = await FileModel.create({
12
+ createdAt: Date.now(),
13
+ data: undefined,
14
+ fileHash: file.hash,
15
+ fileType: file.fileType,
16
+ metadata: file.metadata,
17
+ name: file.name,
18
+ saveMode: 'url',
19
+ size: file.size,
20
+ url: file.url,
21
+ } as any);
22
+
23
+ // get file to base64 url
24
+ const base64 = await this.getBase64ByFileHash(file.hash!);
14
25
 
15
26
  return {
16
27
  id: res.id,
@@ -18,14 +29,24 @@ export class ClientService implements IFileService {
18
29
  };
19
30
  }
20
31
 
32
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
33
+ async checkFileHash(_hash: string) {
34
+ return { isExist: false, metadata: {} };
35
+ }
36
+
21
37
  async getFile(id: string): Promise<FileItem> {
22
38
  const item = await FileModel.findById(id);
23
39
  if (!item) {
24
40
  throw new Error('file not found');
25
41
  }
26
42
 
27
- // arrayBuffer to url
28
- const url = URL.createObjectURL(new Blob([item.data!], { type: item.fileType }));
43
+ // arrayBuffer to blob or base64 to blob
44
+ const blob = !!item.data
45
+ ? new Blob([item.data!], { type: item.fileType })
46
+ : // @ts-ignore
47
+ new Blob([Buffer.from(item.base64!, 'base64')], { type: item.fileType });
48
+
49
+ const url = URL.createObjectURL(blob);
29
50
 
30
51
  return {
31
52
  createdAt: new Date(item.createdAt),
@@ -49,4 +70,11 @@ export class ClientService implements IFileService {
49
70
  async removeAllFiles() {
50
71
  return FileModel.clear();
51
72
  }
73
+
74
+ private async getBase64ByFileHash(hash: string) {
75
+ const fileItem = await clientS3Storage.getObject(hash);
76
+ if (!fileItem) throw new Error('file not found');
77
+
78
+ return Buffer.from(await fileItem.arrayBuffer()).toString('base64');
79
+ }
52
80
  }
@@ -1,5 +1,9 @@
1
- import { ClientService } from './client';
1
+ import { ClientService as DeprecatedService } from './client';
2
+ import { ClientService } from './pglite';
2
3
  import { ServerService } from './server';
3
4
 
5
+ const clientService =
6
+ process.env.NEXT_PUBLIC_CLIENT_DB === 'pglite' ? new ClientService() : new DeprecatedService();
7
+
4
8
  export const fileService =
5
- process.env.NEXT_PUBLIC_SERVICE_MODE === 'server' ? new ServerService() : new ClientService();
9
+ process.env.NEXT_PUBLIC_SERVICE_MODE === 'server' ? new ServerService() : clientService;
@@ -0,0 +1,198 @@
1
+ import { eq } from 'drizzle-orm';
2
+ import { beforeEach, describe, expect, it } from 'vitest';
3
+
4
+ import { clientDB, initializeDB } from '@/database/client/db';
5
+ import { files, globalFiles, users } from '@/database/schemas';
6
+ import { clientS3Storage } from '@/services/file/ClientS3';
7
+ import { UploadFileParams } from '@/types/files';
8
+
9
+ import { ClientService } from './pglite';
10
+
11
+ const userId = 'file-user';
12
+
13
+ const fileService = new ClientService(userId);
14
+
15
+ const mockFile = {
16
+ name: 'mock.png',
17
+ fileType: 'image/png',
18
+ size: 1,
19
+ url: '',
20
+ };
21
+
22
+ beforeEach(async () => {
23
+ await initializeDB();
24
+
25
+ await clientDB.delete(users);
26
+ await clientDB.delete(globalFiles);
27
+ // 创建测试数据
28
+ await clientDB.transaction(async (tx) => {
29
+ await tx.insert(users).values({ id: userId });
30
+ });
31
+ });
32
+
33
+ describe('FileService', () => {
34
+ describe('createFile', () => {
35
+ it('createFile should save the file to the database', async () => {
36
+ const localFile: UploadFileParams = {
37
+ name: 'test',
38
+ fileType: 'image/png',
39
+ url: '',
40
+ size: 1,
41
+ hash: '123',
42
+ };
43
+
44
+ await clientS3Storage.putObject(
45
+ '123',
46
+ new File([new ArrayBuffer(1)], 'test.png', { type: 'image/png' }),
47
+ );
48
+
49
+ const result = await fileService.createFile(localFile);
50
+
51
+ expect(result).toMatchObject({ url: 'data:image/png;base64,AA==' });
52
+ });
53
+
54
+ it('should throw error when file is not found in storage during base64 conversion', async () => {
55
+ const localFile: UploadFileParams = {
56
+ name: 'test',
57
+ fileType: 'image/png',
58
+ url: '',
59
+ size: 1,
60
+ hash: 'non-existing-hash',
61
+ };
62
+
63
+ // 不调用 clientS3Storage.putObject,模拟文件不存在的情况
64
+
65
+ const promise = fileService.createFile(localFile);
66
+
67
+ await expect(promise).rejects.toThrow('file not found');
68
+ });
69
+ });
70
+
71
+ it('removeFile should delete the file from the database', async () => {
72
+ const fileId = '1';
73
+ await clientDB.insert(files).values({ id: fileId, userId, ...mockFile });
74
+
75
+ await fileService.removeFile(fileId);
76
+
77
+ const result = await clientDB.query.files.findFirst({
78
+ where: eq(files.id, fileId),
79
+ });
80
+
81
+ expect(result).toBeUndefined();
82
+ });
83
+
84
+ describe('getFile', () => {
85
+ it('should retrieve and convert local file info to FilePreview', async () => {
86
+ const fileId = 'rwlijweled';
87
+ const file = {
88
+ fileType: 'image/png',
89
+ size: 1,
90
+ name: 'test.png',
91
+ url: 'idb://12312/abc.png',
92
+ hashId: '123tttt',
93
+ };
94
+
95
+ await clientDB.insert(globalFiles).values(file);
96
+
97
+ await clientDB.insert(files).values({
98
+ id: fileId,
99
+ userId,
100
+ ...file,
101
+ createdAt: new Date(1),
102
+ updatedAt: new Date(2),
103
+ fileHash: file.hashId,
104
+ });
105
+
106
+ await clientS3Storage.putObject(
107
+ file.hashId,
108
+ new File([new ArrayBuffer(1)], file.name, { type: file.fileType }),
109
+ );
110
+
111
+ const result = await fileService.getFile(fileId);
112
+
113
+ expect(result).toMatchObject({
114
+ createdAt: new Date(1),
115
+ id: 'rwlijweled',
116
+ size: 1,
117
+ type: 'image/png',
118
+ name: 'test.png',
119
+ updatedAt: new Date(2),
120
+ });
121
+ });
122
+
123
+ it('should throw an error when the file is not found', async () => {
124
+ const fileId = 'non-existent';
125
+
126
+ const getFilePromise = fileService.getFile(fileId);
127
+
128
+ await expect(getFilePromise).rejects.toThrow('file not found');
129
+ });
130
+ });
131
+
132
+ describe('removeFiles', () => {
133
+ it('should delete multiple files from the database', async () => {
134
+ const fileIds = ['1', '2', '3'];
135
+
136
+ // 插入测试文件数据
137
+ await Promise.all(
138
+ fileIds.map((id) => clientDB.insert(files).values({ id, userId, ...mockFile })),
139
+ );
140
+
141
+ await fileService.removeFiles(fileIds);
142
+
143
+ // 验证所有文件都被删除
144
+ const remainingFiles = await clientDB.query.files.findMany({
145
+ where: (fields, { inArray }) => inArray(fields.id, fileIds),
146
+ });
147
+
148
+ expect(remainingFiles).toHaveLength(0);
149
+ });
150
+ });
151
+
152
+ describe('removeAllFiles', () => {
153
+ it('should clear all files for the user', async () => {
154
+ // 插入测试文件数据
155
+ await Promise.all([
156
+ clientDB.insert(files).values({ id: '1', userId, ...mockFile }),
157
+ clientDB.insert(files).values({ id: '2', userId, ...mockFile }),
158
+ ]);
159
+
160
+ await fileService.removeAllFiles();
161
+
162
+ // 验证用户的所有文件都被删除
163
+ const remainingFiles = await clientDB.query.files.findMany({
164
+ where: eq(files.userId, userId),
165
+ });
166
+
167
+ expect(remainingFiles).toHaveLength(0);
168
+ });
169
+ });
170
+
171
+ describe('checkFileHash', () => {
172
+ it('should return true if file hash exists', async () => {
173
+ const hash = 'existing-hash';
174
+ await clientDB.insert(globalFiles).values({
175
+ ...mockFile,
176
+ hashId: hash,
177
+ });
178
+ await clientDB.insert(files).values({
179
+ id: '1',
180
+ userId,
181
+ ...mockFile,
182
+ fileHash: hash,
183
+ });
184
+
185
+ const exists = await fileService.checkFileHash(hash);
186
+
187
+ expect(exists).toMatchObject({ isExist: true });
188
+ });
189
+
190
+ it('should return false if file hash does not exist', async () => {
191
+ const hash = 'non-existing-hash';
192
+
193
+ const exists = await fileService.checkFileHash(hash);
194
+
195
+ expect(exists).toEqual({ isExist: false });
196
+ });
197
+ });
198
+ });
@@ -0,0 +1,84 @@
1
+ import { clientDB } from '@/database/client/db';
2
+ import { FileModel } from '@/database/server/models/file';
3
+ import { BaseClientService } from '@/services/baseClientService';
4
+ import { clientS3Storage } from '@/services/file/ClientS3';
5
+ import { FileItem, UploadFileParams } from '@/types/files';
6
+
7
+ import { IFileService } from './type';
8
+
9
+ export class ClientService extends BaseClientService implements IFileService {
10
+ private get fileModel(): FileModel {
11
+ return new FileModel(clientDB as any, this.userId);
12
+ }
13
+
14
+ async createFile(file: UploadFileParams) {
15
+ // save to local storage
16
+ // we may want to save to a remote server later
17
+ const res = await this.fileModel.create(
18
+ {
19
+ fileHash: file.hash,
20
+ fileType: file.fileType,
21
+ knowledgeBaseId: file.knowledgeBaseId,
22
+ metadata: file.metadata,
23
+ name: file.name,
24
+ size: file.size,
25
+ url: file.url!,
26
+ },
27
+ true,
28
+ );
29
+
30
+ // get file to base64 url
31
+ const base64 = await this.getBase64ByFileHash(file.hash!);
32
+
33
+ return {
34
+ id: res.id,
35
+ url: `data:${file.fileType};base64,${base64}`,
36
+ };
37
+ }
38
+
39
+ async getFile(id: string): Promise<FileItem> {
40
+ const item = await this.fileModel.findById(id);
41
+ if (!item) {
42
+ throw new Error('file not found');
43
+ }
44
+
45
+ // arrayBuffer to url
46
+ const fileItem = await clientS3Storage.getObject(item.fileHash!);
47
+ if (!fileItem) throw new Error('file not found');
48
+
49
+ const url = URL.createObjectURL(fileItem);
50
+
51
+ return {
52
+ createdAt: new Date(item.createdAt),
53
+ id,
54
+ name: item.name,
55
+ size: item.size,
56
+ type: item.fileType,
57
+ updatedAt: new Date(item.updatedAt),
58
+ url,
59
+ };
60
+ }
61
+
62
+ async removeFile(id: string) {
63
+ await this.fileModel.delete(id, false);
64
+ }
65
+
66
+ async removeFiles(ids: string[]) {
67
+ await this.fileModel.deleteMany(ids, false);
68
+ }
69
+
70
+ async removeAllFiles() {
71
+ return this.fileModel.clear();
72
+ }
73
+
74
+ async checkFileHash(hash: string) {
75
+ return this.fileModel.checkHash(hash);
76
+ }
77
+
78
+ private async getBase64ByFileHash(hash: string) {
79
+ const fileItem = await clientS3Storage.getObject(hash);
80
+ if (!fileItem) throw new Error('file not found');
81
+
82
+ return Buffer.from(await fileItem.arrayBuffer()).toString('base64');
83
+ }
84
+ }
@@ -1,12 +1,13 @@
1
- import { FileItem, UploadFileParams } from '@/types/files';
1
+ import { CheckFileHashResult, FileItem, UploadFileParams } from '@/types/files';
2
2
 
3
3
  export interface IFileService {
4
+ checkFileHash(hash: string): Promise<CheckFileHashResult>;
4
5
  createFile(
5
6
  file: UploadFileParams,
6
7
  knowledgeBaseId?: string,
7
8
  ): Promise<{ id: string; url: string }>;
8
9
  getFile(id: string): Promise<FileItem>;
9
10
  removeAllFiles(): Promise<any>;
10
- removeFile(id: string): Promise<any>;
11
- removeFiles(ids: string[]): Promise<any>;
11
+ removeFile(id: string): Promise<void>;
12
+ removeFiles(ids: string[]): Promise<void>;
12
13
  }
@@ -37,6 +37,23 @@ class GitHubService {
37
37
 
38
38
  window.open(url, '_blank');
39
39
  };
40
+
41
+ submitPgliteInitError = (error?: { message: string }) => {
42
+ const body = ['```json', JSON.stringify(error, null, 2), '```'].join('\n');
43
+
44
+ const message = error?.message || '';
45
+
46
+ const url = qs.stringifyUrl({
47
+ query: {
48
+ body,
49
+ labels: '❌ Database Init Error',
50
+ title: `[Database Init Error] ${message}`,
51
+ },
52
+ url: urlJoin(GITHUB, '/issues/new'),
53
+ });
54
+
55
+ window.open(url, '_blank');
56
+ };
40
57
  }
41
58
 
42
59
  export const githubService = new GitHubService();
@@ -1,5 +1,9 @@
1
- import { ClientService } from './client';
1
+ import { ClientService as DeprecatedService } from './client';
2
+ import { ClientService } from './pglite';
2
3
  import { ServerService } from './server';
3
4
 
5
+ const clientService =
6
+ process.env.NEXT_PUBLIC_CLIENT_DB === 'pglite' ? new ClientService() : new DeprecatedService();
7
+
4
8
  export const importService =
5
- process.env.NEXT_PUBLIC_SERVICE_MODE === 'server' ? new ServerService() : new ClientService();
9
+ process.env.NEXT_PUBLIC_SERVICE_MODE === 'server' ? new ServerService() : clientService;