@lobehub/chat 1.36.46 → 1.37.0

package/src/features/InitClientDB/InitIndicator.tsx

@@ -0,0 +1,124 @@
+'use client';
+
+import { Progress } from 'antd';
+import { createStyles } from 'antd-style';
+import { AnimatePresence, motion } from 'framer-motion';
+import { rgba } from 'polished';
+import { memo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Center, Flexbox } from 'react-layout-kit';
+
+import { useGlobalStore } from '@/store/global';
+import { ClientDatabaseInitStages, DatabaseLoadingState } from '@/types/clientDB';
+
+import ErrorResult from './ErrorResult';
+
+const useStyles = createStyles(({ css, token, prefixCls }) => ({
+  bg: css`
+    padding-block: 8px;
+    padding-inline: 8px 32px;
+    background: ${token.colorText};
+    border-radius: 40px;
+  `,
+  container: css`
+    position: fixed;
+    z-index: 1000;
+  `,
+  progress: css`
+    .${prefixCls}-progress-text {
+      font-size: 12px;
+      color: ${token.colorBgContainer} !important;
+    }
+  `,
+  progressReady: css`
+    .${prefixCls}-progress-text {
+      color: ${token.colorSuccessBorder} !important;
+    }
+  `,
+
+  text: css`
+    font-size: 15px;
+    color: ${token.colorBgContainer};
+  `,
+}));
+
+interface InitClientDBProps {
+  bottom?: number;
+  show: boolean;
+}
+
+const InitClientDB = memo<InitClientDBProps>(({ bottom = 80, show }) => {
+  const { styles, theme, cx } = useStyles();
+  const currentStage = useGlobalStore((s) => s.initClientDBStage || DatabaseLoadingState.Idle);
+  const { t } = useTranslation('common');
+  const useInitClientDB = useGlobalStore((s) => s.useInitClientDB);
+
+  useInitClientDB();
+
+  const getStateMessage = (state: DatabaseLoadingState) => {
+    switch (state) {
+      case DatabaseLoadingState.Finished:
+      case DatabaseLoadingState.Ready: {
+        return t('clientDB.initing.ready');
+      }
+
+      case DatabaseLoadingState.Idle: {
+        return t('clientDB.initing.idle');
+      }
+      case DatabaseLoadingState.Initializing: {
+        return t('clientDB.initing.initializing');
+      }
+      case DatabaseLoadingState.LoadingDependencies: {
+        return t('clientDB.initing.loadingDependencies');
+      }
+
+      case DatabaseLoadingState.LoadingWasm: {
+        return t('clientDB.initing.loadingWasmModule');
+      }
+
+      case DatabaseLoadingState.Migrating: {
+        return t('clientDB.initing.migrating');
+      }
+    }
+  };
+
+  const currentStageIndex = ClientDatabaseInitStages.indexOf(currentStage);
+  const isReady = currentStage === DatabaseLoadingState.Finished;
+  const isError = currentStage === DatabaseLoadingState.Error;
+  return (
+    <AnimatePresence>
+      {show && (
+        <Center className={styles.container} style={{ bottom }} width={'100%'}>
+          <motion.div
+            animate={{ opacity: 1, y: 0 }}
+            exit={{ opacity: 0, y: 30 }}
+            initial={{ opacity: 0, y: 30 }}
+            transition={{ duration: 0.3 }}
+          >
+            {isError ? (
+              <ErrorResult />
+            ) : (
+              <Flexbox align={'center'} className={styles.bg} gap={12} horizontal>
+                <Progress
+                  className={cx(styles.progress, isReady && styles.progressReady)}
+                  format={isReady ? undefined : (percent) => percent}
+                  percent={parseInt(
+                    ((currentStageIndex / (ClientDatabaseInitStages.length - 1)) * 100).toFixed(0),
+                  )}
+                  size={40}
+                  strokeColor={isReady ? theme.colorSuccessActive : theme.colorBgContainer}
+                  strokeLinecap={'round'}
+                  strokeWidth={10}
+                  trailColor={rgba(theme.colorBgContainer, 0.1)}
+                  type={'circle'}
+                />
+                <span className={styles.text}>{getStateMessage(currentStage)}</span>
+              </Flexbox>
+            )}
+          </motion.div>
+        </Center>
+      )}
+    </AnimatePresence>
+  );
+});
+export default InitClientDB;

package/src/features/InitClientDB/PGliteSVG.tsx

@@ -0,0 +1,22 @@
+import { LucideIcon, LucideProps } from 'lucide-react';
+import { forwardRef } from 'react';
+
+// @ts-expect-error 类型感觉不对，未来修正
+export const PGliteSVG: LucideIcon = forwardRef<SVGElement, Partial<Omit<LucideProps, 'ref'>>>(
+  ({ size }, ref) => (
+    <svg
+      fill="currentColor"
+      height={size}
+      ref={ref as any}
+      viewBox={'0 0 1024 1024'}
+      width={size}
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        clip-rule="evenodd"
+        d="M941.581 335.737v460.806c0 15.926-12.913 28.836-28.832 28.818l-115.283-.137c-15.243-.018-27.706-11.88-28.703-26.877.011-.569.018-1.138.018-1.711l-.004-172.904c0-47.745-38.736-86.451-86.454-86.451-46.245 0-84.052-36.359-86.342-82.068V191.496l201.708.149c79.484.058 143.892 64.553 143.892 144.092Zm-576-144.281v201.818c0 47.746 38.682 86.456 86.4 86.456h86.4v-5.796c0 66.816 54.13 120.98 120.902 120.98 28.617 0 51.815 23.213 51.815 51.848v149.644c0 .688.011 1.372.025 2.057-.943 15.065-13.453 26.992-28.746 26.992l-144.982-.007.986-201.586c.079-15.915-12.755-28.88-28.66-28.959-15.904-.079-28.861 12.763-28.94 28.678l-.986 201.741v.118l-172.174-.01V623.722c0-15.915-12.895-28.819-28.8-28.819-15.906 0-28.8 12.904-28.8 28.819v201.704l-143.642-.007c-15.905-.004-28.798-12.904-28.798-28.819V335.547c0-79.58 64.471-144.093 144.001-144.092l143.999.001Zm446.544 173.693c0-23.874-19.343-43.228-43.2-43.228-23.861 0-43.2 19.354-43.2 43.228 0 23.875 19.339 43.226 43.2 43.226 23.857 0 43.2-19.351 43.2-43.226Z"
+        fill-rule="evenodd"
+      />
+    </svg>
+  ),
+);

package/src/features/InitClientDB/index.tsx

@@ -0,0 +1,37 @@
+'use client';
+
+import { Spin } from 'antd';
+import dynamic from 'next/dynamic';
+import { memo } from 'react';
+
+import { useGlobalStore } from '@/store/global';
+import { systemStatusSelectors } from '@/store/global/selectors';
+
+const Modal = dynamic(() => import('./EnableModal'), {
+  loading: () => <Spin fullscreen />,
+  ssr: false,
+});
+
+const InitIndicator = dynamic(() => import('./InitIndicator'), {
+  ssr: false,
+});
+
+interface InitClientDBProps {
+  bottom?: number;
+}
+
+const InitClientDB = memo<InitClientDBProps>(({ bottom }) => {
+  const isPgliteNotEnabled = useGlobalStore(systemStatusSelectors.isPgliteNotEnabled);
+  const isPgliteNotInited = useGlobalStore(systemStatusSelectors.isPgliteNotInited);
+
+  return (
+    <>
+      {/* 当用户没有设置启用 pglite 时，强弹窗引导用户来开启弹窗 */}
+      {isPgliteNotEnabled && <Modal open={isPgliteNotEnabled} />}
+      {/* 当用户已经启用 pglite 但没有初始化时，展示初始化指示器 */}
+      {isPgliteNotInited && <InitIndicator bottom={bottom} show={isPgliteNotInited} />}
+    </>
+  );
+});
+
+export default InitClientDB;

package/src/hooks/useCheckPluginsIsInstalled.ts

@@ -3,8 +3,8 @@ import { systemStatusSelectors } from '@/store/global/selectors';
 import { useToolStore } from '@/store/tool';
 
 export const useCheckPluginsIsInstalled = (plugins: string[]) => {
-  const isPgliteInited = useGlobalStore(systemStatusSelectors.isPgliteInited);
+  const isDBInited = useGlobalStore(systemStatusSelectors.isDBInited);
   const checkPluginsIsInstalled = useToolStore((s) => s.useCheckPluginsIsInstalled);
 
-  checkPluginsIsInstalled(isPgliteInited, plugins);
+  checkPluginsIsInstalled(isDBInited, plugins);
 };

package/src/hooks/useFetchInstalledPlugins.ts

@@ -3,8 +3,8 @@ import { systemStatusSelectors } from '@/store/global/selectors';
 import { useToolStore } from '@/store/tool';
 
 export const useFetchInstalledPlugins = () => {
-  const isPgliteInited = useGlobalStore(systemStatusSelectors.isPgliteInited);
+  const isDBInited = useGlobalStore(systemStatusSelectors.isDBInited);
   const [useFetchInstalledPlugins] = useToolStore((s) => [s.useFetchInstalledPlugins]);
 
-  return useFetchInstalledPlugins(isPgliteInited);
+  return useFetchInstalledPlugins(isDBInited);
 };

package/src/hooks/useFetchMessages.ts

@@ -4,12 +4,12 @@ import { systemStatusSelectors } from '@/store/global/selectors';
 import { useSessionStore } from '@/store/session';
 
 export const useFetchMessages = () => {
-  const isPgliteInited = useGlobalStore(systemStatusSelectors.isPgliteInited);
+  const isDBInited = useGlobalStore(systemStatusSelectors.isDBInited);
   const [sessionId] = useSessionStore((s) => [s.activeId]);
   const [activeTopicId, useFetchMessages] = useChatStore((s) => [
     s.activeTopicId,
     s.useFetchMessages,
   ]);
 
-  useFetchMessages(isPgliteInited, sessionId, activeTopicId);
+  useFetchMessages(isDBInited, sessionId, activeTopicId);
 };

package/src/hooks/useFetchSessions.ts

@@ -5,9 +5,9 @@ import { useUserStore } from '@/store/user';
 import { authSelectors } from '@/store/user/slices/auth/selectors';
 
 export const useFetchSessions = () => {
-  const isPgliteInited = useGlobalStore(systemStatusSelectors.isPgliteInited);
+  const isDBInited = useGlobalStore(systemStatusSelectors.isDBInited);
   const isLogin = useUserStore(authSelectors.isLogin);
   const useFetchSessions = useSessionStore((s) => s.useFetchSessions);
 
-  useFetchSessions(isPgliteInited, isLogin);
+  useFetchSessions(isDBInited, isLogin);
 };

package/src/hooks/useFetchThreads.ts

@@ -3,9 +3,9 @@ import { useGlobalStore } from '@/store/global';
 import { systemStatusSelectors } from '@/store/global/selectors';
 
 export const useFetchThreads = (activeTopicId?: string) => {
-  const isPgliteInited = useGlobalStore(systemStatusSelectors.isPgliteInited);
+  const isDBInited = useGlobalStore(systemStatusSelectors.isDBInited);
 
   const [useFetchThreads] = useChatStore((s) => [s.useFetchThreads]);
 
-  useFetchThreads(isPgliteInited, activeTopicId);
+  useFetchThreads(isDBInited, activeTopicId);
 };

package/src/hooks/useFetchTopics.ts

@@ -10,8 +10,8 @@ import { useSessionStore } from '@/store/session';
 export const useFetchTopics = () => {
   const [sessionId] = useSessionStore((s) => [s.activeId]);
   const [activeTopicId, useFetchTopics] = useChatStore((s) => [s.activeTopicId, s.useFetchTopics]);
-  const isPgliteInited = useGlobalStore(systemStatusSelectors.isPgliteInited);
+  const isDBInited = useGlobalStore(systemStatusSelectors.isDBInited);
 
-  useFetchTopics(isPgliteInited, sessionId);
+  useFetchTopics(isDBInited, sessionId);
   useFetchThreads(activeTopicId);
 };

package/src/layout/GlobalProvider/StoreInitialization.tsx

@@ -51,8 +51,8 @@ const StoreInitialization = memo(() => {
    * But during initialization, the value of `enableAuth` might be incorrect cause of the async fetch.
    * So we need to use `isSignedIn` only to determine whether request for the default agent config and user state.
    */
-  const isPgliteInited = useGlobalStore(systemStatusSelectors.isPgliteInited);
-  const isLoginOnInit = isPgliteInited && (enableNextAuth ? isSignedIn : isLogin);
+  const isDBInited = useGlobalStore(systemStatusSelectors.isDBInited);
+  const isLoginOnInit = isDBInited && (enableNextAuth ? isSignedIn : isLogin);
 
   // init inbox agent and default agent config
   useInitAgentStore(isLoginOnInit, serverConfig.defaultAgent?.config);

package/src/services/baseClientService/index.ts

@@ -0,0 +1,9 @@
+const FALLBACK_CLIENT_DB_USER_ID = 'DEFAULT_LOBE_CHAT_USER';
+
+export class BaseClientService {
+  protected readonly userId: string;
+
+  constructor(userId?: string) {
+    this.userId = userId || FALLBACK_CLIENT_DB_USER_ID;
+  }
+}

package/src/services/debug.ts

@@ -1,39 +1,37 @@
-import { DEBUG_MODEL } from '@/database/_deprecated/models/__DEBUG';
-
 class DebugService {
   async insertLargeDataToDB() {
-    await DEBUG_MODEL.createRandomData({
-      messageCount: 100_000,
-      sessionCount: 40,
-      startIndex: 0,
-      topicCount: 200,
-    });
-
-    console.log('已插入10w');
-
-    await DEBUG_MODEL.createRandomData({
-      messageCount: 300_000,
-      sessionCount: 40,
-      startIndex: 100_001,
-      topicCount: 200,
-    });
-    console.log('已插入40w');
-
-    await DEBUG_MODEL.createRandomData({
-      messageCount: 300_000,
-      sessionCount: 40,
-      startIndex: 400_001,
-      topicCount: 200,
-    });
-    console.log('已插入70w');
-
-    await DEBUG_MODEL.createRandomData({
-      messageCount: 300_000,
-      sessionCount: 40,
-      startIndex: 700_001,
-      topicCount: 200,
-    });
-    console.log('已插入100w');
+    // await DEBUG_MODEL.createRandomData({
+    //   messageCount: 100_000,
+    //   sessionCount: 40,
+    //   startIndex: 0,
+    //   topicCount: 200,
+    // });
+    //
+    // console.log('已插入10w');
+    //
+    // await DEBUG_MODEL.createRandomData({
+    //   messageCount: 300_000,
+    //   sessionCount: 40,
+    //   startIndex: 100_001,
+    //   topicCount: 200,
+    // });
+    // console.log('已插入40w');
+    //
+    // await DEBUG_MODEL.createRandomData({
+    //   messageCount: 300_000,
+    //   sessionCount: 40,
+    //   startIndex: 400_001,
+    //   topicCount: 200,
+    // });
+    // console.log('已插入70w');
+    //
+    // await DEBUG_MODEL.createRandomData({
+    //   messageCount: 300_000,
+    //   sessionCount: 40,
+    //   startIndex: 700_001,
+    //   topicCount: 200,
+    // });
+    // console.log('已插入100w');
   }
 }
 

package/src/services/file/index.ts

@@ -1,5 +1,9 @@
-import { ClientService } from './client';
+import { ClientService as DeprecatedService } from './client';
+import { ClientService } from './pglite';
 import { ServerService } from './server';
 
+const clientService =
+  process.env.NEXT_PUBLIC_CLIENT_DB === 'pglite' ? new ClientService() : new DeprecatedService();
+
 export const fileService =
-  process.env.NEXT_PUBLIC_SERVICE_MODE === 'server' ? new ServerService() : new ClientService();
+  process.env.NEXT_PUBLIC_SERVICE_MODE === 'server' ? new ServerService() : clientService;

package/src/services/file/pglite.test.ts

@@ -0,0 +1,198 @@
+import { eq } from 'drizzle-orm';
+import { beforeEach, describe, expect, it } from 'vitest';
+
+import { clientDB, initializeDB } from '@/database/client/db';
+import { files, globalFiles, users } from '@/database/schemas';
+import { clientS3Storage } from '@/services/file/ClientS3';
+import { UploadFileParams } from '@/types/files';
+
+import { ClientService } from './pglite';
+
+const userId = 'file-user';
+
+const fileService = new ClientService(userId);
+
+const mockFile = {
+  name: 'mock.png',
+  fileType: 'image/png',
+  size: 1,
+  url: '',
+};
+
+beforeEach(async () => {
+  await initializeDB();
+
+  await clientDB.delete(users);
+  await clientDB.delete(globalFiles);
+  // 创建测试数据
+  await clientDB.transaction(async (tx) => {
+    await tx.insert(users).values({ id: userId });
+  });
+});
+
+describe('FileService', () => {
+  describe('createFile', () => {
+    it('createFile should save the file to the database', async () => {
+      const localFile: UploadFileParams = {
+        name: 'test',
+        fileType: 'image/png',
+        url: '',
+        size: 1,
+        hash: '123',
+      };
+
+      await clientS3Storage.putObject(
+        '123',
+        new File([new ArrayBuffer(1)], 'test.png', { type: 'image/png' }),
+      );
+
+      const result = await fileService.createFile(localFile);
+
+      expect(result).toMatchObject({ url: 'data:image/png;base64,AA==' });
+    });
+
+    it('should throw error when file is not found in storage during base64 conversion', async () => {
+      const localFile: UploadFileParams = {
+        name: 'test',
+        fileType: 'image/png',
+        url: '',
+        size: 1,
+        hash: 'non-existing-hash',
+      };
+
+      // 不调用 clientS3Storage.putObject，模拟文件不存在的情况
+
+      const promise = fileService.createFile(localFile);
+
+      await expect(promise).rejects.toThrow('file not found');
+    });
+  });
+
+  it('removeFile should delete the file from the database', async () => {
+    const fileId = '1';
+    await clientDB.insert(files).values({ id: fileId, userId, ...mockFile });
+
+    await fileService.removeFile(fileId);
+
+    const result = await clientDB.query.files.findFirst({
+      where: eq(files.id, fileId),
+    });
+
+    expect(result).toBeUndefined();
+  });
+
+  describe('getFile', () => {
+    it('should retrieve and convert local file info to FilePreview', async () => {
+      const fileId = 'rwlijweled';
+      const file = {
+        fileType: 'image/png',
+        size: 1,
+        name: 'test.png',
+        url: 'idb://12312/abc.png',
+        hashId: '123tttt',
+      };
+
+      await clientDB.insert(globalFiles).values(file);
+
+      await clientDB.insert(files).values({
+        id: fileId,
+        userId,
+        ...file,
+        createdAt: new Date(1),
+        updatedAt: new Date(2),
+        fileHash: file.hashId,
+      });
+
+      await clientS3Storage.putObject(
+        file.hashId,
+        new File([new ArrayBuffer(1)], file.name, { type: file.fileType }),
+      );
+
+      const result = await fileService.getFile(fileId);
+
+      expect(result).toMatchObject({
+        createdAt: new Date(1),
+        id: 'rwlijweled',
+        size: 1,
+        type: 'image/png',
+        name: 'test.png',
+        updatedAt: new Date(2),
+      });
+    });
+
+    it('should throw an error when the file is not found', async () => {
+      const fileId = 'non-existent';
+
+      const getFilePromise = fileService.getFile(fileId);
+
+      await expect(getFilePromise).rejects.toThrow('file not found');
+    });
+  });
+
+  describe('removeFiles', () => {
+    it('should delete multiple files from the database', async () => {
+      const fileIds = ['1', '2', '3'];
+
+      // 插入测试文件数据
+      await Promise.all(
+        fileIds.map((id) => clientDB.insert(files).values({ id, userId, ...mockFile })),
+      );
+
+      await fileService.removeFiles(fileIds);
+
+      // 验证所有文件都被删除
+      const remainingFiles = await clientDB.query.files.findMany({
+        where: (fields, { inArray }) => inArray(fields.id, fileIds),
+      });
+
+      expect(remainingFiles).toHaveLength(0);
+    });
+  });
+
+  describe('removeAllFiles', () => {
+    it('should clear all files for the user', async () => {
+      // 插入测试文件数据
+      await Promise.all([
+        clientDB.insert(files).values({ id: '1', userId, ...mockFile }),
+        clientDB.insert(files).values({ id: '2', userId, ...mockFile }),
+      ]);
+
+      await fileService.removeAllFiles();
+
+      // 验证用户的所有文件都被删除
+      const remainingFiles = await clientDB.query.files.findMany({
+        where: eq(files.userId, userId),
+      });
+
+      expect(remainingFiles).toHaveLength(0);
+    });
+  });
+
+  describe('checkFileHash', () => {
+    it('should return true if file hash exists', async () => {
+      const hash = 'existing-hash';
+      await clientDB.insert(globalFiles).values({
+        ...mockFile,
+        hashId: hash,
+      });
+      await clientDB.insert(files).values({
+        id: '1',
+        userId,
+        ...mockFile,
+        fileHash: hash,
+      });
+
+      const exists = await fileService.checkFileHash(hash);
+
+      expect(exists).toMatchObject({ isExist: true });
+    });
+
+    it('should return false if file hash does not exist', async () => {
+      const hash = 'non-existing-hash';
+
+      const exists = await fileService.checkFileHash(hash);
+
+      expect(exists).toEqual({ isExist: false });
+    });
+  });
+});

package/src/services/file/pglite.ts

@@ -0,0 +1,84 @@
+import { clientDB } from '@/database/client/db';
+import { FileModel } from '@/database/server/models/file';
+import { BaseClientService } from '@/services/baseClientService';
+import { clientS3Storage } from '@/services/file/ClientS3';
+import { FileItem, UploadFileParams } from '@/types/files';
+
+import { IFileService } from './type';
+
+export class ClientService extends BaseClientService implements IFileService {
+  private get fileModel(): FileModel {
+    return new FileModel(clientDB as any, this.userId);
+  }
+
+  async createFile(file: UploadFileParams) {
+    // save to local storage
+    // we may want to save to a remote server later
+    const res = await this.fileModel.create(
+      {
+        fileHash: file.hash,
+        fileType: file.fileType,
+        knowledgeBaseId: file.knowledgeBaseId,
+        metadata: file.metadata,
+        name: file.name,
+        size: file.size,
+        url: file.url!,
+      },
+      true,
+    );
+
+    // get file to base64 url
+    const base64 = await this.getBase64ByFileHash(file.hash!);
+
+    return {
+      id: res.id,
+      url: `data:${file.fileType};base64,${base64}`,
+    };
+  }
+
+  async getFile(id: string): Promise<FileItem> {
+    const item = await this.fileModel.findById(id);
+    if (!item) {
+      throw new Error('file not found');
+    }
+
+    // arrayBuffer to url
+    const fileItem = await clientS3Storage.getObject(item.fileHash!);
+    if (!fileItem) throw new Error('file not found');
+
+    const url = URL.createObjectURL(fileItem);
+
+    return {
+      createdAt: new Date(item.createdAt),
+      id,
+      name: item.name,
+      size: item.size,
+      type: item.fileType,
+      updatedAt: new Date(item.updatedAt),
+      url,
+    };
+  }
+
+  async removeFile(id: string) {
+    await this.fileModel.delete(id, false);
+  }
+
+  async removeFiles(ids: string[]) {
+    await this.fileModel.deleteMany(ids, false);
+  }
+
+  async removeAllFiles() {
+    return this.fileModel.clear();
+  }
+
+  async checkFileHash(hash: string) {
+    return this.fileModel.checkHash(hash);
+  }
+
+  private async getBase64ByFileHash(hash: string) {
+    const fileItem = await clientS3Storage.getObject(hash);
+    if (!fileItem) throw new Error('file not found');
+
+    return Buffer.from(await fileItem.arrayBuffer()).toString('base64');
+  }
+}

package/src/services/file/type.ts

@@ -1,12 +1,13 @@
-import { FileItem, UploadFileParams } from '@/types/files';
+import { CheckFileHashResult, FileItem, UploadFileParams } from '@/types/files';
 
 export interface IFileService {
+  checkFileHash(hash: string): Promise<CheckFileHashResult>;
   createFile(
     file: UploadFileParams,
     knowledgeBaseId?: string,
   ): Promise<{ id: string; url: string }>;
   getFile(id: string): Promise<FileItem>;
   removeAllFiles(): Promise<any>;
-  removeFile(id: string): Promise<any>;
-  removeFiles(ids: string[]): Promise<any>;
+  removeFile(id: string): Promise<void>;
+  removeFiles(ids: string[]): Promise<void>;
 }