@lobehub/chat 1.36.45 → 1.37.0

package/src/services/user/pglite.ts

@@ -0,0 +1,92 @@
+import { DeepPartial } from 'utility-types';
+
+import { clientDB } from '@/database/client/db';
+import { users } from '@/database/schemas';
+import { MessageModel } from '@/database/server/models/message';
+import { SessionModel } from '@/database/server/models/session';
+import { UserModel } from '@/database/server/models/user';
+import { BaseClientService } from '@/services/baseClientService';
+import { UserGuide, UserInitializationState, UserPreference } from '@/types/user';
+import { UserSettings } from '@/types/user/settings';
+import { AsyncLocalStorage } from '@/utils/localStorage';
+
+import { IUserService } from './type';
+
+export class ClientService extends BaseClientService implements IUserService {
+  private preferenceStorage: AsyncLocalStorage<UserPreference>;
+
+  private get userModel(): UserModel {
+    return new UserModel(clientDB as any, this.userId);
+  }
+  private get messageModel(): MessageModel {
+    return new MessageModel(clientDB as any, this.userId);
+  }
+  private get sessionModel(): SessionModel {
+    return new SessionModel(clientDB as any, this.userId);
+  }
+
+  constructor(userId?: string) {
+    super(userId);
+    this.preferenceStorage = new AsyncLocalStorage('LOBE_PREFERENCE');
+  }
+
+  async getUserState(): Promise<UserInitializationState> {
+    // if user not exist in the db, create one to make sure the user exist
+    await this.makeSureUserExist();
+
+    const state = await this.userModel.getUserState((encryptKeyVaultsStr) =>
+      encryptKeyVaultsStr ? JSON.parse(encryptKeyVaultsStr) : {},
+    );
+
+    const user = await UserModel.findById(clientDB as any, this.userId);
+    const messageCount = await this.messageModel.count();
+    const sessionCount = await this.sessionModel.count();
+
+    return {
+      ...state,
+      avatar: user?.avatar as string,
+      canEnablePWAGuide: messageCount >= 4,
+      canEnableTrace: messageCount >= 4,
+      hasConversation: messageCount > 0 || sessionCount > 0,
+      isOnboard: true,
+      preference: await this.preferenceStorage.getFromLocalStorage(),
+    };
+  }
+
+  updateUserSettings = async (value: DeepPartial<UserSettings>) => {
+    const { keyVaults, ...res } = value;
+
+    return this.userModel.updateSetting({ ...res, keyVaults: JSON.stringify(keyVaults) });
+  };
+
+  resetUserSettings = async () => {
+    return this.userModel.deleteSetting();
+  };
+
+  async updateAvatar(avatar: string) {
+    await this.userModel.updateUser({ avatar });
+  }
+
+  async updatePreference(preference: Partial<UserPreference>) {
+    await this.preferenceStorage.saveToLocalStorage(preference);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars,unused-imports/no-unused-vars
+  async updateGuide(guide: Partial<UserGuide>) {
+    throw new Error('Method not implemented.');
+  }
+
+  async makeSureUserExist() {
+    const existUsers = await clientDB.query.users.findMany();
+
+    let user: { id: string };
+    if (existUsers.length === 0) {
+      const result = await clientDB.insert(users).values({ id: this.userId }).returning();
+      user = result[0];
+    } else {
+      user = existUsers[0];
+    }
+
+    return user;
+  }
+}

package/src/store/chat/slices/builtinTool/action.test.ts

@@ -2,6 +2,8 @@ import { act, renderHook } from '@testing-library/react';
 import { describe, expect, it, vi } from 'vitest';
 
 import { fileService } from '@/services/file';
+import { ClientService } from '@/services/file/client';
+import { messageService } from '@/services/message';
 import { imageGenerationService } from '@/services/textToImage';
 import { uploadService } from '@/services/upload';
 import { chatSelectors } from '@/store/chat/selectors';
@@ -39,17 +41,22 @@ describe('chatToolSlice', () => {
       vi.spyOn(uploadService, 'getImageFileByUrlWithCORS').mockResolvedValue(
         new File(['1'], 'file.png', { type: 'image/png' }),
       );
-      vi.spyOn(uploadService, 'uploadToClientDB').mockResolvedValue({} as any);
-      vi.spyOn(fileService, 'createFile').mockResolvedValue({ id: mockId, url: '' });
+      vi.spyOn(uploadService, 'uploadToClientS3').mockResolvedValue({} as any);
+      vi.spyOn(ClientService.prototype, 'createFile').mockResolvedValue({
+        id: mockId,
+        url: '',
+      });
       vi.spyOn(result.current, 'toggleDallEImageLoading');
+      vi.spyOn(ClientService.prototype, 'checkFileHash').mockImplementation(
+        async () => ({ isExist: false }) as any,
+      );
 
       await act(async () => {
         await result.current.generateImageFromPrompts(prompts, messageId);
       });
       // For each prompt, loading is toggled on and then off
       expect(imageGenerationService.generateImage).toHaveBeenCalledTimes(prompts.length);
-      expect(uploadService.uploadToClientDB).toHaveBeenCalledTimes(prompts.length);
-
+      expect(uploadService.uploadToClientS3).toHaveBeenCalledTimes(prompts.length);
       expect(result.current.toggleDallEImageLoading).toHaveBeenCalledTimes(prompts.length * 2);
     });
   });
@@ -75,6 +82,7 @@ describe('chatToolSlice', () => {
             content: initialMessageContent,
           }) as ChatMessage,
       );
+      vi.spyOn(messageService, 'updateMessage').mockResolvedValueOnce(undefined);
 
       await act(async () => {
         await result.current.updateImageItem(messageId, updateFunction);

package/src/store/file/slices/upload/action.ts

@@ -6,14 +6,11 @@ import { message } from '@/components/AntdStaticMethods';
 import { LOBE_CHAT_CLOUD } from '@/const/branding';
 import { isServerMode } from '@/const/version';
 import { fileService } from '@/services/file';
-import { ServerService } from '@/services/file/server';
 import { uploadService } from '@/services/upload';
 import { FileMetadata, UploadFileItem } from '@/types/files';
 
 import { FileStore } from '../../store';
 
-const serverFileService = new ServerService();
-
 interface UploadWithProgressParams {
   file: File;
   knowledgeBaseId?: string;
@@ -43,10 +40,6 @@ interface UploadWithProgressResult {
 }
 
 export interface FileUploadAction {
-  internal_uploadToClientDB: (
-    params: Omit<UploadWithProgressParams, 'knowledgeBaseId'>,
-  ) => Promise<UploadWithProgressResult | undefined>;
-  internal_uploadToServer: (params: UploadWithProgressParams) => Promise<UploadWithProgressResult>;
   uploadWithProgress: (
     params: UploadWithProgressParams,
   ) => Promise<UploadWithProgressResult | undefined>;
@@ -57,51 +50,14 @@ export const createFileUploadSlice: StateCreator<
   [['zustand/devtools', never]],
   [],
   FileUploadAction
-> = (set, get) => ({
-  internal_uploadToClientDB: async ({ file, onStatusUpdate, skipCheckFileType }) => {
-    if (!skipCheckFileType && !file.type.startsWith('image')) {
-      onStatusUpdate?.({ id: file.name, type: 'removeFile' });
-      message.info({
-        content: t('upload.fileOnlySupportInServerMode', {
-          cloud: LOBE_CHAT_CLOUD,
-          ext: file.name.split('.').pop(),
-          ns: 'error',
-        }),
-        duration: 5,
-      });
-      return;
-    }
-
-    const fileArrayBuffer = await file.arrayBuffer();
-
-    const hash = sha256(fileArrayBuffer);
-
-    const data = await uploadService.uploadToClientDB(
-      { fileType: file.type, hash, name: file.name, saveMode: 'local', size: file.size },
-      file,
-    );
-
-    onStatusUpdate?.({
-      id: file.name,
-      type: 'updateFile',
-      value: {
-        fileUrl: data.url,
-        id: data.id,
-        status: 'success',
-        uploadState: { progress: 100, restTime: 0, speed: 0 },
-      },
-    });
-
-    return data;
-  },
-
-  internal_uploadToServer: async ({ file, onStatusUpdate, knowledgeBaseId }) => {
+> = () => ({
+  uploadWithProgress: async ({ file, onStatusUpdate, knowledgeBaseId, skipCheckFileType }) => {
     const fileArrayBuffer = await file.arrayBuffer();
 
     // 1. check file hash
     const hash = sha256(fileArrayBuffer);
 
-    const checkStatus = await serverFileService.checkFileHash(hash);
+    const checkStatus = await fileService.checkFileHash(hash);
     let metadata: FileMetadata;
 
     // 2. if file exist, just skip upload
@@ -112,17 +68,37 @@ export const createFileUploadSlice: StateCreator<
         type: 'updateFile',
         value: { status: 'processing', uploadState: { progress: 100, restTime: 0, speed: 0 } },
       });
-    } else {
-      // 2. if file don't exist, need upload files
-      metadata = await uploadService.uploadWithProgress(file, {
-        onProgress: (status, upload) => {
-          onStatusUpdate?.({
-            id: file.name,
-            type: 'updateFile',
-            value: { status: status === 'success' ? 'processing' : status, uploadState: upload },
+    }
+    // 2. if file don't exist, need upload files
+    else {
+      // if is server mode, upload to server s3, or upload to client s3
+      if (isServerMode) {
+        metadata = await uploadService.uploadWithProgress(file, {
+          onProgress: (status, upload) => {
+            onStatusUpdate?.({
+              id: file.name,
+              type: 'updateFile',
+              value: { status: status === 'success' ? 'processing' : status, uploadState: upload },
+            });
+          },
+        });
+      } else {
+        if (!skipCheckFileType && !file.type.startsWith('image')) {
+          onStatusUpdate?.({ id: file.name, type: 'removeFile' });
+          message.info({
+            content: t('upload.fileOnlySupportInServerMode', {
+              cloud: LOBE_CHAT_CLOUD,
+              ext: file.name.split('.').pop(),
+              ns: 'error',
+            }),
+            duration: 5,
           });
-        },
-      });
+          return;
+        }
+
+        // Upload to the indexeddb in the browser
+        metadata = await uploadService.uploadToClientS3(hash, file);
+      }
     }
 
     // 3. use more powerful file type detector to get file type
@@ -138,12 +114,10 @@ export const createFileUploadSlice: StateCreator<
     // 4. create file to db
     const data = await fileService.createFile(
       {
-        createdAt: Date.now(),
         fileType,
         hash,
         metadata,
         name: file.name,
-        saveMode: 'url',
         size: file.size,
         url: metadata.path,
       },
@@ -163,12 +137,4 @@ export const createFileUploadSlice: StateCreator<
 
     return data;
   },
-
-  uploadWithProgress: async (payload) => {
-    const { internal_uploadToServer, internal_uploadToClientDB } = get();
-
-    if (isServerMode) return internal_uploadToServer(payload);
-
-    return internal_uploadToClientDB(payload);
-  },
 });

package/src/store/global/actions/clientDb.ts

@@ -0,0 +1,51 @@
+import { SWRResponse } from 'swr';
+import type { StateCreator } from 'zustand/vanilla';
+
+import { useOnlyFetchOnceSWR } from '@/libs/swr';
+import type { GlobalStore } from '@/store/global';
+import { DatabaseLoadingState, OnStageChange } from '@/types/clientDB';
+
+type InitClientDBParams = { onStateChange: OnStageChange };
+/**
+ * 设置操作
+ */
+export interface GlobalClientDBAction {
+  initializeClientDB: (params?: InitClientDBParams) => Promise<void>;
+  markPgliteEnabled: () => void;
+  useInitClientDB: (params?: InitClientDBParams) => SWRResponse;
+}
+
+export const clientDBSlice: StateCreator<
+  GlobalStore,
+  [['zustand/devtools', never]],
+  [],
+  GlobalClientDBAction
+> = (set, get) => ({
+  initializeClientDB: async (params) => {
+    // if the db has started initialized or not error, just skip.
+    if (
+      get().initClientDBStage !== DatabaseLoadingState.Idle &&
+      get().initClientDBStage !== DatabaseLoadingState.Error
+    )
+      return;
+
+    const { initializeDB } = await import('@/database/client/db');
+    await initializeDB({
+      onError: (error) => {
+        set({ initClientDBError: error });
+      },
+      onProgress: (data) => {
+        set({ initClientDBProcess: data });
+      },
+      onStateChange: (state) => {
+        set({ initClientDBStage: state });
+        params?.onStateChange?.(state);
+      },
+    });
+  },
+  markPgliteEnabled: () => {
+    get().updateSystemStatus({ isEnablePglite: true });
+  },
+  useInitClientDB: (params) =>
+    useOnlyFetchOnceSWR('initClientDB', () => get().initializeClientDB(params)),
+});

package/src/store/global/initialState.ts

@@ -1,5 +1,6 @@
 import { AppRouterInstance } from 'next/dist/shared/lib/app-router-context.shared-runtime';
 
+import { DatabaseLoadingState } from '@/types/clientDB';
 import { SessionDefaultGroup } from '@/types/session';
 import { AsyncLocalStorage } from '@/utils/localStorage';
 
@@ -37,6 +38,10 @@ export interface SystemStatus {
   hidePWAInstaller?: boolean;
   hideThreadLimitAlert?: boolean;
   inputHeight: number;
+  /**
+   * 应用初始化时不启用 PGLite，只有当用户手动开启时才启用
+   */
+  isEnablePglite?: boolean;
   mobileShowPortal?: boolean;
   mobileShowTopic?: boolean;
   sessionsWidth: number;
@@ -50,6 +55,13 @@ export interface SystemStatus {
 
 export interface GlobalState {
   hasNewVersion?: boolean;
+  initClientDBError?: Error;
+  initClientDBProcess?: { costTime?: number; phase: 'wasm' | 'dependencies'; progress: number };
+  /**
+   * 客户端数据库初始化状态
+   * 启动时为 Idle，完成为 Ready，报错为 Error
+   */
+  initClientDBStage: DatabaseLoadingState;
   isMobile?: boolean;
   isStatusInit?: boolean;
   latestVersion?: string;
@@ -76,6 +88,7 @@ export const INITIAL_STATUS = {
 } satisfies SystemStatus;
 
 export const initialState: GlobalState = {
+  initClientDBStage: DatabaseLoadingState.Idle,
   isMobile: false,
   isStatusInit: false,
   sidebarKey: SidebarTabKey.Chat,

package/src/store/global/selectors.ts

@@ -1,4 +1,6 @@
+import { isServerMode, isUsePgliteDB } from '@/const/version';
 import { GlobalStore } from '@/store/global';
+import { DatabaseLoadingState } from '@/types/clientDB';
 
 import { INITIAL_STATUS } from './initialState';
 
@@ -22,17 +24,36 @@ const filePanelWidth = (s: GlobalStore) => s.status.filePanelWidth;
 const inputHeight = (s: GlobalStore) => s.status.inputHeight;
 const threadInputHeight = (s: GlobalStore) => s.status.threadInputHeight;
 
-const isPgliteNotEnabled = () => false;
+const isPgliteNotEnabled = (s: GlobalStore) =>
+  isUsePgliteDB && !isServerMode && s.isStatusInit && !s.status.isEnablePglite;
 
-const isPgliteNotInited = () => false;
+/**
+ * 当且仅当 client db 模式，且 pglite 未初始化完成时返回 true
+ */
+const isPgliteNotInited = (s: GlobalStore) =>
+  isUsePgliteDB &&
+  s.isStatusInit &&
+  s.status.isEnablePglite &&
+  s.initClientDBStage !== DatabaseLoadingState.Ready;
 
-const isPgliteInited = (): boolean => true;
+/**
+ * 当且仅当 client db 模式，且 pglite 初始化完成时返回 true
+ */
+const isPgliteInited = (s: GlobalStore): boolean =>
+  (s.isStatusInit &&
+    s.status.isEnablePglite &&
+    s.initClientDBStage === DatabaseLoadingState.Ready) ||
+  false;
+
+// 这个变量控制 clientdb 是否完成初始化，正常来说，只有 pgliteDB 模式下，才会存在变化，其他时候都是 true
+const isDBInited = (s: GlobalStore): boolean => (isUsePgliteDB ? isPgliteInited(s) : true);
 
 export const systemStatusSelectors = {
   filePanelWidth,
   hidePWAInstaller,
   inZenMode,
   inputHeight,
+  isDBInited,
   isPgliteInited,
   isPgliteNotEnabled,
   isPgliteNotInited,

package/src/store/global/store.ts

@@ -5,15 +5,17 @@ import { StateCreator } from 'zustand/vanilla';
 
 import { createDevtools } from '../middleware/createDevtools';
 import { type GlobalStoreAction, globalActionSlice } from './action';
+import { type GlobalClientDBAction, clientDBSlice } from './actions/clientDb';
 import { type GlobalState, initialState } from './initialState';
 
 //  ===============  聚合 createStoreFn ============ //
 
-export type GlobalStore = GlobalState & GlobalStoreAction;
+export type GlobalStore = GlobalState & GlobalStoreAction & GlobalClientDBAction;
 
 const createStore: StateCreator<GlobalStore, [['zustand/devtools', never]]> = (...parameters) => ({
   ...initialState,
   ...globalActionSlice(...parameters),
+  ...clientDBSlice(...parameters),
 });
 
 //  ===============  实装 useStore ============ //

package/src/store/session/slices/sessionGroup/reducer.test.ts

@@ -10,14 +10,14 @@ describe('sessionGroupsReducer', () => {
     {
       id: nanoid(),
       name: 'Group 1',
-      createdAt: Date.now(),
-      updatedAt: Date.now(),
+      createdAt: new Date(),
+      updatedAt: new Date(),
     },
     {
       id: nanoid(),
       name: 'Group 2',
-      createdAt: Date.now(),
-      updatedAt: Date.now(),
+      createdAt: new Date(),
+      updatedAt: new Date(),
       sort: 1,
     },
   ];
@@ -26,8 +26,8 @@ describe('sessionGroupsReducer', () => {
     const newItem: SessionGroupItem = {
       id: nanoid(),
       name: 'New Group',
-      createdAt: Date.now(),
-      updatedAt: Date.now(),
+      createdAt: new Date(),
+      updatedAt: new Date(),
     };
 
     const result = sessionGroupsReducer(initialState, {

package/src/store/user/slices/common/action.ts

@@ -46,11 +46,9 @@ export const createCommonSlice: StateCreator<
     await mutate(GET_USER_STATE_KEY);
   },
   updateAvatar: async (avatar) => {
-    const { ClientService } = await import('@/services/user/client');
+    const { userClientService } = await import('@/services/user');
 
-    const clientService = new ClientService();
-
-    await clientService.updateAvatar(avatar);
+    await userClientService.updateAvatar(avatar);
     await get().refreshUserState();
   },
 

package/src/types/clientDB.ts

@@ -0,0 +1,29 @@
+// 定义加载状态类型
+export enum DatabaseLoadingState {
+  Error = 'error',
+  Finished = 'finished',
+  Idle = 'idle',
+  Initializing = 'initializing',
+  LoadingDependencies = 'loadingDependencies',
+  LoadingWasm = 'loadingWasm',
+  Migrating = 'migrating',
+  Ready = 'ready',
+}
+
+export const ClientDatabaseInitStages = [
+  DatabaseLoadingState.Idle,
+  DatabaseLoadingState.Initializing,
+  DatabaseLoadingState.LoadingDependencies,
+  DatabaseLoadingState.LoadingWasm,
+  DatabaseLoadingState.Migrating,
+  DatabaseLoadingState.Finished,
+];
+
+// 定义进度回调接口
+export interface ClientDBLoadingProgress {
+  costTime?: number;
+  phase: 'wasm' | 'dependencies';
+  progress: number;
+}
+
+export type OnStageChange = (state: DatabaseLoadingState) => void;

package/src/types/files/upload.ts

@@ -53,7 +53,6 @@ export const FileMetadataSchema = z.object({
 export type FileMetadata = z.infer<typeof FileMetadataSchema>;
 
 export const UploadFileSchema = z.object({
-  data: z.instanceof(ArrayBuffer).optional(),
   /**
    * file type
    * @example 'image/png'
@@ -77,7 +76,6 @@ export const UploadFileSchema = z.object({
    * local mean save the raw file into data
    * url mean upload the file to a cdn and then save the url
    */
-  saveMode: z.enum(['local', 'url']),
   /**
    * file size
    */
@@ -89,3 +87,11 @@ export const UploadFileSchema = z.object({
 });
 
 export type UploadFileParams = z.infer<typeof UploadFileSchema>;
+
+export interface CheckFileHashResult {
+  fileType?: string;
+  isExist: boolean;
+  metadata?: unknown;
+  size?: number;
+  url?: string;
+}

package/src/types/importer.ts

@@ -9,9 +9,8 @@ import {
 } from '@/types/message';
 import { MetaData } from '@/types/meta';
 import { SessionGroupId } from '@/types/session';
-import { ChatTopic } from '@/types/topic';
 
-interface ImportSession {
+export interface ImportSession {
   config: LobeAgentConfig;
   createdAt: string;
   group?: SessionGroupId;
@@ -22,7 +21,7 @@ interface ImportSession {
   updatedAt: string;
 }
 
-interface ImportMessage {
+export interface ImportMessage {
   content: string;
   createdAt: number;
   error?: ChatMessageError;
@@ -64,19 +63,32 @@ interface ImportMessage {
   updatedAt: number;
 }
 
-interface ImportSessionGroup {
+export interface ImportSessionGroup {
   createdAt: number;
   id: string;
   name: string;
   sort?: number | null;
   updatedAt: number;
 }
+export interface ImportTopic {
+  createdAt: number;
+  favorite?: boolean;
+  historySummary?: string;
+  id: string;
+  metadata?: {
+    model?: string;
+    provider?: string;
+  };
+  sessionId?: string;
+  title: string;
+  updatedAt: number;
+}
 
 export interface ImporterEntryData {
   messages?: ImportMessage[];
   sessionGroups?: ImportSessionGroup[];
   sessions?: ImportSession[];
-  topics?: ChatTopic[];
+  topics?: ImportTopic[];
   version: number;
 }
 

package/src/types/meta.ts

@@ -21,19 +21,10 @@ export const LobeMetaDataSchema = z.object({
 export type MetaData = z.infer<typeof LobeMetaDataSchema>;
 
 export interface BaseDataModel {
-  /**
-   * @deprecated
-   */
-  createAt?: number;
-
   createdAt: number;
 
   id: string;
   meta: MetaData;
 
-  /**
-   * @deprecated
-   */
-  updateAt?: number;
   updatedAt: number;
 }

package/src/types/session/sessionGroup.ts

@@ -8,11 +8,11 @@ export enum SessionDefaultGroup {
 export type SessionGroupId = SessionDefaultGroup | string;
 
 export interface SessionGroupItem {
-  createdAt: number;
+  createdAt: Date;
   id: string;
   name: string;
-  sort?: number;
-  updatedAt: number;
+  sort?: number | null;
+  updatedAt: Date;
 }
 
 export type SessionGroups = SessionGroupItem[];