@lobehub/lobehub 2.0.0-next.4 → 2.0.0-next.41

package/src/services/file/_deprecated.test.ts

@@ -1,119 +0,0 @@
-import { Mock, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
-
-import { FileModel } from '@/database/_deprecated/models/file';
-import { DB_File } from '@/database/_deprecated/schemas/files';
-import { fileEnv } from '@/envs/file';
-import { clientS3Storage } from '@/services/file/ClientS3';
-import { serverConfigSelectors } from '@/store/serverConfig/selectors';
-import { createServerConfigStore } from '@/store/serverConfig/store';
-
-import { ClientService } from './_deprecated';
-
-const fileService = new ClientService();
-
-beforeAll(() => {
-  createServerConfigStore();
-});
-// Mocks for the FileModel
-vi.mock('@/database/_deprecated/models/file', () => ({
-  FileModel: {
-    create: vi.fn(),
-    delete: vi.fn(),
-    findById: vi.fn(),
-    clear: vi.fn(),
-  },
-}));
-
-let s3Domain: string;
-
-vi.mock('@/config/file', () => ({
-  fileEnv: {
-    get NEXT_PUBLIC_S3_DOMAIN() {
-      return s3Domain;
-    },
-  },
-}));
-
-// Mocks for the URL and Blob objects
-global.URL.createObjectURL = vi.fn();
-global.Blob = vi.fn();
-
-beforeEach(() => {
-  // Reset all mocks before each test
-  vi.resetAllMocks();
-  s3Domain = '';
-});
-
-describe('FileService', () => {
-  it('createFile should save the file to the database', async () => {
-    const localFile = {
-      name: 'test',
-      fileType: 'image/png',
-      url: 'client-s3://123',
-      size: 1,
-      hash: '123',
-    };
-
-    await clientS3Storage.putObject(
-      '123',
-      new File([new ArrayBuffer(1)], 'test.png', { type: 'image/png' }),
-    );
-
-    (FileModel.create as Mock).mockResolvedValue(localFile);
-
-    const result = await fileService.createFile(localFile);
-
-    expect(result).toEqual({ url: 'data:image/png;base64,AA==' });
-  });
-
-  it('removeFile should delete the file from the database', async () => {
-    const fileId = '1';
-    (FileModel.delete as Mock).mockResolvedValue(true);
-
-    const result = await fileService.removeFile(fileId);
-
-    expect(FileModel.delete).toHaveBeenCalledWith(fileId);
-    expect(result).toBe(true);
-  });
-
-  describe('getFile', () => {
-    it('should retrieve and convert local file info to FilePreview', async () => {
-      const fileId = '1';
-      const fileData = {
-        name: 'test',
-        data: new ArrayBuffer(1),
-        fileType: 'image/png',
-        saveMode: 'local',
-        size: 1,
-        createdAt: 1,
-        updatedAt: 2,
-      } as DB_File;
-
-      (FileModel.findById as Mock).mockResolvedValue(fileData);
-      (global.URL.createObjectURL as Mock).mockReturnValue('blob:test');
-      (global.Blob as Mock).mockImplementation(() => ['test']);
-
-      const result = await fileService.getFile(fileId);
-
-      expect(FileModel.findById).toHaveBeenCalledWith(fileId);
-      expect(result).toEqual({
-        createdAt: new Date(1),
-        id: '1',
-        size: 1,
-        type: 'image/png',
-        name: 'test',
-        url: 'blob:test',
-        updatedAt: new Date(2),
-      });
-    });
-
-    it('should throw an error when the file is not found', async () => {
-      const fileId = 'non-existent';
-      (FileModel.findById as Mock).mockResolvedValue(null);
-
-      const getFilePromise = fileService.getFile(fileId);
-
-      await expect(getFilePromise).rejects.toThrow('file not found');
-    });
-  });
-});

package/src/services/file/_deprecated.ts

@@ -1,80 +0,0 @@
-import { FileModel } from '@/database/_deprecated/models/file';
-import { clientS3Storage } from '@/services/file/ClientS3';
-import { FileItem, UploadFileParams } from '@/types/files';
-
-import { IFileService } from './type';
-
-export class ClientService implements IFileService {
-  async createFile(file: UploadFileParams) {
-    // save to local storage
-    // we may want to save to a remote server later
-    const res = await FileModel.create({
-      createdAt: Date.now(),
-      data: undefined,
-      fileHash: file.hash,
-      fileType: file.fileType,
-      metadata: file.metadata,
-      name: file.name,
-      saveMode: 'url',
-      size: file.size,
-      url: file.url,
-    } as any);
-
-    // get file to base64 url
-    const base64 = await this.getBase64ByFileHash(file.hash!);
-
-    return {
-      id: res.id,
-      url: `data:${file.fileType};base64,${base64}`,
-    };
-  }
-
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async checkFileHash(_hash: string) {
-    return { isExist: false, metadata: {}, url: '' };
-  }
-
-  async getFile(id: string): Promise<FileItem> {
-    const item = await FileModel.findById(id);
-    if (!item) {
-      throw new Error('file not found');
-    }
-
-    // arrayBuffer to blob or base64 to blob
-    const blob = !!item.data
-      ? new Blob([item.data!], { type: item.fileType })
-      : // @ts-ignore
-        new Blob([Buffer.from(item.base64!, 'base64')], { type: item.fileType });
-
-    const url = URL.createObjectURL(blob);
-
-    return {
-      createdAt: new Date(item.createdAt),
-      id,
-      name: item.name,
-      size: item.size,
-      type: item.fileType,
-      updatedAt: new Date(item.updatedAt),
-      url,
-    };
-  }
-
-  async removeFile(id: string) {
-    return FileModel.delete(id);
-  }
-
-  async removeFiles(ids: string[]) {
-    await Promise.all(ids.map((id) => FileModel.delete(id)));
-  }
-
-  async removeAllFiles() {
-    return FileModel.clear();
-  }
-
-  private async getBase64ByFileHash(hash: string) {
-    const fileItem = await clientS3Storage.getObject(hash);
-    if (!fileItem) throw new Error('file not found');
-
-    return Buffer.from(await fileItem.arrayBuffer()).toString('base64');
-  }
-}

package/src/services/file/client.test.ts

@@ -1,199 +0,0 @@
-import { eq } from 'drizzle-orm';
-import { beforeEach, describe, expect, it } from 'vitest';
-
-import { clientDB, initializeDB } from '@/database/client/db';
-import { files, globalFiles, users } from '@/database/schemas';
-import { clientS3Storage } from '@/services/file/ClientS3';
-import { UploadFileParams } from '@/types/files';
-
-import { ClientService } from './client';
-
-const userId = 'file-user';
-
-const fileService = new ClientService(userId);
-
-const mockFile = {
-  name: 'mock.png',
-  fileType: 'image/png',
-  size: 1,
-  url: '',
-};
-
-beforeEach(async () => {
-  await initializeDB();
-
-  await clientDB.delete(users);
-  await clientDB.delete(globalFiles);
-  // 创建测试数据
-  await clientDB.transaction(async (tx) => {
-    await tx.insert(users).values({ id: userId });
-  });
-});
-
-describe('FileService', () => {
-  describe('createFile', () => {
-    it('createFile should save the file to the database', async () => {
-      const localFile: UploadFileParams = {
-        name: 'test',
-        fileType: 'image/png',
-        url: '',
-        size: 1,
-        hash: '123',
-      };
-
-      await clientS3Storage.putObject(
-        '123',
-        new File([new ArrayBuffer(1)], 'test.png', { type: 'image/png' }),
-      );
-
-      const result = await fileService.createFile(localFile);
-
-      expect(result).toMatchObject({ url: 'data:image/png;base64,AA==' });
-    });
-
-    it('should throw error when file is not found in storage during base64 conversion', async () => {
-      const localFile: UploadFileParams = {
-        name: 'test',
-        fileType: 'image/png',
-        url: '',
-        size: 1,
-        hash: 'non-existing-hash',
-      };
-
-      // 不调用 clientS3Storage.putObject，模拟文件不存在的情况
-
-      const promise = fileService.createFile(localFile);
-
-      await expect(promise).rejects.toThrow('file not found');
-    });
-  });
-
-  it('removeFile should delete the file from the database', async () => {
-    const fileId = '1';
-    await clientDB.insert(files).values({ id: fileId, userId, ...mockFile });
-
-    await fileService.removeFile(fileId);
-
-    const result = await clientDB.query.files.findFirst({
-      where: eq(files.id, fileId),
-    });
-
-    expect(result).toBeUndefined();
-  });
-
-  describe('getFile', () => {
-    it('should retrieve and convert local file info to FilePreview', async () => {
-      const fileId = 'rwlijweled';
-      const file = {
-        fileType: 'image/png',
-        size: 1,
-        name: 'test.png',
-        url: 'idb://12312/abc.png',
-        hashId: '123tttt',
-      };
-
-      await clientDB.insert(globalFiles).values({ ...file, creator: userId });
-
-      await clientDB.insert(files).values({
-        id: fileId,
-        userId,
-        ...file,
-        createdAt: new Date(1),
-        updatedAt: new Date(2),
-        fileHash: file.hashId,
-      });
-
-      await clientS3Storage.putObject(
-        file.hashId,
-        new File([new ArrayBuffer(1)], file.name, { type: file.fileType }),
-      );
-
-      const result = await fileService.getFile(fileId);
-
-      expect(result).toMatchObject({
-        createdAt: new Date(1),
-        id: 'rwlijweled',
-        size: 1,
-        type: 'image/png',
-        name: 'test.png',
-        updatedAt: new Date(2),
-      });
-    });
-
-    it('should throw an error when the file is not found', async () => {
-      const fileId = 'non-existent';
-
-      const getFilePromise = fileService.getFile(fileId);
-
-      await expect(getFilePromise).rejects.toThrow('file not found');
-    });
-  });
-
-  describe('removeFiles', () => {
-    it('should delete multiple files from the database', async () => {
-      const fileIds = ['1', '2', '3'];
-
-      // 插入测试文件数据
-      await Promise.all(
-        fileIds.map((id) => clientDB.insert(files).values({ id, userId, ...mockFile })),
-      );
-
-      await fileService.removeFiles(fileIds);
-
-      // 验证所有文件都被删除
-      const remainingFiles = await clientDB.query.files.findMany({
-        where: (fields, { inArray }) => inArray(fields.id, fileIds),
-      });
-
-      expect(remainingFiles).toHaveLength(0);
-    });
-  });
-
-  describe('removeAllFiles', () => {
-    it('should clear all files for the user', async () => {
-      // 插入测试文件数据
-      await Promise.all([
-        clientDB.insert(files).values({ id: '1', userId, ...mockFile }),
-        clientDB.insert(files).values({ id: '2', userId, ...mockFile }),
-      ]);
-
-      await fileService.removeAllFiles();
-
-      // 验证用户的所有文件都被删除
-      const remainingFiles = await clientDB.query.files.findMany({
-        where: eq(files.userId, userId),
-      });
-
-      expect(remainingFiles).toHaveLength(0);
-    });
-  });
-
-  describe('checkFileHash', () => {
-    it('should return true if file hash exists', async () => {
-      const hash = 'existing-hash';
-      await clientDB.insert(globalFiles).values({
-        ...mockFile,
-        hashId: hash,
-        creator: userId,
-      });
-      await clientDB.insert(files).values({
-        id: '1',
-        userId,
-        ...mockFile,
-        fileHash: hash,
-      });
-
-      const exists = await fileService.checkFileHash(hash);
-
-      expect(exists).toMatchObject({ isExist: true });
-    });
-
-    it('should return false if file hash does not exist', async () => {
-      const hash = 'non-existing-hash';
-
-      const exists = await fileService.checkFileHash(hash);
-
-      expect(exists).toEqual({ isExist: false });
-    });
-  });
-});

package/src/services/file/client.ts

@@ -1,85 +0,0 @@
-import { clientDB } from '@/database/client/db';
-import { FileModel } from '@/database/models/file';
-import { BaseClientService } from '@/services/baseClientService';
-import { clientS3Storage } from '@/services/file/ClientS3';
-
-import { IFileService } from './type';
-
-export class ClientService extends BaseClientService implements IFileService {
-  private get fileModel(): FileModel {
-    return new FileModel(clientDB as any, this.userId);
-  }
-
-  createFile: IFileService['createFile'] = async (file) => {
-    const { isExist } = await this.fileModel.checkHash(file.hash!);
-
-    // save to local storage
-    // we may want to save to a remote server later
-    const res = await this.fileModel.create(
-      {
-        fileHash: file.hash,
-        fileType: file.fileType,
-        knowledgeBaseId: file.knowledgeBaseId,
-        metadata: file.metadata,
-        name: file.name,
-        size: file.size,
-        url: file.url!,
-      },
-      !isExist,
-    );
-
-    // get file to base64 url
-    const base64 = await this.getBase64ByFileHash(file.hash!);
-
-    return {
-      id: res.id,
-      url: `data:${file.fileType};base64,${base64}`,
-    };
-  };
-
-  getFile: IFileService['getFile'] = async (id) => {
-    const item = await this.fileModel.findById(id);
-    if (!item) {
-      throw new Error('file not found');
-    }
-
-    // arrayBuffer to url
-    const fileItem = await clientS3Storage.getObject(item.fileHash!);
-    if (!fileItem) throw new Error('file not found');
-
-    const url = URL.createObjectURL(fileItem);
-
-    return {
-      createdAt: new Date(item.createdAt),
-      id,
-      name: item.name,
-      size: item.size,
-      type: item.fileType,
-      updatedAt: new Date(item.updatedAt),
-      url,
-    };
-  };
-
-  removeFile: IFileService['removeFile'] = async (id) => {
-    await this.fileModel.delete(id, false);
-  };
-
-  removeFiles: IFileService['removeFiles'] = async (ids) => {
-    await this.fileModel.deleteMany(ids, false);
-  };
-
-  removeAllFiles: IFileService['removeAllFiles'] = async () => {
-    return this.fileModel.clear();
-  };
-
-  checkFileHash: IFileService['checkFileHash'] = async (hash) => {
-    return this.fileModel.checkHash(hash);
-  };
-
-  private getBase64ByFileHash = async (hash: string) => {
-    const fileItem = await clientS3Storage.getObject(hash);
-    if (!fileItem) throw new Error('file not found');
-
-    return Buffer.from(await fileItem.arrayBuffer()).toString('base64');
-  };
-}

package/src/services/file/server.ts

@@ -1,53 +0,0 @@
-import { lambdaClient } from '@/libs/trpc/client';
-import { QueryFileListParams, QueryFileListSchemaType, UploadFileParams } from '@/types/files';
-
-import { IFileService } from './type';
-
-interface CreateFileParams extends Omit<UploadFileParams, 'url'> {
-  knowledgeBaseId?: string;
-  url: string;
-}
-
-export class ServerService implements IFileService {
-  createFile: IFileService['createFile'] = async (params, knowledgeBaseId) => {
-    return lambdaClient.file.createFile.mutate({ ...params, knowledgeBaseId } as CreateFileParams);
-  };
-
-  getFile: IFileService['getFile'] = async (id) => {
-    const item = await lambdaClient.file.findById.query({ id });
-
-    if (!item) {
-      throw new Error('file not found');
-    }
-
-    return { ...item, type: item.fileType };
-  };
-
-  removeFile: IFileService['removeFile'] = async (id) => {
-    await lambdaClient.file.removeFile.mutate({ id });
-  };
-
-  removeFiles: IFileService['removeFiles'] = async (ids) => {
-    await lambdaClient.file.removeFiles.mutate({ ids });
-  };
-
-  removeAllFiles: IFileService['removeAllFiles'] = async () => {
-    await lambdaClient.file.removeAllFiles.mutate();
-  };
-
-  getFiles = async (params: QueryFileListParams) => {
-    return lambdaClient.file.getFiles.query(params as QueryFileListSchemaType);
-  };
-
-  getFileItem = async (id: string) => {
-    return lambdaClient.file.getFileItemById.query({ id });
-  };
-
-  checkFileHash: IFileService['checkFileHash'] = async (hash) => {
-    return lambdaClient.file.checkFileHash.mutate({ hash });
-  };
-
-  removeFileAsyncTask = async (id: string, type: 'embedding' | 'chunk') => {
-    return lambdaClient.file.removeFileAsyncTask.mutate({ id, type });
-  };
-}

package/src/services/file/type.ts

@@ -1,13 +0,0 @@
-import { CheckFileHashResult, FileItem, UploadFileParams } from '@/types/files';
-
-export interface IFileService {
-  checkFileHash(hash: string): Promise<CheckFileHashResult>;
-  createFile(
-    file: UploadFileParams,
-    knowledgeBaseId?: string,
-  ): Promise<{ id: string; url: string }>;
-  getFile(id: string): Promise<FileItem>;
-  removeAllFiles(): Promise<any>;
-  removeFile(id: string): Promise<void>;
-  removeFiles(ids: string[]): Promise<void>;
-}

package/src/services/import/_deprecated.ts

@@ -1,115 +0,0 @@
-import { MessageModel } from '@/database/_deprecated/models/message';
-import { SessionModel } from '@/database/_deprecated/models/session';
-import { SessionGroupModel } from '@/database/_deprecated/models/sessionGroup';
-import { TopicModel } from '@/database/_deprecated/models/topic';
-import { useUserStore } from '@/store/user';
-import { ConfigFile } from '@/types/exportConfig';
-import { ImportStage, ImporterEntryData, OnImportCallbacks } from '@/types/importer';
-import { UserSettings } from '@/types/user/settings';
-
-export interface ImportResult {
-  added: number;
-  errors: number;
-  skips: number;
-  updated?: number;
-}
-export interface ImportResults {
-  messages?: ImportResult;
-  sessionGroups?: ImportResult;
-  sessions?: ImportResult;
-  topics?: ImportResult;
-  type?: string;
-}
-
-export class ClientService {
-  importSettings = async (settings: UserSettings) => {
-    await useUserStore.getState().importAppSettings(settings);
-  };
-
-  importData = async (
-    config: ImporterEntryData,
-    callbacks?: OnImportCallbacks,
-  ): Promise<ImportResults> => {
-    callbacks?.onStageChange?.(ImportStage.Importing);
-    const time = Date.now();
-
-    const { messages = [], sessionGroups = [], sessions = [], topics = [] } = config;
-
-    let messageResult: ImportResult | undefined;
-    let sessionResult: ImportResult | undefined;
-    let sessionGroupResult: ImportResult | undefined;
-    let topicResult: ImportResult | undefined;
-
-    if (messages.length > 0) {
-      const res = await MessageModel.batchCreate(messages as any);
-      messageResult = this.mapImportResult(res);
-    }
-
-    if (sessionGroups.length > 0) {
-      const res = await SessionGroupModel.batchCreate(sessionGroups as any);
-      sessionGroupResult = this.mapImportResult(res);
-    }
-
-    if (topics.length > 0) {
-      const res = await TopicModel.batchCreate(topics as any);
-      topicResult = this.mapImportResult(res);
-    }
-
-    if (sessions.length > 0) {
-      const data = await SessionModel.batchCreate(sessions as any);
-      sessionResult = this.mapImportResult(data);
-    }
-
-    const result = {
-      messages: messageResult,
-      sessionGroups: sessionGroupResult,
-      sessions: sessionResult,
-      topics: topicResult,
-    };
-
-    const duration = Date.now() - time;
-    callbacks?.onStageChange?.(ImportStage.Success);
-    callbacks?.onSuccess?.(result, duration);
-
-    return result;
-  };
-
-  private mapImportResult = (input: {
-    added: number;
-    errors?: Error[];
-    skips: string[];
-  }): ImportResult => {
-    return {
-      added: input.added,
-      errors: input.errors?.length || 0,
-      skips: input.skips.length,
-    };
-  };
-
-  importConfigState = async (config: ConfigFile, callbacks?: OnImportCallbacks): Promise<void> => {
-    if (config.exportType === 'settings') {
-      await this.importSettings(config.state.settings);
-      callbacks?.onStageChange?.(ImportStage.Success);
-      return;
-    }
-
-    if (config.exportType === 'all') {
-      await this.importSettings(config.state.settings);
-    }
-
-    await this.importData(
-      {
-        messages: (config.state as any).messages || [],
-        sessionGroups: (config.state as any).sessionGroups || [],
-        sessions: (config.state as any).sessions || [],
-        topics: (config.state as any).topics || [],
-        version: config.version,
-      },
-      callbacks,
-    );
-  };
-
-  importPgData = async () => {
-    throw new Error('Not implemented');
-  };
-}