@lobehub/lobehub 2.0.0-next.3 → 2.0.0-next.31

package/src/services/file/index.ts

@@ -1,13 +1,62 @@
-import { isDesktop } from '@/const/version';
+import { lambdaClient } from '@/libs/trpc/client';
+import {
+  CheckFileHashResult,
+  FileItem,
+  QueryFileListParams,
+  QueryFileListSchemaType,
+  UploadFileParams,
+} from '@/types/files';
 
-import { ClientService as DeprecatedService } from './_deprecated';
-import { ClientService } from './client';
-import { ServerService } from './server';
+interface CreateFileParams extends Omit<UploadFileParams, 'url'> {
+  knowledgeBaseId?: string;
+  url: string;
+}
 
-const clientService =
-  process.env.NEXT_PUBLIC_CLIENT_DB === 'pglite' ? new ClientService() : new DeprecatedService();
+export class FileService {
+  createFile = async (
+    params: UploadFileParams,
+    knowledgeBaseId?: string,
+  ): Promise<{ id: string; url: string }> => {
+    return lambdaClient.file.createFile.mutate({ ...params, knowledgeBaseId } as CreateFileParams);
+  };
 
-export const fileService =
-  process.env.NEXT_PUBLIC_SERVICE_MODE === 'server' || isDesktop
-    ? new ServerService()
-    : clientService;
+  getFile = async (id: string): Promise<FileItem> => {
+    const item = await lambdaClient.file.findById.query({ id });
+
+    if (!item) {
+      throw new Error('file not found');
+    }
+
+    return { ...item, type: item.fileType };
+  };
+
+  removeFile = async (id: string): Promise<void> => {
+    await lambdaClient.file.removeFile.mutate({ id });
+  };
+
+  removeFiles = async (ids: string[]): Promise<void> => {
+    await lambdaClient.file.removeFiles.mutate({ ids });
+  };
+
+  removeAllFiles = async () => {
+    await lambdaClient.file.removeAllFiles.mutate();
+  };
+
+  getFiles = async (params: QueryFileListParams) => {
+    return lambdaClient.file.getFiles.query(params as QueryFileListSchemaType);
+  };
+
+  getFileItem = async (id: string) => {
+    return lambdaClient.file.getFileItemById.query({ id });
+  };
+
+  checkFileHash = async (hash: string): Promise<CheckFileHashResult> => {
+    return lambdaClient.file.checkFileHash.mutate({ hash });
+  };
+
+  removeFileAsyncTask = async (id: string, type: 'embedding' | 'chunk') => {
+    return lambdaClient.file.removeFileAsyncTask.mutate({ id, type });
+  };
+}
+
+export const fileService = new FileService();

package/src/services/import/index.ts

@@ -1,9 +1,134 @@
-import { ClientService as DeprecatedService } from './_deprecated';
-import { ClientService } from './client';
-import { ServerService } from './server';
+import { DefaultErrorShape } from '@trpc/server/unstable-core-do-not-import';
 
-const clientService =
-  process.env.NEXT_PUBLIC_CLIENT_DB === 'pglite' ? new ClientService() : new DeprecatedService();
+import { lambdaClient } from '@/libs/trpc/client';
+import { uploadService } from '@/services/upload';
+import { useUserStore } from '@/store/user';
+import { ImportPgDataStructure } from '@/types/export';
+import { ImporterEntryData, ImportStage, OnImportCallbacks } from '@/types/importer';
+import { UserSettings } from '@/types/user/settings';
+import { uuid } from '@/utils/uuid';
 
-export const importService =
-  process.env.NEXT_PUBLIC_SERVICE_MODE === 'server' ? new ServerService() : clientService;
+class ImportService {
+  importSettings = async (settings: UserSettings): Promise<void> => {
+    await useUserStore.getState().importAppSettings(settings);
+  };
+
+  importData = async (data: ImporterEntryData, callbacks?: OnImportCallbacks): Promise<void> => {
+    const handleError = (e: unknown) => {
+      callbacks?.onStageChange?.(ImportStage.Error);
+      const error = e as DefaultErrorShape;
+
+      callbacks?.onError?.({
+        code: error.data.code,
+        httpStatus: error.data.httpStatus,
+        message: error.message,
+        path: error.data.path,
+      });
+    };
+
+    const totalLength =
+      (data.messages?.length || 0) +
+      (data.sessionGroups?.length || 0) +
+      (data.sessions?.length || 0) +
+      (data.topics?.length || 0);
+
+    if (totalLength < 500) {
+      callbacks?.onStageChange?.(ImportStage.Importing);
+      const time = Date.now();
+      try {
+        const result = await lambdaClient.importer.importByPost.mutate({ data });
+        const duration = Date.now() - time;
+
+        callbacks?.onStageChange?.(ImportStage.Success);
+        callbacks?.onSuccess?.(result.results, duration);
+      } catch (e) {
+        handleError(e);
+      }
+
+      return;
+    }
+
+    await this.uploadData(data, { callbacks, handleError });
+  };
+
+  importPgData = async (
+    data: ImportPgDataStructure,
+    options?: {
+      callbacks?: OnImportCallbacks;
+      overwriteExisting?: boolean;
+    },
+  ): Promise<void> => {
+    const { callbacks } = options || {};
+
+    const handleError = (e: unknown) => {
+      callbacks?.onStageChange?.(ImportStage.Error);
+      const error = e as DefaultErrorShape;
+
+      callbacks?.onError?.({
+        code: error.data.code,
+        httpStatus: error.data.httpStatus,
+        message: error.message,
+        path: error.data.path,
+      });
+    };
+
+    const totalLength = Object.values(data.data)
+      .map((d) => d.length)
+      .reduce((a, b) => a + b, 0);
+
+    if (totalLength < 500) {
+      callbacks?.onStageChange?.(ImportStage.Importing);
+      const time = Date.now();
+      try {
+        const result = await lambdaClient.importer.importPgByPost.mutate(data);
+        const duration = Date.now() - time;
+
+        callbacks?.onStageChange?.(ImportStage.Success);
+        callbacks?.onSuccess?.(result.results, duration);
+      } catch (e) {
+        handleError(e);
+      }
+
+      return;
+    }
+
+    await this.uploadData(data, { callbacks, handleError });
+  };
+
+  private uploadData = async (
+    data: object,
+    { callbacks, handleError }: { callbacks?: OnImportCallbacks; handleError: (e: unknown) => any },
+  ) => {
+    // if the data is too large, upload it to S3 and upload by file
+    const filename = `${uuid()}.json`;
+
+    let pathname;
+    try {
+      callbacks?.onStageChange?.(ImportStage.Uploading);
+      const result = await uploadService.uploadDataToS3(data, {
+        filename,
+        onProgress: (status, state) => {
+          callbacks?.onFileUploading?.(state);
+        },
+        pathname: `import_config/${filename}`,
+      });
+      pathname = result.data.path;
+      console.log(pathname);
+    } catch {
+      throw new Error('Upload Error');
+    }
+
+    callbacks?.onStageChange?.(ImportStage.Importing);
+    const time = Date.now();
+    try {
+      const result = await lambdaClient.importer.importByFile.mutate({ pathname });
+      const duration = Date.now() - time;
+      callbacks?.onStageChange?.(ImportStage.Success);
+      callbacks?.onSuccess?.(result.results, duration);
+    } catch (e) {
+      handleError(e);
+    }
+  };
+}
+
+export const importService = new ImportService();