@lobehub/chat 1.76.1 → 1.77.1

package/src/features/User/UserPanel/useMenu.tsx

@@ -9,7 +9,6 @@ import {
   Feather,
   FileClockIcon,
   HardDriveDownload,
-  HardDriveUpload,
   LifeBuoy,
   LogOut,
   Mail,
@@ -32,10 +31,8 @@ import {
   UTM_SOURCE,
   mailTo,
 } from '@/const/url';
-import { isServerMode } from '@/const/version';
 import DataImporter from '@/features/DataImporter';
 import { usePWAInstall } from '@/hooks/usePWAInstall';
-import { configService } from '@/services/config';
 import { featureFlagsSelectors, useServerConfigStore } from '@/store/serverConfig';
 import { useUserStore } from '@/store/user';
 import { authSelectors } from '@/store/user/selectors';
@@ -119,40 +116,8 @@ export const useMenu = () => {
         {
           icon: <Icon icon={HardDriveDownload} />,
           key: 'import',
-          label: <DataImporter>{t('import')}</DataImporter>,
+          label: <DataImporter>{t('importData')}</DataImporter>,
         },
-        isServerMode
-          ? null
-          : {
-              children: [
-                {
-                  key: 'allAgent',
-                  label: t('exportType.allAgent'),
-                  onClick: configService.exportAgents,
-                },
-                {
-                  key: 'allAgentWithMessage',
-                  label: t('exportType.allAgentWithMessage'),
-                  onClick: configService.exportSessions,
-                },
-                {
-                  key: 'globalSetting',
-                  label: t('exportType.globalSetting'),
-                  onClick: configService.exportSettings,
-                },
-                {
-                  type: 'divider',
-                },
-                {
-                  key: 'all',
-                  label: t('exportType.all'),
-                  onClick: configService.exportAll,
-                },
-              ],
-              icon: <Icon icon={HardDriveUpload} />,
-              key: 'export',
-              label: t('export'),
-            },
         {
           type: 'divider',
         },

package/src/features/User/__tests__/useMenu.test.tsx

@@ -81,7 +81,6 @@ describe('useMenu', () => {
       expect(mainItems?.some((item) => item?.key === 'profile')).toBe(true);
       expect(mainItems?.some((item) => item?.key === 'setting')).toBe(true);
       expect(mainItems?.some((item) => item?.key === 'import')).toBe(true);
-      expect(mainItems?.some((item) => item?.key === 'export')).toBe(true);
       expect(mainItems?.some((item) => item?.key === 'changelog')).toBe(true);
       expect(logoutItems.some((item) => item?.key === 'logout')).toBe(true);
     });
@@ -100,7 +99,6 @@ describe('useMenu', () => {
       expect(mainItems?.some((item) => item?.key === 'profile')).toBe(true);
       expect(mainItems?.some((item) => item?.key === 'setting')).toBe(true);
       expect(mainItems?.some((item) => item?.key === 'import')).toBe(true);
-      expect(mainItems?.some((item) => item?.key === 'export')).toBe(true);
       expect(mainItems?.some((item) => item?.key === 'changelog')).toBe(true);
       expect(logoutItems.some((item) => item?.key === 'logout')).toBe(false);
     });

package/src/locales/default/common.ts

@@ -186,7 +186,7 @@ export default {
   },
   fullscreen: '全屏模式',
   historyRange: '历史范围',
-  import: '导入配置',
+  importData: '导入数据',
   importModal: {
     error: {
       desc: '非常抱歉，数据导入过程发生异常。请尝试重新导入，或 <1>提交问题</1>，我们将会第一时间帮你排查问题。',
@@ -209,6 +209,7 @@ export default {
       skips: '重复跳过',
       topics: '话题',
       type: '数据类型',
+      update: '记录更新',
     },
     title: '导入数据',
     uploading: {
@@ -217,6 +218,16 @@ export default {
       speed: '上传速度',
     },
   },
+  importPreview: {
+    confirmImport: '确认导入',
+    tables: {
+      count: '记录数',
+      name: '表名',
+    },
+    title: '导入数据预览',
+    totalRecords: '总计将导入 {{count}} 条记录',
+    totalTables: '{{count}} 个表',
+  },
   information: '社区与资讯',
   installPWA: '安装浏览器应用 (PWA)',
   lang: {

package/src/locales/default/error.ts

@@ -16,6 +16,16 @@ export default {
     detail: '错误详情',
     title: '请求失败',
   },
+  import: {
+    importConfigFile: {
+      description: '出错原因: {{reason}}',
+      title: '导入失败',
+    },
+    incompatible: {
+      description: '该文件由更高版本导出，请尝试升级到最新版本后再重新导入',
+      title: '当前应用不支持导入该文件',
+    },
+  },
   loginRequired: {
     desc: '即将自动跳转到登录页面',
     title: '请登录后使用该功能',

package/src/locales/default/setting.ts

@@ -348,6 +348,33 @@ export default {
     },
     title: '主题设置',
   },
+  storage: {
+    actions: {
+      export: {
+        button: '导出',
+        exportType: {
+          agent: '导出助手设定',
+          agentWithMessage: '导出助手和消息',
+          all: '导出全局设置和所有助手数据',
+          allAgent: '导出所有助手设定',
+          allAgentWithMessage: '导出所有助手和消息',
+          globalSetting: '导出全局设置',
+        },
+        title: '导出数据',
+      },
+      import: {
+        button: '导入',
+        title: '导入数据',
+      },
+      title: '高级操作',
+    },
+    desc: '当前浏览器中的存储用量',
+    embeddings: {
+      used: '向量存储',
+    },
+    title: '数据存储',
+    used: '存储用量',
+  },
   submitAgentModal: {
     button: '提交助手',
     identifier: 'identifier 助手标识符',
@@ -441,6 +468,7 @@ export default {
     'hotkey': '快捷键',
     'llm': '语言模型',
     'provider': 'AI 服务商',
+    'storage': '数据存储',
     'sync': '云端同步',
     'system-agent': '系统助手',
     'tts': '语音服务',

package/src/server/routers/lambda/exporter.ts

@@ -0,0 +1,25 @@
+import { DrizzleMigrationModel } from '@/database/models/drizzleMigration';
+import { DataExporterRepos } from '@/database/repositories/dataExporter';
+import { serverDB } from '@/database/server';
+import { authedProcedure, router } from '@/libs/trpc';
+import { ExportDatabaseData } from '@/types/export';
+
+const exportProcedure = authedProcedure.use(async (opts) => {
+  const { ctx } = opts;
+  const dataExporterRepos = new DataExporterRepos(serverDB, ctx.userId);
+  const drizzleMigration = new DrizzleMigrationModel(serverDB);
+
+  return opts.next({
+    ctx: { dataExporterRepos, drizzleMigration },
+  });
+});
+
+export const exporterRouter = router({
+  exportData: exportProcedure.mutation(async ({ ctx }): Promise<ExportDatabaseData> => {
+    const data = await ctx.dataExporterRepos.export(5);
+
+    const schemaHash = await ctx.drizzleMigration.getLatestMigrationHash();
+
+    return { data, schemaHash };
+  }),
+});

package/src/server/routers/lambda/importer.ts

@@ -5,7 +5,8 @@ import { DataImporterRepos } from '@/database/repositories/dataImporter';
 import { serverDB } from '@/database/server';
 import { authedProcedure, router } from '@/libs/trpc';
 import { S3 } from '@/server/modules/S3';
-import { ImportResults, ImporterEntryData } from '@/types/importer';
+import { ImportPgDataStructure } from '@/types/export';
+import { ImportResultData, ImporterEntryData } from '@/types/importer';
 
 const importProcedure = authedProcedure.use(async (opts) => {
   const { ctx } = opts;
@@ -19,7 +20,7 @@ const importProcedure = authedProcedure.use(async (opts) => {
 export const importerRouter = router({
   importByFile: importProcedure
     .input(z.object({ pathname: z.string() }))
-    .mutation(async ({ input, ctx }): Promise<ImportResults> => {
+    .mutation(async ({ input, ctx }): Promise<ImportResultData> => {
       let data: ImporterEntryData | undefined;
 
       try {
@@ -37,6 +38,10 @@ export const importerRouter = router({
         });
       }
 
+      if ('schemaHash' in data) {
+        return ctx.dataImporterService.importPgData(data as unknown as ImportPgDataStructure);
+      }
+
       return ctx.dataImporterService.importData(data);
     }),
 
@@ -52,7 +57,18 @@ export const importerRouter = router({
         }),
       }),
     )
-    .mutation(async ({ input, ctx }): Promise<ImportResults> => {
+    .mutation(async ({ input, ctx }): Promise<ImportResultData> => {
       return ctx.dataImporterService.importData(input.data);
     }),
+  importPgByPost: importProcedure
+    .input(
+      z.object({
+        data: z.record(z.string(), z.array(z.any())),
+        mode: z.enum(['pglite', 'postgres']),
+        schemaHash: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }): Promise<ImportResultData> => {
+      return ctx.dataImporterService.importPgData(input);
+    }),
 });

package/src/server/routers/lambda/index.ts

@@ -7,6 +7,7 @@ import { agentRouter } from './agent';
 import { aiModelRouter } from './aiModel';
 import { aiProviderRouter } from './aiProvider';
 import { chunkRouter } from './chunk';
+import { exporterRouter } from './exporter';
 import { fileRouter } from './file';
 import { importerRouter } from './importer';
 import { knowledgeBaseRouter } from './knowledgeBase';
@@ -24,6 +25,7 @@ export const lambdaRouter = router({
   aiModel: aiModelRouter,
   aiProvider: aiProviderRouter,
   chunk: chunkRouter,
+  exporter: exporterRouter,
   file: fileRouter,
   healthcheck: publicProcedure.query(() => "i'm live!"),
   importer: importerRouter,

package/src/services/config.ts

@@ -1,162 +1,107 @@
-import { importService } from '@/services/import';
-import { messageService } from '@/services/message';
-import { sessionService } from '@/services/session';
-import { topicService } from '@/services/topic';
-import { useSessionStore } from '@/store/session';
-import { sessionSelectors } from '@/store/session/selectors';
-import { useUserStore } from '@/store/user';
-import { settingsSelectors } from '@/store/user/selectors';
-import { ConfigFile } from '@/types/exportConfig';
-import { ImportStage, OnImportCallbacks } from '@/types/importer';
-import { createConfigFile, exportConfigFile } from '@/utils/config';
-
-export interface ImportResult {
-  added: number;
-  errors: number;
-  skips: number;
-}
-export interface ImportResults {
-  messages?: ImportResult;
-  sessionGroups?: ImportResult;
-  sessions?: ImportResult;
-  topics?: ImportResult;
-  type?: string;
-}
+import dayjs from 'dayjs';
+
+import { BRANDING_NAME } from '@/const/branding';
+import { isDeprecatedEdition, isServerMode } from '@/const/version';
+import { CURRENT_CONFIG_VERSION } from '@/migrations';
+import { ImportPgDataStructure } from '@/types/export';
+import { downloadFile } from '@/utils/client/downloadFile';
+import { exportJSONFile } from '@/utils/client/exportFile';
+
+import { exportService } from './export';
+import { configService as deprecatedExportService } from './export/_deprecated';
 
-/**
- * @deprecated
- */
 class ConfigService {
-  importConfigState = async (config: ConfigFile, callbacks?: OnImportCallbacks): Promise<void> => {
-    if (config.exportType === 'settings') {
-      await importService.importSettings(config.state.settings);
-      callbacks?.onStageChange?.(ImportStage.Success);
+  exportAll = async () => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const config = await deprecatedExportService.exportAll();
+      const filename = `${BRANDING_NAME}-config-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(config, filename);
       return;
     }
 
-    if (config.exportType === 'all') {
-      await importService.importSettings(config.state.settings);
-    }
-
-    await importService.importData(
-      {
-        messages: (config.state as any).messages || [],
-        sessionGroups: (config.state as any).sessionGroups || [],
-        sessions: (config.state as any).sessions || [],
-        topics: (config.state as any).topics || [],
-        version: config.version,
-      },
-      callbacks,
-    );
-  };
-
-  // TODO: Separate export feature into a new service like importService
-
-  /**
-   * export all agents
-   */
-  exportAgents = async () => {
-    const agents = await sessionService.getSessionsByType('agent');
-    const sessionGroups = await sessionService.getSessionGroups();
-
-    const config = createConfigFile('agents', { sessionGroups, sessions: agents });
-
-    exportConfigFile(config, 'agents');
-  };
+    const { data, url } = await exportService.exportData();
+    const filename = `${dayjs().format('YYYY-MM-DD-hh-mm')}_${BRANDING_NAME}-data.json`;
 
-  /**
-   * export all sessions
-   */
-  exportSessions = async () => {
-    const sessions = await sessionService.getSessionsByType();
-    const sessionGroups = await sessionService.getSessionGroups();
-    const messages = await messageService.getAllMessages();
-    const topics = await topicService.getAllTopics();
+    // if url exists, means export data from server and upload the data to S3
+    // just need to download the file
+    if (url) {
+      await downloadFile(url, filename);
+      return;
+    }
 
-    const config = createConfigFile('sessions', { messages, sessionGroups, sessions, topics });
+    // or export to file with the data
+    const result = await this.createDataStructure(data, isServerMode ? 'postgres' : 'pglite');
 
-    exportConfigFile(config, 'sessions');
+    exportJSONFile(result, filename);
   };
 
-  /**
-   * export a session
-   */
-  exportSingleSession = async (id: string) => {
-    const session = this.getSession(id);
-    if (!session) return;
-
-    const messages = await messageService.getAllMessagesInSession(id);
-    const topics = await topicService.getTopics({ sessionId: id });
-
-    const config = createConfigFile('singleSession', { messages, sessions: [session], topics });
-
-    exportConfigFile(config, `${session.meta?.title}-session`);
+  exportAgents = async () => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const config = await deprecatedExportService.exportAgents();
+      const filename = `${BRANDING_NAME}-agents-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(config, filename);
+      return;
+    }
   };
 
-  /**
-   * export a topic
-   */
-  exportSingleTopic = async (sessionId: string, topicId: string) => {
-    const session = this.getSession(sessionId);
-    if (!session) return;
-
-    const messages = await messageService.getMessages(sessionId, topicId);
-    const topics = await topicService.getTopics({ sessionId });
-
-    const topic = topics.find((item) => item.id === topicId);
-    if (!topic) return;
+  exportSingleAgent = async (agentId: string) => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const result = await deprecatedExportService.exportSingleAgent(agentId);
+      if (!result) return;
 
-    const config = createConfigFile('singleSession', {
-      messages,
-      sessions: [session],
-      topics: [topic],
-    });
-
-    exportConfigFile(config, `${topic.title}-topic`);
+      const filename = `${BRANDING_NAME}-${result.title}-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(result.config, filename);
+      return;
+    }
   };
 
-  exportSingleAgent = async (id: string) => {
-    const agent = this.getAgent(id);
-    if (!agent) return;
-
-    const config = createConfigFile('agents', { sessionGroups: [], sessions: [agent] });
-
-    exportConfigFile(config, agent.meta?.title || 'agent');
+  exportSessions = async () => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const config = await deprecatedExportService.exportSessions();
+      const filename = `${BRANDING_NAME}-sessions-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(config, filename);
+      return;
+    }
   };
 
-  /**
-   * export settings
-   */
   exportSettings = async () => {
-    const settings = this.getSettings();
-
-    const config = createConfigFile('settings', { settings });
-
-    exportConfigFile(config, 'settings');
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const config = await deprecatedExportService.exportSessions();
+      const filename = `${BRANDING_NAME}-settings-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(config, filename);
+      return;
+    }
   };
 
-  /**
-   * export all data
-   */
-  exportAll = async () => {
-    const sessions = await sessionService.getSessionsByType();
-    const sessionGroups = await sessionService.getSessionGroups();
-    const messages = await messageService.getAllMessages();
-    const topics = await topicService.getAllTopics();
-    const settings = this.getSettings();
-
-    const config = createConfigFile('all', { messages, sessionGroups, sessions, settings, topics });
+  exportSingleSession = async (sessionId: string) => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const data = await deprecatedExportService.exportSingleSession(sessionId);
+      if (!data) return;
 
-    exportConfigFile(config, 'config');
+      const filename = `${BRANDING_NAME}-${data.title}-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(data.config, filename);
+      return;
+    }
   };
 
-  private getSettings = () => settingsSelectors.exportSettings(useUserStore.getState());
-
-  private getSession = (id: string) =>
-    sessionSelectors.getSessionById(id)(useSessionStore.getState());
+  private createDataStructure = async (
+    data: any,
+    mode: 'pglite' | 'postgres',
+  ): Promise<ImportPgDataStructure> => {
+    const { default: json } = await import('@/database/client/migrations.json');
+    const latestHash = json.at(-1)?.hash;
+    if (!latestHash) {
+      throw new Error('Not find database sql hash');
+    }
 
-  private getAgent = (id: string) =>
-    sessionSelectors.getSessionById(id)(useSessionStore.getState());
+    return { data, mode, schemaHash: latestHash };
+  };
 }
 
 export const configService = new ConfigService();

package/src/services/export/_deprecated.ts

@@ -0,0 +1,155 @@
+import { Migration } from '@/migrations';
+import { messageService } from '@/services/message';
+import { sessionService } from '@/services/session';
+import { topicService } from '@/services/topic';
+import { useSessionStore } from '@/store/session';
+import { sessionSelectors } from '@/store/session/selectors';
+import { useUserStore } from '@/store/user';
+import { settingsSelectors } from '@/store/user/selectors';
+import {
+  ConfigFileAgents,
+  ConfigFileAll,
+  ConfigFileSessions,
+  ConfigFileSettings,
+  ConfigFileSingleSession,
+  ConfigModelMap,
+  ExportType,
+} from '@/types/exportConfig';
+
+type CreateConfigFileState<T extends ExportType> = ConfigModelMap[T]['state'];
+
+type CreateConfigFile<T extends ExportType> = ConfigModelMap[T]['file'];
+
+const createConfigFile = <T extends ExportType>(
+  type: T,
+  state: CreateConfigFileState<T>,
+): CreateConfigFile<T> => {
+  switch (type) {
+    case 'agents': {
+      return {
+        exportType: 'agents',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileAgents;
+    }
+
+    case 'sessions': {
+      return {
+        exportType: 'sessions',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileSessions;
+    }
+
+    case 'settings': {
+      return {
+        exportType: 'settings',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileSettings;
+    }
+
+    case 'singleSession': {
+      return {
+        exportType: 'sessions',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileSingleSession;
+    }
+
+    case 'all': {
+      return {
+        exportType: 'all',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileAll;
+    }
+  }
+
+  throw new Error('缺少正确的导出类型，请检查实现...');
+};
+
+/**
+ * @deprecated
+ */
+export class ConfigService {
+  /**
+   * export all agents
+   */
+  exportAgents = async () => {
+    const agents = await sessionService.getSessionsByType('agent');
+    const sessionGroups = await sessionService.getSessionGroups();
+
+    return createConfigFile('agents', { sessionGroups, sessions: agents });
+  };
+
+  /**
+   * export all sessions
+   */
+  exportSessions = async () => {
+    const sessions = await sessionService.getSessionsByType();
+    const sessionGroups = await sessionService.getSessionGroups();
+    const messages = await messageService.getAllMessages();
+    const topics = await topicService.getAllTopics();
+
+    return createConfigFile('sessions', { messages, sessionGroups, sessions, topics });
+  };
+
+  /**
+   * export a session
+   */
+  exportSingleSession = async (id: string) => {
+    const session = this.getSession(id);
+    if (!session) return;
+
+    const messages = await messageService.getAllMessagesInSession(id);
+    const topics = await topicService.getTopics({ sessionId: id });
+
+    const config = createConfigFile('singleSession', { messages, sessions: [session], topics });
+    return { config, title: `${session.meta?.title}-session` };
+  };
+
+  exportSingleAgent = async (id: string) => {
+    const agent = this.getAgent(id);
+    if (!agent) return;
+
+    const config = createConfigFile('agents', { sessionGroups: [], sessions: [agent] });
+
+    return { config, title: `${agent.meta?.title}-session` };
+  };
+
+  /**
+   * export settings
+   */
+  exportSettings = async () => {
+    const settings = this.getSettings();
+
+    return createConfigFile('settings', { settings });
+  };
+
+  /**
+   * export all data
+   */
+  exportAll = async () => {
+    const sessions = await sessionService.getSessionsByType();
+    const sessionGroups = await sessionService.getSessionGroups();
+    const messages = await messageService.getAllMessages();
+    const topics = await topicService.getAllTopics();
+    const settings = this.getSettings();
+
+    return createConfigFile('all', { messages, sessionGroups, sessions, settings, topics });
+  };
+
+  private getSettings = () => settingsSelectors.exportSettings(useUserStore.getState());
+
+  private getSession = (id: string) =>
+    sessionSelectors.getSessionById(id)(useSessionStore.getState());
+
+  private getAgent = (id: string) =>
+    sessionSelectors.getSessionById(id)(useSessionStore.getState());
+}
+
+/**
+ * @deprecated
+ */
+export const configService = new ConfigService();

package/src/services/export/client.ts

@@ -0,0 +1,15 @@
+import { clientDB } from '@/database/client/db';
+import { DataExporterRepos } from '@/database/repositories/dataExporter';
+import { BaseClientService } from '@/services/baseClientService';
+
+export class ClientService extends BaseClientService {
+  private get dataExporterRepos(): DataExporterRepos {
+    return new DataExporterRepos(clientDB as any, this.userId);
+  }
+
+  exportData = async () => {
+    const data = await this.dataExporterRepos.export();
+
+    return { data, url: undefined };
+  };
+}