@lobehub/chat 1.76.0 → 1.77.0

package/src/locales/default/setting.ts

@@ -348,6 +348,33 @@ export default {
     },
     title: '主题设置',
   },
+  storage: {
+    actions: {
+      export: {
+        button: '导出',
+        exportType: {
+          agent: '导出助手设定',
+          agentWithMessage: '导出助手和消息',
+          all: '导出全局设置和所有助手数据',
+          allAgent: '导出所有助手设定',
+          allAgentWithMessage: '导出所有助手和消息',
+          globalSetting: '导出全局设置',
+        },
+        title: '导出数据',
+      },
+      import: {
+        button: '导入',
+        title: '导入数据',
+      },
+      title: '高级操作',
+    },
+    desc: '当前浏览器中的存储用量',
+    embeddings: {
+      used: '向量存储',
+    },
+    title: '数据存储',
+    used: '存储用量',
+  },
   submitAgentModal: {
     button: '提交助手',
     identifier: 'identifier 助手标识符',
@@ -441,6 +468,7 @@ export default {
     'hotkey': '快捷键',
     'llm': '语言模型',
     'provider': 'AI 服务商',
+    'storage': '数据存储',
     'sync': '云端同步',
     'system-agent': '系统助手',
     'tts': '语音服务',

package/src/server/routers/lambda/exporter.ts

@@ -0,0 +1,25 @@
+import { DrizzleMigrationModel } from '@/database/models/drizzleMigration';
+import { DataExporterRepos } from '@/database/repositories/dataExporter';
+import { serverDB } from '@/database/server';
+import { authedProcedure, router } from '@/libs/trpc';
+import { ExportDatabaseData } from '@/types/export';
+
+const exportProcedure = authedProcedure.use(async (opts) => {
+  const { ctx } = opts;
+  const dataExporterRepos = new DataExporterRepos(serverDB, ctx.userId);
+  const drizzleMigration = new DrizzleMigrationModel(serverDB);
+
+  return opts.next({
+    ctx: { dataExporterRepos, drizzleMigration },
+  });
+});
+
+export const exporterRouter = router({
+  exportData: exportProcedure.mutation(async ({ ctx }): Promise<ExportDatabaseData> => {
+    const data = await ctx.dataExporterRepos.export(5);
+
+    const schemaHash = await ctx.drizzleMigration.getLatestMigrationHash();
+
+    return { data, schemaHash };
+  }),
+});

package/src/server/routers/lambda/importer.ts

@@ -5,7 +5,8 @@ import { DataImporterRepos } from '@/database/repositories/dataImporter';
 import { serverDB } from '@/database/server';
 import { authedProcedure, router } from '@/libs/trpc';
 import { S3 } from '@/server/modules/S3';
-import { ImportResults, ImporterEntryData } from '@/types/importer';
+import { ImportPgDataStructure } from '@/types/export';
+import { ImportResultData, ImporterEntryData } from '@/types/importer';
 
 const importProcedure = authedProcedure.use(async (opts) => {
   const { ctx } = opts;
@@ -19,7 +20,7 @@ const importProcedure = authedProcedure.use(async (opts) => {
 export const importerRouter = router({
   importByFile: importProcedure
     .input(z.object({ pathname: z.string() }))
-    .mutation(async ({ input, ctx }): Promise<ImportResults> => {
+    .mutation(async ({ input, ctx }): Promise<ImportResultData> => {
       let data: ImporterEntryData | undefined;
 
       try {
@@ -37,6 +38,10 @@ export const importerRouter = router({
         });
       }
 
+      if ('schemaHash' in data) {
+        return ctx.dataImporterService.importPgData(data as unknown as ImportPgDataStructure);
+      }
+
       return ctx.dataImporterService.importData(data);
     }),
 
@@ -52,7 +57,18 @@ export const importerRouter = router({
         }),
       }),
     )
-    .mutation(async ({ input, ctx }): Promise<ImportResults> => {
+    .mutation(async ({ input, ctx }): Promise<ImportResultData> => {
       return ctx.dataImporterService.importData(input.data);
     }),
+  importPgByPost: importProcedure
+    .input(
+      z.object({
+        data: z.record(z.string(), z.array(z.any())),
+        mode: z.enum(['pglite', 'postgres']),
+        schemaHash: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }): Promise<ImportResultData> => {
+      return ctx.dataImporterService.importPgData(input);
+    }),
 });

package/src/server/routers/lambda/index.ts

@@ -7,6 +7,7 @@ import { agentRouter } from './agent';
 import { aiModelRouter } from './aiModel';
 import { aiProviderRouter } from './aiProvider';
 import { chunkRouter } from './chunk';
+import { exporterRouter } from './exporter';
 import { fileRouter } from './file';
 import { importerRouter } from './importer';
 import { knowledgeBaseRouter } from './knowledgeBase';
@@ -24,6 +25,7 @@ export const lambdaRouter = router({
   aiModel: aiModelRouter,
   aiProvider: aiProviderRouter,
   chunk: chunkRouter,
+  exporter: exporterRouter,
   file: fileRouter,
   healthcheck: publicProcedure.query(() => "i'm live!"),
   importer: importerRouter,

package/src/services/config.ts

@@ -1,162 +1,107 @@
-import { importService } from '@/services/import';
-import { messageService } from '@/services/message';
-import { sessionService } from '@/services/session';
-import { topicService } from '@/services/topic';
-import { useSessionStore } from '@/store/session';
-import { sessionSelectors } from '@/store/session/selectors';
-import { useUserStore } from '@/store/user';
-import { settingsSelectors } from '@/store/user/selectors';
-import { ConfigFile } from '@/types/exportConfig';
-import { ImportStage, OnImportCallbacks } from '@/types/importer';
-import { createConfigFile, exportConfigFile } from '@/utils/config';
-
-export interface ImportResult {
-  added: number;
-  errors: number;
-  skips: number;
-}
-export interface ImportResults {
-  messages?: ImportResult;
-  sessionGroups?: ImportResult;
-  sessions?: ImportResult;
-  topics?: ImportResult;
-  type?: string;
-}
+import dayjs from 'dayjs';
+
+import { BRANDING_NAME } from '@/const/branding';
+import { isDeprecatedEdition, isServerMode } from '@/const/version';
+import { CURRENT_CONFIG_VERSION } from '@/migrations';
+import { ImportPgDataStructure } from '@/types/export';
+import { downloadFile } from '@/utils/client/downloadFile';
+import { exportJSONFile } from '@/utils/client/exportFile';
+
+import { exportService } from './export';
+import { configService as deprecatedExportService } from './export/_deprecated';
 
-/**
- * @deprecated
- */
 class ConfigService {
-  importConfigState = async (config: ConfigFile, callbacks?: OnImportCallbacks): Promise<void> => {
-    if (config.exportType === 'settings') {
-      await importService.importSettings(config.state.settings);
-      callbacks?.onStageChange?.(ImportStage.Success);
+  exportAll = async () => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const config = await deprecatedExportService.exportAll();
+      const filename = `${BRANDING_NAME}-config-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(config, filename);
       return;
     }
 
-    if (config.exportType === 'all') {
-      await importService.importSettings(config.state.settings);
-    }
-
-    await importService.importData(
-      {
-        messages: (config.state as any).messages || [],
-        sessionGroups: (config.state as any).sessionGroups || [],
-        sessions: (config.state as any).sessions || [],
-        topics: (config.state as any).topics || [],
-        version: config.version,
-      },
-      callbacks,
-    );
-  };
-
-  // TODO: Separate export feature into a new service like importService
-
-  /**
-   * export all agents
-   */
-  exportAgents = async () => {
-    const agents = await sessionService.getSessionsByType('agent');
-    const sessionGroups = await sessionService.getSessionGroups();
-
-    const config = createConfigFile('agents', { sessionGroups, sessions: agents });
-
-    exportConfigFile(config, 'agents');
-  };
+    const { data, url } = await exportService.exportData();
+    const filename = `${dayjs().format('YYYY-MM-DD-hh-mm')}_${BRANDING_NAME}-data.json`;
 
-  /**
-   * export all sessions
-   */
-  exportSessions = async () => {
-    const sessions = await sessionService.getSessionsByType();
-    const sessionGroups = await sessionService.getSessionGroups();
-    const messages = await messageService.getAllMessages();
-    const topics = await topicService.getAllTopics();
+    // if url exists, means export data from server and upload the data to S3
+    // just need to download the file
+    if (url) {
+      await downloadFile(url, filename);
+      return;
+    }
 
-    const config = createConfigFile('sessions', { messages, sessionGroups, sessions, topics });
+    // or export to file with the data
+    const result = await this.createDataStructure(data, isServerMode ? 'postgres' : 'pglite');
 
-    exportConfigFile(config, 'sessions');
+    exportJSONFile(result, filename);
   };
 
-  /**
-   * export a session
-   */
-  exportSingleSession = async (id: string) => {
-    const session = this.getSession(id);
-    if (!session) return;
-
-    const messages = await messageService.getAllMessagesInSession(id);
-    const topics = await topicService.getTopics({ sessionId: id });
-
-    const config = createConfigFile('singleSession', { messages, sessions: [session], topics });
-
-    exportConfigFile(config, `${session.meta?.title}-session`);
+  exportAgents = async () => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const config = await deprecatedExportService.exportAgents();
+      const filename = `${BRANDING_NAME}-agents-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(config, filename);
+      return;
+    }
   };
 
-  /**
-   * export a topic
-   */
-  exportSingleTopic = async (sessionId: string, topicId: string) => {
-    const session = this.getSession(sessionId);
-    if (!session) return;
-
-    const messages = await messageService.getMessages(sessionId, topicId);
-    const topics = await topicService.getTopics({ sessionId });
-
-    const topic = topics.find((item) => item.id === topicId);
-    if (!topic) return;
+  exportSingleAgent = async (agentId: string) => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const result = await deprecatedExportService.exportSingleAgent(agentId);
+      if (!result) return;
 
-    const config = createConfigFile('singleSession', {
-      messages,
-      sessions: [session],
-      topics: [topic],
-    });
-
-    exportConfigFile(config, `${topic.title}-topic`);
+      const filename = `${BRANDING_NAME}-${result.title}-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(result.config, filename);
+      return;
+    }
   };
 
-  exportSingleAgent = async (id: string) => {
-    const agent = this.getAgent(id);
-    if (!agent) return;
-
-    const config = createConfigFile('agents', { sessionGroups: [], sessions: [agent] });
-
-    exportConfigFile(config, agent.meta?.title || 'agent');
+  exportSessions = async () => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const config = await deprecatedExportService.exportSessions();
+      const filename = `${BRANDING_NAME}-sessions-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(config, filename);
+      return;
+    }
   };
 
-  /**
-   * export settings
-   */
   exportSettings = async () => {
-    const settings = this.getSettings();
-
-    const config = createConfigFile('settings', { settings });
-
-    exportConfigFile(config, 'settings');
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const config = await deprecatedExportService.exportSessions();
+      const filename = `${BRANDING_NAME}-settings-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(config, filename);
+      return;
+    }
   };
 
-  /**
-   * export all data
-   */
-  exportAll = async () => {
-    const sessions = await sessionService.getSessionsByType();
-    const sessionGroups = await sessionService.getSessionGroups();
-    const messages = await messageService.getAllMessages();
-    const topics = await topicService.getAllTopics();
-    const settings = this.getSettings();
-
-    const config = createConfigFile('all', { messages, sessionGroups, sessions, settings, topics });
+  exportSingleSession = async (sessionId: string) => {
+    // TODO: remove this in V2
+    if (isDeprecatedEdition) {
+      const data = await deprecatedExportService.exportSingleSession(sessionId);
+      if (!data) return;
 
-    exportConfigFile(config, 'config');
+      const filename = `${BRANDING_NAME}-${data.title}-v${CURRENT_CONFIG_VERSION}.json`;
+      exportJSONFile(data.config, filename);
+      return;
+    }
   };
 
-  private getSettings = () => settingsSelectors.exportSettings(useUserStore.getState());
-
-  private getSession = (id: string) =>
-    sessionSelectors.getSessionById(id)(useSessionStore.getState());
+  private createDataStructure = async (
+    data: any,
+    mode: 'pglite' | 'postgres',
+  ): Promise<ImportPgDataStructure> => {
+    const { default: json } = await import('@/database/client/migrations.json');
+    const latestHash = json.at(-1)?.hash;
+    if (!latestHash) {
+      throw new Error('Not find database sql hash');
+    }
 
-  private getAgent = (id: string) =>
-    sessionSelectors.getSessionById(id)(useSessionStore.getState());
+    return { data, mode, schemaHash: latestHash };
+  };
 }
 
 export const configService = new ConfigService();

package/src/services/export/_deprecated.ts

@@ -0,0 +1,155 @@
+import { Migration } from '@/migrations';
+import { messageService } from '@/services/message';
+import { sessionService } from '@/services/session';
+import { topicService } from '@/services/topic';
+import { useSessionStore } from '@/store/session';
+import { sessionSelectors } from '@/store/session/selectors';
+import { useUserStore } from '@/store/user';
+import { settingsSelectors } from '@/store/user/selectors';
+import {
+  ConfigFileAgents,
+  ConfigFileAll,
+  ConfigFileSessions,
+  ConfigFileSettings,
+  ConfigFileSingleSession,
+  ConfigModelMap,
+  ExportType,
+} from '@/types/exportConfig';
+
+type CreateConfigFileState<T extends ExportType> = ConfigModelMap[T]['state'];
+
+type CreateConfigFile<T extends ExportType> = ConfigModelMap[T]['file'];
+
+const createConfigFile = <T extends ExportType>(
+  type: T,
+  state: CreateConfigFileState<T>,
+): CreateConfigFile<T> => {
+  switch (type) {
+    case 'agents': {
+      return {
+        exportType: 'agents',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileAgents;
+    }
+
+    case 'sessions': {
+      return {
+        exportType: 'sessions',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileSessions;
+    }
+
+    case 'settings': {
+      return {
+        exportType: 'settings',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileSettings;
+    }
+
+    case 'singleSession': {
+      return {
+        exportType: 'sessions',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileSingleSession;
+    }
+
+    case 'all': {
+      return {
+        exportType: 'all',
+        state,
+        version: Migration.targetVersion,
+      } as ConfigFileAll;
+    }
+  }
+
+  throw new Error('缺少正确的导出类型，请检查实现...');
+};
+
+/**
+ * @deprecated
+ */
+export class ConfigService {
+  /**
+   * export all agents
+   */
+  exportAgents = async () => {
+    const agents = await sessionService.getSessionsByType('agent');
+    const sessionGroups = await sessionService.getSessionGroups();
+
+    return createConfigFile('agents', { sessionGroups, sessions: agents });
+  };
+
+  /**
+   * export all sessions
+   */
+  exportSessions = async () => {
+    const sessions = await sessionService.getSessionsByType();
+    const sessionGroups = await sessionService.getSessionGroups();
+    const messages = await messageService.getAllMessages();
+    const topics = await topicService.getAllTopics();
+
+    return createConfigFile('sessions', { messages, sessionGroups, sessions, topics });
+  };
+
+  /**
+   * export a session
+   */
+  exportSingleSession = async (id: string) => {
+    const session = this.getSession(id);
+    if (!session) return;
+
+    const messages = await messageService.getAllMessagesInSession(id);
+    const topics = await topicService.getTopics({ sessionId: id });
+
+    const config = createConfigFile('singleSession', { messages, sessions: [session], topics });
+    return { config, title: `${session.meta?.title}-session` };
+  };
+
+  exportSingleAgent = async (id: string) => {
+    const agent = this.getAgent(id);
+    if (!agent) return;
+
+    const config = createConfigFile('agents', { sessionGroups: [], sessions: [agent] });
+
+    return { config, title: `${agent.meta?.title}-session` };
+  };
+
+  /**
+   * export settings
+   */
+  exportSettings = async () => {
+    const settings = this.getSettings();
+
+    return createConfigFile('settings', { settings });
+  };
+
+  /**
+   * export all data
+   */
+  exportAll = async () => {
+    const sessions = await sessionService.getSessionsByType();
+    const sessionGroups = await sessionService.getSessionGroups();
+    const messages = await messageService.getAllMessages();
+    const topics = await topicService.getAllTopics();
+    const settings = this.getSettings();
+
+    return createConfigFile('all', { messages, sessionGroups, sessions, settings, topics });
+  };
+
+  private getSettings = () => settingsSelectors.exportSettings(useUserStore.getState());
+
+  private getSession = (id: string) =>
+    sessionSelectors.getSessionById(id)(useSessionStore.getState());
+
+  private getAgent = (id: string) =>
+    sessionSelectors.getSessionById(id)(useSessionStore.getState());
+}
+
+/**
+ * @deprecated
+ */
+export const configService = new ConfigService();

package/src/services/export/client.ts

@@ -0,0 +1,15 @@
+import { clientDB } from '@/database/client/db';
+import { DataExporterRepos } from '@/database/repositories/dataExporter';
+import { BaseClientService } from '@/services/baseClientService';
+
+export class ClientService extends BaseClientService {
+  private get dataExporterRepos(): DataExporterRepos {
+    return new DataExporterRepos(clientDB as any, this.userId);
+  }
+
+  exportData = async () => {
+    const data = await this.dataExporterRepos.export();
+
+    return { data, url: undefined };
+  };
+}

package/src/services/export/index.ts

@@ -0,0 +1,6 @@
+import { isServerMode } from '@/const/version';
+
+import { ClientService } from './client';
+import { ServerService } from './server';
+
+export const exportService = isServerMode ? new ServerService() : new ClientService();

package/src/services/export/server.ts

@@ -0,0 +1,9 @@
+import { lambdaClient } from '@/libs/trpc/client';
+
+import { IExportService } from './type';
+
+export class ServerService implements IExportService {
+  exportData: IExportService['exportData'] = async () => {
+    return await lambdaClient.exporter.exportData.mutate();
+  };
+}

package/src/services/export/type.ts

@@ -0,0 +1,5 @@
+import { ExportDatabaseData } from '@/types/export';
+
+export interface IExportService {
+  exportData(): Promise<ExportDatabaseData>;
+}

package/src/services/import/_deprecated.ts

@@ -2,11 +2,25 @@ import { MessageModel } from '@/database/_deprecated/models/message';
 import { SessionModel } from '@/database/_deprecated/models/session';
 import { SessionGroupModel } from '@/database/_deprecated/models/sessionGroup';
 import { TopicModel } from '@/database/_deprecated/models/topic';
-import { ImportResult, ImportResults } from '@/services/config';
 import { useUserStore } from '@/store/user';
+import { ConfigFile } from '@/types/exportConfig';
 import { ImportStage, ImporterEntryData, OnImportCallbacks } from '@/types/importer';
 import { UserSettings } from '@/types/user/settings';
 
+export interface ImportResult {
+  added: number;
+  errors: number;
+  skips: number;
+  updated?: number;
+}
+export interface ImportResults {
+  messages?: ImportResult;
+  sessionGroups?: ImportResult;
+  sessions?: ImportResult;
+  topics?: ImportResult;
+  type?: string;
+}
+
 export class ClientService {
   importSettings = async (settings: UserSettings) => {
     await useUserStore.getState().importAppSettings(settings);
@@ -71,4 +85,31 @@ export class ClientService {
       skips: input.skips.length,
     };
   };
+
+  importConfigState = async (config: ConfigFile, callbacks?: OnImportCallbacks): Promise<void> => {
+    if (config.exportType === 'settings') {
+      await this.importSettings(config.state.settings);
+      callbacks?.onStageChange?.(ImportStage.Success);
+      return;
+    }
+
+    if (config.exportType === 'all') {
+      await this.importSettings(config.state.settings);
+    }
+
+    await this.importData(
+      {
+        messages: (config.state as any).messages || [],
+        sessionGroups: (config.state as any).sessionGroups || [],
+        sessions: (config.state as any).sessions || [],
+        topics: (config.state as any).topics || [],
+        version: config.version,
+      },
+      callbacks,
+    );
+  };
+
+  importPgData = async () => {
+    throw new Error('Not implemented');
+  };
 }

package/src/services/import/client.test.ts

@@ -2,7 +2,7 @@ import { eq, inArray } from 'drizzle-orm';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { clientDB, initializeDB } from '@/database/client/db';
-import mockImportData from '@/database/repositories/dataImporter/__tests__/fixtures/messages.json';
+import mockImportData from '@/database/repositories/dataImporter/deprecated/__tests__/fixtures/messages.json';
 import {
   agents,
   agentsToSessions,