@lobehub/chat 0.106.0 → 0.107.0

package/src/app/settings/common/Common.tsx

@@ -10,6 +10,8 @@ import { FORM_STYLE } from '@/const/layoutTokens';
 import { DEFAULT_SETTINGS } from '@/const/settings';
 import AvatarWithUpload from '@/features/AvatarWithUpload';
 import { localeOptions } from '@/locales/options';
+import { useChatStore } from '@/store/chat';
+import { useFileStore } from '@/store/files';
 import { settingsSelectors, useGlobalStore } from '@/store/global';
 import { usePluginStore } from '@/store/plugin';
 import { useSessionStore } from '@/store/session';
@@ -22,7 +24,13 @@ type SettingItemGroup = ItemGroup;
 const Common = memo(() => {
   const { t } = useTranslation('setting');
   const [form] = AntForm.useForm();
+
   const clearSessions = useSessionStore((s) => s.clearSessions);
+  const [clearTopics, clearAllMessages] = useChatStore((s) => [
+    s.removeAllTopics,
+    s.clearAllMessages,
+  ]);
+  const [removeAllFiles] = useFileStore((s) => [s.removeAllFiles]);
   const resetPluginSettings = usePluginStore((s) => s.resetPluginSettings);
 
   const settings = useGlobalStore(settingsSelectors.currentSettings, isEqual);
@@ -56,9 +64,13 @@ const Common = memo(() => {
         danger: true,
       },
       okText: t('ok', { ns: 'common' }),
-      onOk: () => {
-        clearSessions();
+      onOk: async () => {
+        await clearSessions();
         resetPluginSettings();
+        await clearTopics();
+        await removeAllFiles();
+        await clearAllMessages();
+
         message.success(t('danger.clear.success'));
       },
       title: t('danger.clear.confirm'),
@@ -161,8 +173,8 @@ const Common = memo(() => {
             {t('danger.clear.action')}
           </Button>
         ),
-        desc: t('danger.clear.title'),
-        label: t('danger.clear.desc'),
+        desc: t('danger.clear.desc'),
+        label: t('danger.clear.title'),
         minWidth: undefined,
       },
     ],

package/src/app/welcome/features/Banner/index.tsx

@@ -1,20 +1,19 @@
 'use client';
 
 import { Icon } from '@lobehub/ui';
-import { Button, Upload } from 'antd';
+import { Button } from 'antd';
 import { SendHorizonal } from 'lucide-react';
-import { memo, useCallback } from 'react';
+import { memo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { Flexbox } from 'react-layout-kit';
 
-import { useImportConfig } from '@/hooks/useImportConfig';
+import DataImporter from '@/features/DataImporter';
 import { useSessionStore } from '@/store/session';
 
 import Hero from './Hero';
 import { useStyles } from './style';
 
 const Banner = memo<{ mobile?: boolean }>(({ mobile }) => {
-  const { importConfig } = useImportConfig();
   const { t } = useTranslation('welcome');
   const { styles } = useStyles();
   const [switchSession, switchBackToChat, router, isMobile] = useSessionStore((s) => [
@@ -23,10 +22,6 @@ const Banner = memo<{ mobile?: boolean }>(({ mobile }) => {
     s.router,
     s.isMobile,
   ]);
-  const handleImport = useCallback((e: any) => {
-    importConfig(e);
-    switchSession();
-  }, []);
 
   return (
     <>
@@ -40,11 +35,15 @@ const Banner = memo<{ mobile?: boolean }>(({ mobile }) => {
         justify={'center'}
         width={'100%'}
       >
-        <Upload maxCount={1} onChange={handleImport} showUploadList={false}>
+        <DataImporter
+          onFinishImport={() => {
+            switchSession();
+          }}
+        >
           <Button block={mobile} size={'large'}>
             {t('button.import')}
           </Button>
-        </Upload>
+        </DataImporter>
         <Button
           block={mobile}
           onClick={() => (isMobile ? router?.push('/chat') : switchBackToChat())}

package/src/chains/summaryTitle.ts

@@ -1,7 +1,10 @@
+import { chatHelpers } from '@/store/chat/helpers';
 import { getCurrentLanguage } from '@/store/global/helpers';
-import { chatHelpers } from '@/store/session/slices/chat/helpers';
 import { LanguageModel } from '@/types/llm';
-import { OpenAIChatMessage, OpenAIChatStreamPayload } from '@/types/openai/chat';
+import {
+  OpenAIChatMessage,
+  OpenAIChatStreamPayload,
+} from '@/types/openai/chat';
 
 export const chainSummaryTitle = async (
   messages: OpenAIChatMessage[],

package/src/components/DataStyleModal/index.tsx

@@ -0,0 +1,64 @@
+import { Icon, Modal } from '@lobehub/ui';
+import { createStyles } from 'antd-style';
+import { LucideIcon } from 'lucide-react';
+import { rgba } from 'polished';
+import { ReactNode, memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+const useStyles = createStyles(({ css, token, prefixCls, isDarkMode }) => ({
+  modalTitle: css`
+    &.${prefixCls}-modal-header {
+      height: 80px;
+      background:
+        linear-gradient(
+          180deg,
+          ${rgba(token.colorBgElevated, 0)},
+          ${token.colorBgContainer} ${isDarkMode ? '80' : '140'}px
+        ),
+        fixed 0 0 /10px 10px radial-gradient(${token.colorFill} 1px, transparent 0);
+    }
+
+    & .${prefixCls}-modal-title {
+      font-size: 24px;
+    }
+  `,
+}));
+
+interface DataStyleModalProps {
+  children: ReactNode;
+  icon: LucideIcon;
+  onOpenChange?: (open: boolean) => void;
+  open: boolean;
+  title: string;
+  width?: number;
+}
+
+const DataStyleModal = memo<DataStyleModalProps>(
+  ({ icon, onOpenChange, title, open, children, width = 550 }) => {
+    const { styles } = useStyles();
+
+    return (
+      <Modal
+        afterOpenChange={onOpenChange}
+        centered
+        classNames={{
+          header: styles.modalTitle,
+        }}
+        closable={false}
+        footer={null}
+        open={open}
+        title={
+          <Flexbox gap={8} horizontal>
+            <Icon icon={icon} />
+            {title}
+          </Flexbox>
+        }
+        width={width}
+      >
+        {children}
+      </Modal>
+    );
+  },
+);
+
+export default DataStyleModal;

package/src/config/client.ts

@@ -28,6 +28,8 @@ declare global {
       NEXT_PUBLIC_I18N_DEBUG: string;
       NEXT_PUBLIC_I18N_DEBUG_BROWSER: string;
       NEXT_PUBLIC_I18N_DEBUG_SERVER: string;
+
+      NEXT_PUBLIC_DEVELOPER_DEBUG: string;
     }
   }
 }
@@ -63,4 +65,7 @@ export const getClientConfig = () => ({
   I18N_DEBUG: process.env.NEXT_PUBLIC_I18N_DEBUG === '1',
   I18N_DEBUG_BROWSER: process.env.NEXT_PUBLIC_I18N_DEBUG_BROWSER === '1',
   I18N_DEBUG_SERVER: process.env.NEXT_PUBLIC_I18N_DEBUG_SERVER === '1',
+
+  // developer debug mode
+  DEBUG_MODE: process.env.NEXT_PUBLIC_DEVELOPER_DEBUG === '1',
 });

package/src/const/session.ts

@@ -1 +1,22 @@
+import { DEFAULT_AGENT_META, DEFAULT_INBOX_AVATAR } from '@/const/meta';
+import { DEFAULT_AGENT_CONFIG } from '@/const/settings';
+import { LobeAgentSession, LobeSessionType } from '@/types/session';
+import { merge } from '@/utils/merge';
+
 export const INBOX_SESSION_ID = 'inbox';
+
+export const DEFAULT_AGENT_LOBE_SESSION: LobeAgentSession = {
+  config: DEFAULT_AGENT_CONFIG,
+  createdAt: Date.now(),
+  id: '',
+  meta: DEFAULT_AGENT_META,
+  type: LobeSessionType.Agent,
+  updatedAt: Date.now(),
+};
+
+export const DEFAULT_INBOX_SESSION: LobeAgentSession = merge(DEFAULT_AGENT_LOBE_SESSION, {
+  id: 'inbox',
+  meta: {
+    avatar: DEFAULT_INBOX_AVATAR,
+  },
+});

package/src/const/settings.ts

@@ -1,8 +1,8 @@
 import { getClientConfig } from '@/config/client';
 import { DEFAULT_OPENAI_MODEL_LIST } from '@/const/llm';
 import { DEFAULT_AGENT_META } from '@/const/meta';
+import { LobeAgentConfig, LobeAgentTTSConfig } from '@/types/agent';
 import { LanguageModel } from '@/types/llm';
-import { LobeAgentConfig, LobeAgentTTSConfig } from '@/types/session';
 import {
   GlobalBaseSettings,
   GlobalDefaultAgent,

package/src/database/core/__tests__/db.test.ts

@@ -1,7 +1,7 @@
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 
-import { DBModel } from '@/types/database/db';
-import { LocalFile } from '@/types/database/files';
+import { DBModel } from '@/database/core/types/db';
+import { DB_File } from '@/database/schemas/files';
 
 import { LocalDB } from '../db';
 
@@ -24,7 +24,7 @@ describe('LocalDB', () => {
   });
 
   it('should allow adding a file', async () => {
-    const file: DBModel<LocalFile> = {
+    const file: DBModel<DB_File> = {
       id: 'file1',
       name: 'testfile.txt',
       data: new ArrayBuffer(3),
@@ -32,6 +32,7 @@ describe('LocalDB', () => {
       fileType: 'plain/text',
       size: 3,
       createdAt: Date.now(),
+      updatedAt: Date.now(),
     };
 
     await db.files.add(file);
@@ -40,7 +41,7 @@ describe('LocalDB', () => {
   });
 
   it('should allow updating a file', async () => {
-    const file: DBModel<LocalFile> = {
+    const file: DBModel<DB_File> = {
       id: 'file1',
       name: 'testfile.txt',
       data: new ArrayBuffer(3),
@@ -48,6 +49,7 @@ describe('LocalDB', () => {
       fileType: 'plain/text',
       size: 3,
       createdAt: Date.now(),
+      updatedAt: Date.now(),
     };
     await db.files.add(file);
 
@@ -59,13 +61,14 @@ describe('LocalDB', () => {
   });
 
   it('should allow deleting a file', async () => {
-    const file: DBModel<LocalFile> = {
+    const file: DBModel<DB_File> = {
       id: 'file1',
       name: 'testfile.txt',
       data: new ArrayBuffer(3),
       saveMode: 'local',
       fileType: 'plain/text',
       size: 3,
+      updatedAt: Date.now(),
       createdAt: Date.now(),
     };
     await db.files.add(file);

package/src/database/core/__tests__/model.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
-import { ZodSchema, z } from 'zod';
+import { z } from 'zod';
 
 import { BaseModel } from '../model';
 
@@ -37,7 +37,7 @@ describe('BaseModel', () => {
         content: 'Hello, World!',
       };
 
-      const result = await baseModel['add'](validData);
+      const result = await baseModel['_add'](validData);
 
       expect(result).toHaveProperty('id');
       expect(console.error).not.toHaveBeenCalled();
@@ -49,7 +49,7 @@ describe('BaseModel', () => {
         content: 'Hello, World!',
       };
 
-      await expect(baseModel['add'](invalidData)).rejects.toThrow(TypeError);
+      await expect(baseModel['_add'](invalidData)).rejects.toThrow(TypeError);
     });
   });
 });

package/src/database/core/db.ts

@@ -1,25 +1,53 @@
 import Dexie from 'dexie';
 
-import { LobeDBSchemaMap as SchemaMap, lobeDBSchema } from '@/database/core/schema';
-import { DBModel } from '@/types/database/db';
+import { DB_Message } from '@/database/schemas/message';
+import { DB_Session } from '@/database/schemas/session';
+import { DB_Topic } from '@/database/schemas/topic';
+import { DBModel } from '@/database/core/types/db';
+import { DB_File } from '@/database/schemas/files';
 
-export type LocalDBSchema = {
-  [t in keyof SchemaMap]: {
-    model: SchemaMap[t];
-    table: Dexie.Table<DBModel<SchemaMap[t]>, string>;
-  };
-};
+import { dbSchemaV1, dbSchemaV2 } from './schemas';
+
+interface LobeDBSchemaMap {
+  files: DB_File;
+  messages: DB_Message;
+  sessions: DB_Session;
+  topics: DB_Topic;
+}
 
 // Define a local DB
 export class LocalDB extends Dexie {
-  public files: LocalDBSchema['files']['table'];
+  public files: LobeDBTable<'files'>;
+  public sessions: LobeDBTable<'sessions'>;
+  public messages: LobeDBTable<'messages'>;
+  public topics: LobeDBTable<'topics'>;
 
   constructor() {
     super('LOBE_CHAT_DB');
-    this.version(1).stores(lobeDBSchema);
+    this.version(1).stores(dbSchemaV1);
+
+    this.version(2).stores(dbSchemaV2);
 
     this.files = this.table('files');
+    this.sessions = this.table('sessions');
+    this.messages = this.table('messages');
+    this.topics = this.table('topics');
   }
 }
 
 export const LocalDBInstance = new LocalDB();
+
+// ================================================ //
+// ================================================ //
+// ================================================ //
+// ================================================ //
+// ================================================ //
+
+// types helper
+export type LocalDBSchema = {
+  [t in keyof LobeDBSchemaMap]: {
+    model: LobeDBSchemaMap[t];
+    table: Dexie.Table<DBModel<LobeDBSchemaMap[t]>, string>;
+  };
+};
+type LobeDBTable<T extends keyof LobeDBSchemaMap> = LocalDBSchema[T]['table'];

package/src/database/core/index.ts

@@ -1 +1,2 @@
+export { LocalDBInstance } from './db';
 export * from './model';

package/src/database/core/model.ts

@@ -1,24 +1,24 @@
-import { ZodSchema } from 'zod';
+import Dexie, { BulkError } from 'dexie';
+import { ZodObject } from 'zod';
 
-import { DBModel } from '@/types/database/db';
-import { LocalFile } from '@/types/database/files';
+import { DBBaseFieldsSchema } from '@/database/core/types/db';
 import { nanoid } from '@/utils/uuid';
 
 import { LocalDB, LocalDBInstance, LocalDBSchema } from './db';
 
-export class BaseModel<N extends keyof LocalDBSchema = any> {
-  private readonly db: LocalDB;
-  private readonly schema: ZodSchema;
+export class BaseModel<N extends keyof LocalDBSchema = any, T = LocalDBSchema[N]['table']> {
+  protected readonly db: LocalDB;
+  private readonly schema: ZodObject<any>;
   private readonly _tableName: keyof LocalDBSchema;
 
-  constructor(table: N, schema: ZodSchema, db = LocalDBInstance) {
+  constructor(table: N, schema: ZodObject<any>, db = LocalDBInstance) {
     this.db = db;
     this.schema = schema;
     this._tableName = table;
   }
 
   get table() {
-    return this.db[this._tableName];
+    return this.db[this._tableName] as Dexie.Table;
   }
 
   /**
@@ -26,7 +26,7 @@ export class BaseModel<N extends keyof LocalDBSchema = any> {
    * @param data
    * @param id
    */
-  protected async add<T = LocalDBSchema[N]['model']>(data: T, id: string | number = nanoid()) {
+  protected async _add<T = LocalDBSchema[N]['model']>(data: T, id: string | number = nanoid()) {
     const result = this.schema.safeParse(data);
 
     if (!result.success) {
@@ -40,10 +40,121 @@ export class BaseModel<N extends keyof LocalDBSchema = any> {
 
     const tableName = this._tableName;
 
-    const record: DBModel<LocalFile> = { ...result.data, createdAt: Date.now(), id };
+    const record: any = { ...result.data, createdAt: Date.now(), id, updatedAt: Date.now() };
 
     const newId = await this.db[tableName].add(record);
 
     return { id: newId };
   }
+
+  /**
+   * Batch create new records
+   * @param dataArray An array of data to be added
+   * @param options
+   * @param options.generateId
+   * @param options.createWithNewId
+   */
+  protected async _batchAdd<T = LocalDBSchema[N]['model']>(
+    dataArray: T[],
+    options: {
+      /**
+       * always create with a new id
+       */
+      createWithNewId?: boolean;
+      idGenerator?: () => string;
+    } = {},
+  ): Promise<{
+    added: number;
+    errors?: Error[];
+    ids: string[];
+    skips: string[];
+    success: boolean;
+  }> {
+    const { idGenerator = nanoid, createWithNewId = false } = options;
+    const validatedData = [];
+    const errors = [];
+    const skips: string[] = [];
+
+    for (const data of dataArray) {
+      const schemaWithId = this.schema.merge(DBBaseFieldsSchema.partial());
+
+      const result = schemaWithId.safeParse(data);
+
+      if (result.success) {
+        const item = result.data;
+        const autoId = idGenerator();
+
+        const id = createWithNewId ? autoId : item.id ?? autoId;
+
+        // skip if the id already exists
+        if (await this.table.get(id)) {
+          skips.push(id);
+          continue;
+        }
+
+        validatedData.push({
+          ...item,
+          createdAt: item.createdAt ?? Date.now(),
+          id,
+          updatedAt: item.updatedAt ?? Date.now(),
+        });
+      } else {
+        errors.push(result.error);
+
+        const errorMsg = `[${this.db.name}][${
+          this._tableName
+        }] Failed to create the record. Data: ${JSON.stringify(data)}. Errors: ${result.error}`;
+        console.error(new TypeError(errorMsg));
+      }
+    }
+    if (validatedData.length === 0) {
+      // No valid data to add
+      return { added: 0, errors, ids: [], skips, success: false };
+    }
+
+    // Using bulkAdd to insert validated data
+    try {
+      await this.table.bulkAdd(validatedData);
+
+      return {
+        added: validatedData.length,
+        ids: validatedData.map((item) => item.id),
+        skips,
+        success: true,
+      };
+    } catch (error) {
+      const bulkError = error as BulkError;
+      // Handle bulkAdd errors here
+      console.error(`[${this.db.name}][${this._tableName}] Bulk add error:`, bulkError);
+      // Return the number of successfully added records and errors
+      return {
+        added: validatedData.length - skips.length - bulkError.failures.length,
+        errors: bulkError.failures,
+        ids: validatedData.map((item) => item.id),
+        skips,
+        success: false,
+      };
+    }
+  }
+
+  protected async _update(id: string, data: Partial<T>) {
+    // we need to check whether the data is valid
+    // pick data related schema from the full schema
+    const keys = Object.keys(data);
+    const partialSchema = this.schema.pick(Object.fromEntries(keys.map((key) => [key, true])));
+
+    const result = partialSchema.safeParse(data);
+    if (!result.success) {
+      const errorMsg = `[${this.db.name}][${this._tableName}] Failed to update the record:${id}. Error: ${result.error}`;
+
+      const newError = new TypeError(errorMsg);
+      // make this error show on console to help debug
+      console.error(newError);
+      throw newError;
+    }
+
+    const success = await this.table.update(id, { ...data, updatedAt: Date.now() });
+
+    return { success };
+  }
 }

package/src/database/core/schemas.ts

@@ -0,0 +1,22 @@
+// ************************************** //
+// ******* Version 1 - 2023-11-14 ******* //
+// ************************************** //
+// - Initial database schema with `files` table
+
+export const dbSchemaV1 = {
+  files: '&id, name, fileType, saveMode',
+};
+
+// ************************************** //
+// ******* Version 2 - 2023-11-27 ******* //
+// ************************************** //
+// - Added `sessions` 、`messages` 、`topics` tables
+// - Added `createdAt` and `updatedAt` fields to all
+export const dbSchemaV2 = {
+  files: '&id, name, fileType, saveMode, createdAt, updatedAt, messageId, sessionId',
+
+  messages:
+    '&id, role, content, fromModel, favorite, plugin.identifier, plugin.apiName, translate.content, createdAt, updatedAt, sessionId, topicId, quotaId, parentId, [sessionId+topicId]',
+  sessions: '&id, type, group, meta.title, meta.description, meta.tags, createdAt, updatedAt',
+  topics: '&id, title, favorite, createdAt, updatedAt, sessionId',
+};

package/src/database/core/types/db.ts

@@ -0,0 +1,13 @@
+import { z } from 'zod';
+
+export type DBModel<T> = T & {
+  createdAt: number;
+  id: string;
+  updatedAt: number;
+};
+
+export const DBBaseFieldsSchema = z.object({
+  createdAt: z.number(),
+  id: z.string(),
+  updatedAt: z.number(),
+});