@lobehub/chat 0.105.2 → 0.107.0

package/src/migrations/index.ts

@@ -1,11 +1,25 @@
+import { VersionController } from '@/migrations/VersionController';
 import { ConfigStateAll } from '@/types/exportConfig';
-import { VersionController } from '@/utils/VersionController';
+
+import { MigrationV0ToV1 } from './FromV0ToV1';
+import { MigrationV1ToV2 } from './FromV1ToV2';
 
 // 当前最新的版本号
-export const CURRENT_CONFIG_VERSION = 1;
+export const CURRENT_CONFIG_VERSION = 2;
 
 // 历史记录版本升级模块
-export const ConfigMigrations = [];
+const ConfigMigrations = [
+  /**
+   * 2023.11.27
+   * 从单 key 数据库转换为基于 dexie 的关系型结构
+   */
+  MigrationV1ToV2,
+  /**
+   * 2023.07.11
+   * just the first version, Nothing to do
+   */
+  MigrationV0ToV1,
+];
 
 export const Migration = new VersionController<ConfigStateAll>(
   ConfigMigrations,

package/src/services/__tests__/file.test.ts

@@ -1,7 +1,7 @@
 import { Mock, beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { FileModel } from '@/database/models/file';
-import { LocalFile } from '@/types/database/files';
+import { DB_File } from '@/database/schemas/files';
 
 import { fileService } from '../file';
 
@@ -25,7 +25,7 @@ describe('FileService', () => {
   });
 
   it('uploadFile should save the file to the database', async () => {
-    const localFile: LocalFile = {
+    const localFile: DB_File = {
       name: 'test',
       data: new ArrayBuffer(1),
       fileType: 'image/png',
@@ -53,7 +53,7 @@ describe('FileService', () => {
 
   it('getFile should retrieve and convert file info to FilePreview', async () => {
     const fileId = '1';
-    const fileData: LocalFile = {
+    const fileData: DB_File = {
       name: 'test',
       data: new ArrayBuffer(1),
       fileType: 'image/png',

package/src/services/chat.ts

@@ -3,9 +3,12 @@ import { merge } from 'lodash-es';
 
 import { VISION_MODEL_WHITE_LIST } from '@/const/llm';
 import { DEFAULT_AGENT_CONFIG } from '@/const/settings';
+import { filesSelectors, useFileStore } from '@/store/files';
 import { usePluginStore } from '@/store/plugin';
 import { pluginSelectors } from '@/store/plugin/selectors';
-import type { OpenAIChatStreamPayload } from '@/types/openai/chat';
+import { ChatMessage } from '@/types/chatMessage';
+import type { OpenAIChatMessage, OpenAIChatStreamPayload } from '@/types/openai/chat';
+import { UserMessageContentPart } from '@/types/openai/chat';
 import { fetchAIFactory, getMessageError } from '@/utils/fetch';
 
 import { createHeaderWithOpenAI } from './_header';
@@ -15,9 +18,13 @@ interface FetchOptions {
   signal?: AbortSignal | undefined;
 }
 
+interface GetChatCompletionPayload extends Partial<Omit<OpenAIChatStreamPayload, 'messages'>> {
+  messages: ChatMessage[];
+}
+
 class ChatService {
-  getChatCompletion = (
-    { plugins: enabledPlugins, ...params }: Partial<OpenAIChatStreamPayload>,
+  createAssistantMessage = async (
+    { plugins: enabledPlugins, messages, ...params }: GetChatCompletionPayload,
     options?: FetchOptions,
   ) => {
     const payload = merge(
@@ -28,7 +35,11 @@ class ChatService {
       },
       params,
     );
-    // ============   preprocess tools   ============ //
+    // ============  1. preprocess messages   ============ //
+
+    const oaiMessages = this.processMessages(messages);
+
+    // ============  2. preprocess tools   ============ //
 
     const filterTools = pluginSelectors.enabledSchema(enabledPlugins)(usePluginStore.getState());
 
@@ -41,8 +52,21 @@ class ChatService {
 
     const functions = shouldUseTools ? filterTools : undefined;
 
+    return this.getChatCompletion({ ...params, functions, messages: oaiMessages }, options);
+  };
+
+  getChatCompletion = (params: Partial<OpenAIChatStreamPayload>, options?: FetchOptions) => {
+    const payload = merge(
+      {
+        model: DEFAULT_AGENT_CONFIG.model,
+        stream: true,
+        ...DEFAULT_AGENT_CONFIG.params,
+      },
+      params,
+    );
+
     return fetch(OPENAI_URLS.chat, {
-      body: JSON.stringify({ ...payload, functions }),
+      body: JSON.stringify(payload),
       headers: createHeaderWithOpenAI({ 'Content-Type': 'application/json' }),
       method: 'POST',
       signal: options?.signal,
@@ -79,6 +103,43 @@ class ChatService {
   };
 
   fetchPresetTaskResult = fetchAIFactory(this.getChatCompletion);
+
+  private processMessages = (messages: ChatMessage[]): OpenAIChatMessage[] => {
+    // handle content type for vision model
+    // for the models with visual ability, add image url to content
+    // refs: https://platform.openai.com/docs/guides/vision/quick-start
+    const getContent = (m: ChatMessage) => {
+      if (!m.files) return m.content;
+
+      const imageList = filesSelectors.getImageUrlOrBase64ByList(m.files)(useFileStore.getState());
+
+      if (imageList.length === 0) return m.content;
+
+      return [
+        { text: m.content, type: 'text' },
+        ...imageList.map(
+          (i) => ({ image_url: { detail: 'auto', url: i.url }, type: 'image_url' }) as const,
+        ),
+      ] as UserMessageContentPart[];
+    };
+
+    return messages.map((m): OpenAIChatMessage => {
+      switch (m.role) {
+        case 'user': {
+          return { content: getContent(m), role: m.role };
+        }
+
+        case 'function': {
+          const name = m.plugin?.identifier as string;
+          return { content: m.content, name, role: m.role };
+        }
+
+        default: {
+          return { content: m.content, role: m.role };
+        }
+      }
+    });
+  };
 }
 
 export const chatService = new ChatService();

package/src/services/config.ts

@@ -0,0 +1,184 @@
+import { messageService } from '@/services/message';
+import { sessionService } from '@/services/session';
+import { topicService } from '@/services/topic';
+import { settingsSelectors, useGlobalStore } from '@/store/global';
+import { useSessionStore } from '@/store/session';
+import { sessionExportSelectors, sessionSelectors } from '@/store/session/selectors';
+import { ChatMessage } from '@/types/chatMessage';
+import { ConfigFile } from '@/types/exportConfig';
+import { LobeSessions } from '@/types/session';
+import { GlobalSettings } from '@/types/settings';
+import { ChatTopic } from '@/types/topic';
+import { createConfigFile, exportConfigFile } from '@/utils/config';
+
+export interface ImportResult {
+  added: number;
+  errors: number;
+  skips: number;
+}
+export interface ImportResults {
+  messages?: ImportResult;
+  sessions: ImportResult;
+  topics?: ImportResult;
+}
+
+class ConfigService {
+  /**
+   * import sessions from files
+   * @param sessions
+   */
+  importSessions = async (sessions: LobeSessions) => {
+    return await sessionService.batchCreateSessions(sessions);
+  };
+  importMessages = async (messages: ChatMessage[]) => {
+    return messageService.batchCreate(messages);
+  };
+  importSettings = async (settings: GlobalSettings) => {
+    useGlobalStore.getState().importAppSettings(settings);
+  };
+  importTopics = async (topics: ChatTopic[]) => {
+    return topicService.batchCreateTopics(topics);
+  };
+
+  importConfigState = async (config: ConfigFile): Promise<ImportResults | undefined> => {
+    switch (config.exportType) {
+      case 'settings': {
+        await this.importSettings(config.state.settings);
+
+        break;
+      }
+
+      case 'agents': {
+        const data = await this.importSessions(config.state.sessions);
+        return {
+          sessions: this.mapImportResult(data),
+        };
+      }
+
+      case 'all': {
+        await this.importSettings(config.state.settings);
+
+        const [sessions, messages, topics] = await Promise.all([
+          this.importSessions(config.state.sessions),
+          this.importMessages(config.state.messages),
+          this.importTopics(config.state.topics),
+        ]);
+
+        return {
+          messages: this.mapImportResult(messages),
+          sessions: this.mapImportResult(sessions),
+          topics: this.mapImportResult(topics),
+        };
+      }
+
+      case 'sessions': {
+        const [sessions, messages, topics] = await Promise.all([
+          this.importSessions(config.state.sessions),
+          this.importMessages(config.state.messages),
+          this.importTopics(config.state.topics),
+        ]);
+
+        return {
+          messages: this.mapImportResult(messages),
+          sessions: this.mapImportResult(sessions),
+          topics: this.mapImportResult(topics),
+        };
+      }
+    }
+  };
+
+  /**
+   * export all agents
+   */
+  exportAgents = async () => {
+    const agents = await sessionService.getAllAgents();
+
+    const config = createConfigFile('agents', { sessions: agents });
+
+    exportConfigFile(config, 'agents');
+  };
+
+  /**
+   * export all sessions
+   */
+  exportSessions = async () => {
+    const sessions = await sessionService.getSessions();
+    const messages = await messageService.getAllMessages();
+    const topics = await topicService.getAllTopics();
+
+    const config = createConfigFile('sessions', { messages, sessions, topics });
+
+    exportConfigFile(config, 'sessions');
+  };
+
+  /**
+   * export a session
+   */
+  exportSingleSession = async (id: string) => {
+    const session = this.getSession(id);
+    if (!session) return;
+
+    const messages = await messageService.getMessages(id);
+    const topics = await topicService.getTopics({ sessionId: id });
+
+    const config = createConfigFile('singleSession', { messages, sessions: [session], topics });
+
+    exportConfigFile(config, `${session.meta?.title}-session`);
+  };
+
+  exportSingleAgent = async (id: string) => {
+    const agent = this.getAgent(id);
+    if (!agent) return;
+
+    const config = createConfigFile('agents', { sessions: [agent] });
+
+    exportConfigFile(config, agent.meta?.title || 'agent');
+  };
+
+  /**
+   * export settings
+   */
+  exportSettings = async () => {
+    const settings = this.getSettings();
+
+    const config = createConfigFile('settings', { settings });
+
+    exportConfigFile(config, 'settings');
+  };
+
+  /**
+   * export all data
+   */
+  exportAll = async () => {
+    const sessions = await sessionService.getSessions();
+    const messages = await messageService.getAllMessages();
+    const topics = await topicService.getAllTopics();
+    const settings = this.getSettings();
+
+    const config = createConfigFile('all', { messages, sessions, settings, topics });
+
+    exportConfigFile(config, 'config');
+  };
+
+  private getSettings = () => settingsSelectors.exportSettings(useGlobalStore.getState());
+
+  private getSession = (id: string) =>
+    sessionSelectors.getSessionById(id)(useSessionStore.getState());
+
+  private getAgent = (id: string) =>
+    sessionExportSelectors.getExportAgent(id)(useSessionStore.getState());
+
+  private mapImportResult = (input: {
+    added: number;
+    errors?: Error[];
+    skips: string[];
+  }): ImportResult => {
+    return {
+      added: input.added,
+      errors: input.errors?.length || 0,
+      skips: input.skips.length,
+    };
+  };
+}
+
+export const configService = new ConfigService();

package/src/services/debug.ts

@@ -0,0 +1,13 @@
+import { DEBUG_MODEL } from '@/database/models/__DEBUG';
+
+class DebugService {
+  async insertLargeDataToDB() {
+    return DEBUG_MODEL.createRandomData({
+      messageCount: 400_000,
+      sessionCount: 40,
+      topicCount: 200,
+    });
+  }
+}
+
+export const debugService = new DebugService();

package/src/services/file.ts

@@ -1,9 +1,9 @@
 import { FileModel } from '@/database/models/file';
-import { LocalFile } from '@/types/database/files';
+import { DB_File } from '@/database/schemas/files';
 import { FilePreview } from '@/types/files';
 
 class FileService {
-  async uploadFile(file: LocalFile) {
+  async uploadFile(file: DB_File) {
     // save to local storage
     // we may want to save to a remote server later
     return FileModel.create(file);
@@ -13,6 +13,10 @@ class FileService {
     return FileModel.delete(id);
   }
 
+  async removeAllFiles() {
+    return FileModel.clear();
+  }
+
   async getFile(id: string): Promise<FilePreview> {
     const item = await FileModel.findById(id);
     if (!item) {

package/src/services/github.ts

@@ -0,0 +1,25 @@
+import qs from 'query-string';
+import urlJoin from 'url-join';
+
+import { GITHUB } from '@/const/url';
+
+class GitHubService {
+  submitDBV1UpgradeError = (version: number, error?: { message: string }) => {
+    const body = ['```json', JSON.stringify(error, null, 2), '```'].join('\n');
+
+    const message = error?.message || '';
+
+    const url = qs.stringifyUrl({
+      query: {
+        body,
+        labels: '❌ Database Migration Error',
+        title: `[Migration Error V${version}] ${message}`,
+      },
+      url: urlJoin(GITHUB, '/issues/new'),
+    });
+
+    window.open(url, '_blank');
+  };
+}
+
+export const githubService = new GitHubService();

package/src/services/message.ts

@@ -0,0 +1,79 @@
+import { CreateMessageParams, MessageModel } from '@/database/models/message';
+import {
+  ChatMessage,
+  ChatMessageError,
+  ChatPluginPayload,
+  ChatTTS,
+  ChatTranslate,
+} from '@/types/chatMessage';
+import { LLMRoleType } from '@/types/llm';
+
+export class MessageService {
+  async create(data: CreateMessageParams) {
+    const { id } = await MessageModel.create(data);
+
+    return id;
+  }
+
+  async batchCreate(messages: ChatMessage[]) {
+    return MessageModel.batchCreate(messages);
+  }
+
+  async hasMessages() {
+    const isEmpty = await MessageModel.isEmpty();
+    return !isEmpty;
+  }
+
+  async getMessages(sessionId: string, topicId?: string): Promise<ChatMessage[]> {
+    return MessageModel.query({ sessionId, topicId });
+  }
+
+  async removeMessage(id: string) {
+    return MessageModel.delete(id);
+  }
+
+  async updateMessageContent(id: string, content: string) {
+    return MessageModel.update(id, { content });
+  }
+
+  async updateMessageError(id: string, error: ChatMessageError) {
+    return MessageModel.update(id, { error });
+  }
+
+  async updateMessageTranslate(id: string, data: Partial<ChatTranslate> | null) {
+    return MessageModel.update(id, { translate: data as ChatTranslate });
+  }
+
+  async updateMessageTTS(id: string, data: Partial<ChatTTS> | null) {
+    return MessageModel.update(id, { tts: data as ChatTTS });
+  }
+
+  async removeMessages(assistantId: string, topicId?: string) {
+    return MessageModel.batchDelete(assistantId, topicId);
+  }
+
+  async clearAllMessage() {
+    return MessageModel.clearTable();
+  }
+
+  async bindMessagesToTopic(topicId: string, messageIds: string[]) {
+    return MessageModel.batchUpdate(messageIds, { topicId });
+  }
+
+  async updateMessageRole(id: string, role: LLMRoleType) {
+    return MessageModel.update(id, { role });
+  }
+
+  async updateMessagePlugin(id: string, plugin: ChatPluginPayload) {
+    return MessageModel.update(id, { plugin });
+  }
+  async updateMessagePluginState(id: string, key: string, value: any) {
+    return MessageModel.update(id, { pluginState: { [key]: value } });
+  }
+
+  async getAllMessages() {
+    return MessageModel.queryAll();
+  }
+}
+
+export const messageService = new MessageService();

package/src/services/session.ts

@@ -0,0 +1,63 @@
+import { DeepPartial } from 'utility-types';
+
+import { SessionModel } from '@/database/models/session';
+import { LobeAgentConfig } from '@/types/agent';
+import { MetaData } from '@/types/meta';
+import { LobeAgentSession, LobeSessionType, LobeSessions, SessionGroupKey } from '@/types/session';
+
+class SessionService {
+  async createNewSession(
+    type: LobeSessionType,
+    defaultValue: Partial<LobeAgentSession>,
+  ): Promise<string> {
+    const item = await SessionModel.create(type, defaultValue);
+    if (!item) {
+      throw new Error('session create Error');
+    }
+    return item.id;
+  }
+
+  async batchCreateSessions(importSessions: LobeSessions) {
+    return SessionModel.batchCreate(importSessions);
+  }
+
+  async getSessions(): Promise<LobeSessions> {
+    return SessionModel.query();
+  }
+
+  async getAllAgents(): Promise<LobeSessions> {
+    // TODO: add a filter to get only agents
+    return await SessionModel.query();
+  }
+
+  async removeSession(id: string) {
+    return SessionModel.delete(id);
+  }
+
+  async removeAllSessions() {
+    return SessionModel.clearTable();
+  }
+
+  async updateSessionGroup(id: string, group: SessionGroupKey) {
+    return SessionModel.update(id, { group });
+  }
+
+  async updateSessionMeta(activeId: string, meta: Partial<MetaData>) {
+    return SessionModel.update(activeId, { meta });
+  }
+
+  async updateSessionConfig(activeId: string, config: DeepPartial<LobeAgentConfig>) {
+    return SessionModel.updateConfig(activeId, config);
+  }
+
+  async hasSessions() {
+    const isEmpty = await SessionModel.isEmpty();
+    return !isEmpty;
+  }
+
+  async searchSessions(keyword: string) {
+    return SessionModel.queryByKeyword(keyword);
+  }
+}
+
+export const sessionService = new SessionService();

package/src/services/topic.ts

@@ -0,0 +1,56 @@
+import { CreateTopicParams, QueryTopicParams, TopicModel } from '@/database/models/topic';
+import { ChatTopic } from '@/types/topic';
+
+class TopicService {
+  async createTopic(params: CreateTopicParams): Promise<string> {
+    const item = await TopicModel.create(params);
+
+    if (!item) {
+      throw new Error('topic create Error');
+    }
+
+    return item.id;
+  }
+
+  async getTopics(params: QueryTopicParams): Promise<ChatTopic[]> {
+    return TopicModel.query(params);
+  }
+
+  async removeTopic(id: string) {
+    return TopicModel.delete(id);
+  }
+
+  async removeTopics(sessionId: string) {
+    return TopicModel.batchDeleteBySessionId(sessionId);
+  }
+
+  async batchRemoveTopics(topics: string[]) {
+    return TopicModel.batchDelete(topics);
+  }
+
+  async removeAllTopic() {
+    return TopicModel.clearTable();
+  }
+
+  async updateFavorite(id: string, newState?: boolean) {
+    return TopicModel.toggleFavorite(id, newState);
+  }
+
+  async batchCreateTopics(importTopics: ChatTopic[]) {
+    return TopicModel.batchCreate(importTopics as any);
+  }
+
+  updateTitle(topicId: string, text: string) {
+    return TopicModel.update(topicId, { title: text });
+  }
+
+  async getAllTopics() {
+    return TopicModel.queryAll();
+  }
+
+  async searchTopics(keyword: string) {
+    return TopicModel.queryByKeyword(keyword);
+  }
+}
+
+export const topicService = new TopicService();