@lobehub/chat 0.100.4 → 0.101.0

package/src/store/files/store.ts

@@ -0,0 +1,27 @@
+import { devtools } from 'zustand/middleware';
+import { shallow } from 'zustand/shallow';
+import { createWithEqualityFn } from 'zustand/traditional';
+import { StateCreator } from 'zustand/vanilla';
+
+import { isDev } from '@/utils/env';
+
+import { FilesStoreState, initialState } from './initialState';
+import { FileAction, createFileSlice } from './slices/images';
+
+//  ===============  聚合 createStoreFn ============ //
+
+export type FileStore = FilesStoreState & FileAction;
+
+const createStore: StateCreator<FileStore, [['zustand/devtools', never]]> = (...parameters) => ({
+  ...initialState,
+  ...createFileSlice(...parameters),
+});
+
+//  ===============  实装 useStore ============ //
+
+export const useFileStore = createWithEqualityFn<FileStore>()(
+  devtools(createStore, {
+    name: 'LobeChat_File' + (isDev ? '_DEV' : ''),
+  }),
+  shallow,
+);

package/src/store/global/selectors/settings.test.ts

@@ -75,7 +75,7 @@ describe('settingsSelectors', () => {
       });
     });
   });
-  //
+
   // describe('defaultAgent', () => {
   //   it('should merge DEFAULT_AGENT and s.settings.defaultAgent correctly', () => {
   //     const s: GlobalStore = {
@@ -95,8 +95,6 @@ describe('settingsSelectors', () => {
   //
   //     const result = settingsSelectors.defaultAgent(s);
   //
-  //
-  //
   //     expect(result).toEqual(expected);
   //   });
   // });

package/src/store/session/slices/agentConfig/selectors.ts

@@ -1,6 +1,6 @@
 import { t } from 'i18next';
 
-import { DEFAULT_OPENAI_MODEL_LIST } from '@/const/llm';
+import { DEFAULT_OPENAI_MODEL_LIST, VISION_MODEL_WHITE_LIST } from '@/const/llm';
 import { DEFAULT_AVATAR, DEFAULT_BACKGROUND_COLOR } from '@/const/meta';
 import { SessionStore } from '@/store/session';
 import { LanguageModel } from '@/types/llm';
@@ -49,6 +49,11 @@ const currentAgentModel = (s: SessionStore): LanguageModel | string => {
   return config?.model || LanguageModel.GPT3_5;
 };
 
+const modelHasVisionAbility = (s: SessionStore): boolean => {
+  const model = currentAgentModel(s);
+  return VISION_MODEL_WHITE_LIST.includes(model);
+};
+
 const currentAgentPlugins = (s: SessionStore) => {
   const config = currentAgentConfig(s);
 
@@ -87,5 +92,6 @@ export const agentSelectors = {
   getDescription,
   getTitle,
   hasSystemRole,
+  modelHasVisionAbility,
   showTokenTag,
 };

package/src/store/session/slices/chat/actions/message.ts

@@ -1,10 +1,14 @@
 import { template } from 'lodash-es';
 import { StateCreator } from 'zustand/vanilla';
 
+import { VISION_MODEL_WHITE_LIST } from '@/const/llm';
 import { LOADING_FLAT } from '@/const/message';
+import { VISION_MODEL_DEFAULT_MAX_TOKENS } from '@/const/settings';
 import { fetchChatModel } from '@/services/chatModel';
+import { filesSelectors, useFileStore } from '@/store/files';
 import { SessionStore } from '@/store/session';
 import { ChatMessage } from '@/types/chatMessage';
+import { OpenAIChatMessage, UserMessageContentPart } from '@/types/openai/chat';
 import { fetchSSE } from '@/utils/fetch';
 import { isFunctionMessageAtStart, testFunctionMessageAtEnd } from '@/utils/message';
 import { setNamespace } from '@/utils/storeDebug';
@@ -12,6 +16,7 @@ import { nanoid } from '@/utils/uuid';
 
 import { agentSelectors } from '../../agentConfig/selectors';
 import { sessionSelectors } from '../../session/selectors';
+import { FileDispatch, filesReducer } from '../reducers/files';
 import { MessageDispatch, messagesReducer } from '../reducers/message';
 import { chatSelectors } from '../selectors';
 import { getSlicedMessagesWithConfig } from '../utils';
@@ -37,6 +42,10 @@ export interface ChatMessageAction {
    * @param id - 消息 ID
    */
   deleteMessage: (id: string) => void;
+  /**
+   * agent files dispatch method
+   */
+  dispatchAgentFile: (payload: FileDispatch) => void;
   /**
    * 分发消息
    * @param payload - 消息分发参数
@@ -66,14 +75,15 @@ export interface ChatMessageAction {
    * 发送消息
    * @param text - 消息文本
    */
-  sendMessage: (text: string) => Promise<void>;
+  sendMessage: (text: string, images?: { id: string; url: string }[]) => Promise<void>;
   stopGenerateMessage: () => void;
-
   toggleChatLoading: (
     loading: boolean,
     id?: string,
     action?: string,
   ) => AbortController | undefined;
+
+  updateInputMessage: (message: string) => void;
 }
 
 export const chatMessage: StateCreator<
@@ -94,7 +104,6 @@ export const chatMessage: StateCreator<
     // after remove topic , go back to default topic
     toggleTopic();
   },
-
   coreProcessMessage: async (messages, userMessageId) => {
     const { dispatchMessage, fetchAIChatMessage, triggerFunctionCall, activeTopicId } = get();
 
@@ -163,6 +172,16 @@ export const chatMessage: StateCreator<
   deleteMessage: (id) => {
     get().dispatchMessage({ id, type: 'deleteMessage' });
   },
+
+  dispatchAgentFile: (payload) => {
+    const { activeId } = get();
+    const session = sessionSelectors.currentSession(get());
+    if (!activeId || !session) return;
+
+    const files = filesReducer(session.files || [], payload);
+
+    get().dispatchSession({ files, id: activeId, type: 'updateSessionFiles' });
+  },
   dispatchMessage: (payload) => {
     const { activeId } = get();
     const session = sessionSelectors.currentSession(get());
@@ -186,17 +205,17 @@ export const chatMessage: StateCreator<
 
     const compiler = template(config.inputTemplate, { interpolate: /{{([\S\s]+?)}}/g });
 
-    // ========================== //
-    //   对 messages 做统一预处理    //
-    // ========================== //
+    // ================================== //
+    //   messages uniformly preprocess    //
+    // ================================== //
 
-    // 1. 按参数设定截断长度
-    const slicedMessages = getSlicedMessagesWithConfig(messages, config);
+    // 1. slice messages with config
+    let preprocessMsgs = getSlicedMessagesWithConfig(messages, config);
 
-    // 2. 替换 inputMessage 模板
-    const postMessages = !config.inputTemplate
-      ? slicedMessages
-      : slicedMessages.map((m) => {
+    // 2. replace inputMessage template
+    preprocessMsgs = !config.inputTemplate
+      ? preprocessMsgs
+      : preprocessMsgs.map((m) => {
           if (m.role === 'user') {
             try {
               return { ...m, content: compiler({ text: m.content }) };
@@ -206,12 +225,41 @@ export const chatMessage: StateCreator<
               return m;
             }
           }
+
           return m;
         });
 
-    // 3. 添加 systemRole
+    // 3. add systemRole
     if (config.systemRole) {
-      postMessages.unshift({ content: config.systemRole, role: 'system' } as ChatMessage);
+      preprocessMsgs.unshift({ content: config.systemRole, role: 'system' } as ChatMessage);
+    }
+
+    let postMessages: OpenAIChatMessage[] = preprocessMsgs;
+
+    // 4. handle content type for vision model
+    // for the models with visual ability, add image url to content
+    // refs: https://platform.openai.com/docs/guides/vision/quick-start
+    if (VISION_MODEL_WHITE_LIST.includes(config.model)) {
+      postMessages = preprocessMsgs.map((m) => {
+        if (!m.files) return m;
+
+        const imageList = filesSelectors.getImageUrlOrBase64ByList(m.files)(
+          useFileStore.getState(),
+        );
+
+        if (imageList.length === 0) return m;
+
+        const content: UserMessageContentPart[] = [
+          { text: m.content, type: 'text' },
+          ...imageList.map(
+            (i) => ({ image_url: { detail: 'auto', url: i.url }, type: 'image_url' }) as const,
+          ),
+        ];
+        return { ...m, content };
+      });
+
+      // due to vision model's default max_tokens is very small, we need to set the max_tokens a larger one.
+      if (!config.params.max_tokens) config.params.max_tokens = VISION_MODEL_DEFAULT_MAX_TOKENS;
     }
 
     const fetcher = () =>
@@ -248,7 +296,7 @@ export const chatMessage: StateCreator<
 
     toggleChatLoading(false, undefined, t('generateMessage(end)') as string);
 
-    // also exist message like this: 请稍等，我帮您查询一下。{"function_call": {"name": "plugin-identifier____recommendClothes____standalone", "arguments": "{\n "mood": "",\n "gender": "man"\n}"}}
+    // also exist message like this:  请稍等，我帮您查询一下。{"function_call": {"name": "plugin-identifier____recommendClothes____standalone", "arguments": "{\n "mood": "",\n "gender": "man"\n}"}}
     if (!isFunctionCall) {
       const { content, valid } = testFunctionMessageAtEnd(output);
 
@@ -305,13 +353,27 @@ export const chatMessage: StateCreator<
     await coreProcessMessage(contextMessages, latestMsg.id);
   },
 
-  sendMessage: async (message) => {
-    const { dispatchMessage, coreProcessMessage, activeTopicId } = get();
+  sendMessage: async (message, files) => {
+    const { dispatchMessage, dispatchAgentFile, coreProcessMessage, activeTopicId } = get();
     const session = sessionSelectors.currentSession(get());
     if (!session || !message) return;
 
     const userId = nanoid();
-    dispatchMessage({ id: userId, message, role: 'user', type: 'addMessage' });
+
+    dispatchMessage({
+      id: userId,
+      message: message,
+      role: 'user',
+      type: 'addMessage',
+    });
+
+    // if message has attached with files, then add files to message and the agent
+    if (files && files.length > 0) {
+      const fileIdList = files.map((f) => f.id);
+      dispatchMessage({ id: userId, key: 'files', type: 'updateMessage', value: fileIdList });
+
+      dispatchAgentFile({ files: fileIdList, type: 'addFiles' });
+    }
 
     // if there is activeTopicId，then add topicId to message
     if (activeTopicId) {
@@ -350,4 +412,7 @@ export const chatMessage: StateCreator<
       set({ abortController: undefined, chatLoadingId: undefined }, false, action);
     }
   },
+  updateInputMessage: (message) => {
+    set({ inputMessage: message }, false, t('updateInputMessage'));
+  },
 });

package/src/store/session/slices/chat/initialState.ts

@@ -2,9 +2,12 @@ export interface ChatState {
   abortController?: AbortController;
   activeTopicId?: string;
   chatLoadingId?: string;
+  inputMessage: string;
   renameTopicId?: string;
   shareLoading?: boolean;
   topicLoadingId?: string;
 }
 
-export const initialChatState: ChatState = {};
+export const initialChatState: ChatState = {
+  inputMessage: '',
+};

package/src/store/session/slices/chat/reducers/files.test.ts

@@ -0,0 +1,38 @@
+import { FileDispatch, FilesState, filesReducer } from './files';
+
+describe('filesReducer', () => {
+  it('should add a file to the state', () => {
+    const initialState: FilesState = ['file1', 'file2'];
+    const action: FileDispatch = { type: 'addFile', file: 'file3' };
+    const newState = filesReducer(initialState, action);
+    expect(newState).toEqual(['file1', 'file2', 'file3']);
+  });
+
+  it('should delete a file from the state by ID', () => {
+    const initialState: FilesState = ['file1', 'file2', 'file3'];
+    const action: FileDispatch = { type: 'deleteFile', id: 'file2' };
+    const newState = filesReducer(initialState, action);
+    expect(newState).toEqual(['file1', 'file3']);
+  });
+
+  it('should return the state unchanged if file ID does not exist for deletion', () => {
+    const initialState: FilesState = ['file1', 'file2', 'file3'];
+    const action: FileDispatch = { type: 'deleteFile', id: 'file4' };
+    const newState = filesReducer(initialState, action);
+    expect(newState).toEqual(['file1', 'file2', 'file3']);
+  });
+
+  it('should add multiple files to the state', () => {
+    const initialState: FilesState = ['file1', 'file2'];
+    const action: FileDispatch = { type: 'addFiles', files: ['file3', 'file4'] };
+    const newState = filesReducer(initialState, action);
+    expect(newState).toEqual(['file1', 'file2', 'file3', 'file4']);
+  });
+
+  it('should return the initial state if the action type is unknown', () => {
+    const initialState: FilesState = ['file1', 'file2'];
+    const action = { type: 'unknown', id: 'file1' };
+    const newState = filesReducer(initialState, action as FileDispatch);
+    expect(newState).toEqual(['file1', 'file2']);
+  });
+});

package/src/store/session/slices/chat/reducers/files.ts

@@ -0,0 +1,37 @@
+import { produce } from 'immer';
+
+export type FilesState = string[];
+
+export type FileDispatch =
+  | { file: string; type: 'addFile' }
+  | { id: string; type: 'deleteFile' }
+  | { files: string[]; type: 'addFiles' };
+
+export const filesReducer = (state: FilesState, payload: FileDispatch): FilesState => {
+  switch (payload.type) {
+    case 'addFile': {
+      return produce(state, (draftState) => {
+        draftState.push(payload.file);
+      });
+    }
+
+    case 'deleteFile': {
+      return produce(state, (draftState) => {
+        const index = draftState.indexOf(payload.id);
+        if (index !== -1) {
+          draftState.splice(index, 1);
+        }
+      });
+    }
+
+    case 'addFiles': {
+      return produce(state, (draftState) => {
+        draftState.push(...payload.files);
+      });
+    }
+
+    default: {
+      return state;
+    }
+  }
+};

package/src/store/session/slices/chat/selectors/chat.ts

@@ -41,6 +41,7 @@ export const currentChats = (s: SessionStore): ChatMessage[] => {
   return getChatsById(s.activeId)(s);
 };
 
+const initTime = Date.now();
 // 针对新助手添加初始化时的自定义消息
 export const currentChatsWithGuideMessage = (s: SessionStore): ChatMessage[] => {
   const data = currentChats(s);
@@ -67,14 +68,14 @@ export const currentChatsWithGuideMessage = (s: SessionStore): ChatMessage[] =>
 
   const emptyInboxGuideMessage = {
     content: isInbox ? inboxMsg : !!meta.description ? agentSystemRoleMsg : agentMsg,
-    createAt: Date.now(),
+    createAt: initTime,
     extra: {},
     id: 'default',
     meta: meta || {
       avatar: DEFAULT_INBOX_AVATAR,
     },
     role: 'assistant',
-    updateAt: Date.now(),
+    updateAt: initTime,
   } as ChatMessage;
 
   return [emptyInboxGuideMessage];

package/src/store/session/slices/session/initialState.ts

@@ -25,6 +25,7 @@ export const initLobeSession: LobeAgentSession = {
   chats: {},
   config: initialLobeAgentConfig,
   createAt: Date.now(),
+  files: [],
   id: '',
   meta: DEFAULT_AGENT_META,
   type: LobeSessionType.Agent,

package/src/store/session/slices/session/reducers/session.test.ts

@@ -206,6 +206,103 @@ describe('sessionsReducer', () => {
     });
   });
 
+  describe('updateSessionFiles', () => {
+    it('should update session files when valid id, key, and value are provided', () => {
+      const state: LobeSessions = {
+        'session-id': {
+          id: 'session-id',
+          config: {
+            model: 'gpt-3.5-turbo',
+            params: {},
+            systemRole: 'system-role',
+          },
+          type: 'agent',
+          meta: {
+            avatar: 'avatar-url',
+            backgroundColor: 'background-color',
+            description: 'description',
+            tags: ['tag1', 'tag2'],
+            title: 'title',
+          },
+        } as LobeAgentSession,
+      };
+      const id = 'session-id';
+      const files = ['new-file'];
+      const payload: SessionDispatch = { type: 'updateSessionFiles', id, files };
+
+      const newState = sessionsReducer(state, payload);
+
+      expect(newState).toEqual({
+        'session-id': {
+          ...state['session-id'],
+          files: ['new-file'],
+        },
+      });
+    });
+
+    it('should not change state when invalid id, key, and value are provided', () => {
+      const state: LobeSessions = {
+        'session-id': {
+          id: 'session-id',
+          config: {
+            model: 'gpt-3.5-turbo',
+            params: {},
+            systemRole: 'system-role',
+          },
+          type: 'agent',
+          meta: {
+            avatar: 'avatar-url',
+            backgroundColor: 'background-color',
+            description: 'description',
+            tags: ['tag1', 'tag2'],
+            title: 'title',
+          },
+        } as LobeAgentSession,
+      };
+      const id = 'non-existent-id';
+      const files = ['new-file'];
+      const payload: SessionDispatch = { type: 'updateSessionFiles', id, files };
+
+      const newState = sessionsReducer(state, payload);
+
+      expect(newState).toEqual(state);
+    });
+
+    it('should not change state when valid id, invalid files  are provided', () => {
+      const state: LobeSessions = {
+        'session-id': {
+          id: 'session-id',
+          config: {
+            model: 'gpt-3.5-turbo',
+            params: {},
+            systemRole: 'system-role',
+          },
+          type: 'agent',
+          meta: {
+            avatar: 'avatar-url',
+            backgroundColor: 'background-color',
+            description: 'description',
+            tags: ['tag1', 'tag2'],
+            title: 'title',
+          },
+        } as LobeAgentSession,
+      };
+      const id = 'session-id';
+
+      const errFile1 = 123213412 as any;
+      const res1 = sessionsReducer(state, { type: 'updateSessionFiles', id, files: errFile1 });
+      expect(res1).toEqual(state);
+
+      const errFile2 = undefined as any;
+      const res2 = sessionsReducer(state, { type: 'updateSessionFiles', id, files: errFile2 });
+      expect(res2).toEqual(state);
+
+      const errFile3 = ['state', 123] as any;
+      const res3 = sessionsReducer(state, { type: 'updateSessionFiles', id, files: errFile3 });
+      expect(res3).toEqual(state);
+    });
+  });
+
   describe('updateSessionConfig', () => {
     it('should update session config when valid id and partial config are provided', () => {
       const state: LobeSessions = {
@@ -345,7 +442,7 @@ describe('sessionsReducer', () => {
     });
   });
 
-  test('should not change state when invalid operation type is provided', () => {
+  it('should not change state when invalid operation type is provided', () => {
     const state: LobeSessions = {
       session1: {
         id: 'session1',

package/src/store/session/slices/session/reducers/session.ts

@@ -1,4 +1,5 @@
 import { produce } from 'immer';
+import { z } from 'zod';
 
 import { ChatMessageMap } from '@/types/chatMessage';
 import { MetaData } from '@/types/meta';
@@ -39,7 +40,7 @@ interface UpdateSessionChat {
 }
 
 /**
- * @title 更新会话聊天上下文
+ * 更新会话聊天上下文
  */
 interface UpdateSessionTopic {
   /**
@@ -51,6 +52,15 @@ interface UpdateSessionTopic {
   type: 'updateSessionTopic';
 }
 
+/**
+ * 更新会话聊天上下文
+ */
+interface UpdateSessionFiles {
+  files: string[];
+  id: string;
+  type: 'updateSessionFiles';
+}
+
 interface UpdateSessionMeta {
   id: string;
   key: keyof MetaData;
@@ -76,6 +86,7 @@ export type SessionDispatch =
   | UpdateSessionMeta
   | UpdateSessionAgentConfig
   | UpdateSessionTopic
+  | UpdateSessionFiles
   | ToggleSessionPinned;
 
 export const sessionsReducer = (state: LobeSessions, payload: SessionDispatch): LobeSessions => {
@@ -136,6 +147,20 @@ export const sessionsReducer = (state: LobeSessions, payload: SessionDispatch):
       });
     }
 
+    case 'updateSessionFiles': {
+      return produce(state, (draft) => {
+        const session = draft[payload.id];
+        if (!session || !payload.files) return;
+
+        const schema = z.array(z.string());
+
+        const { success } = schema.safeParse(payload.files);
+        if (!success) return;
+
+        session.files = payload.files;
+      });
+    }
+
     case 'updateSessionConfig': {
       return produce(state, (draft) => {
         const { id, config } = payload;

package/src/types/chatMessage.ts

@@ -36,18 +36,19 @@ export interface ChatMessage extends BaseDataModel {
     translate?: ChatTranslate;
   } & Record<string, any>;
 
+  files?: string[];
   /**
    * replace with plugin
    * @deprecated
    */
   function_call?: OpenAIFunctionCall;
+
   name?: string;
 
   parentId?: string;
-
   plugin?: PluginRequestPayload;
-  pluginState?: any;
 
+  pluginState?: any;
   // 引用
   quotaId?: string;
   /**

package/src/types/database/db.ts

@@ -0,0 +1,4 @@
+export type DBModel<T> = T & {
+  createdAt: number;
+  id: string;
+};

package/src/types/database/files.ts

@@ -0,0 +1,38 @@
+import { z } from 'zod';
+
+export const LocalFileSchema = z.object({
+  /**
+   * create Time
+   */
+  createdAt: z.number().optional(),
+  /**
+   * file data array buffer
+   */
+  data: z.instanceof(ArrayBuffer),
+  /**
+   * file type
+   * @example 'image/png'
+   */
+  fileType: z.string(),
+  /**
+   * file name
+   * @example 'test.png'
+   */
+  name: z.string(),
+  /**
+   * the mode database save the file
+   * local mean save the raw file into data
+   * url mean upload the file to a cdn and then save the url
+   */
+  saveMode: z.enum(['local', 'url']),
+  /**
+   * file size
+   */
+  size: z.number(),
+  /**
+   * file url if saveMode is url
+   */
+  url: z.string().url().optional(),
+});
+
+export type LocalFile = z.infer<typeof LocalFileSchema>;

package/src/types/files.ts

@@ -0,0 +1,8 @@
+import { LocalFile } from './database/files';
+
+export interface FilePreview extends Pick<LocalFile, 'saveMode' | 'fileType'> {
+  base64Url?: string;
+  data?: ArrayBuffer;
+  name: string;
+  url: string;
+}

package/src/types/openai/chat.ts

@@ -1,12 +1,26 @@
 import { OpenAIFunctionCall } from '@/types/chatMessage';
 import { LLMRoleType } from '@/types/llm';
 
+interface UserMessageContentPartText {
+  text: string;
+  type: 'text';
+}
+interface UserMessageContentPartImage {
+  image_url: {
+    detail?: 'auto' | 'low' | 'high';
+    url: string;
+  };
+  type: 'image_url';
+}
+
+export type UserMessageContentPart = UserMessageContentPartText | UserMessageContentPartImage;
+
 export interface OpenAIChatMessage {
   /**
    * @title 内容
    * @description 消息内容
    */
-  content: string;
+  content: string | UserMessageContentPart[];
 
   function_call?: OpenAIFunctionCall;
   name?: string;

package/src/types/session.ts

@@ -19,6 +19,7 @@ interface LobeSessionBase extends BaseDataModel {
    * 聊天记录
    */
   chats: ChatMessageMap;
+  files?: string[];
   /**
    * 置顶
    */