@lobehub/chat 0.122.9 → 0.123.1

package/src/services/chat.ts

@@ -2,24 +2,27 @@ import { PluginRequestPayload, createHeadersWithPluginSettings } from '@lobehub/
 import { produce } from 'immer';
 import { merge } from 'lodash-es';
 
-import { isVisionModel } from '@/const/llm';
 import { DEFAULT_AGENT_CONFIG } from '@/const/settings';
+import { ModelProvider } from '@/libs/agent-runtime';
 import { filesSelectors, useFileStore } from '@/store/file';
+import { useGlobalStore } from '@/store/global';
+import { modelProviderSelectors } from '@/store/global/selectors';
 import { useToolStore } from '@/store/tool';
 import { pluginSelectors, toolSelectors } from '@/store/tool/selectors';
 import { ChatMessage } from '@/types/message';
-import type { OpenAIChatMessage, OpenAIChatStreamPayload } from '@/types/openai/chat';
+import type { ChatStreamPayload, OpenAIChatMessage } from '@/types/openai/chat';
 import { UserMessageContentPart } from '@/types/openai/chat';
 import { fetchAIFactory, getMessageError } from '@/utils/fetch';
 
+import { createHeaderWithAuth } from './_auth';
 import { createHeaderWithOpenAI } from './_header';
-import { OPENAI_URLS, PLUGINS_URLS } from './_url';
+import { PLUGINS_URLS } from './_url';
 
 interface FetchOptions {
   signal?: AbortSignal | undefined;
 }
 
-interface GetChatCompletionPayload extends Partial<Omit<OpenAIChatStreamPayload, 'messages'>> {
+interface GetChatCompletionPayload extends Partial<Omit<ChatStreamPayload, 'messages'>> {
   messages: ChatMessage[];
 }
 
@@ -48,29 +51,39 @@ class ChatService {
 
     const filterTools = toolSelectors.enabledSchema(enabledPlugins)(useToolStore.getState());
 
+    // check this model can use function call
+    const canUseFC = modelProviderSelectors.modelEnabledFunctionCall(payload.model)(
+      useGlobalStore.getState(),
+    );
     // the rule that model can use tools:
     // 1. tools is not empty
-    // 2. model is not in vision white list, because vision model can't use tools
-    // TODO: we need to find some method to let vision model use tools
-    const shouldUseTools = filterTools.length > 0 && !isVisionModel(payload.model);
+    // 2. model can use function call
+    const shouldUseTools = filterTools.length > 0 && canUseFC;
+
     const tools = shouldUseTools ? filterTools : undefined;
 
     return this.getChatCompletion({ ...params, messages: oaiMessages, tools }, options);
   };
 
-  getChatCompletion = (params: Partial<OpenAIChatStreamPayload>, options?: FetchOptions) => {
+  getChatCompletion = async (params: Partial<ChatStreamPayload>, options?: FetchOptions) => {
+    const { provider = ModelProvider.OpenAI, ...res } = params;
     const payload = merge(
       {
         model: DEFAULT_AGENT_CONFIG.model,
         stream: true,
         ...DEFAULT_AGENT_CONFIG.params,
       },
-      params,
+      res,
     );
 
-    return fetch(OPENAI_URLS.chat, {
+    const headers = await createHeaderWithAuth({
+      headers: { 'Content-Type': 'application/json' },
+      provider,
+    });
+
+    return fetch(`/api/chat/${provider}`, {
       body: JSON.stringify(payload),
-      headers: createHeaderWithOpenAI({ 'Content-Type': 'application/json' }),
+      headers,
       method: 'POST',
       signal: options?.signal,
     });
@@ -112,7 +125,7 @@ class ChatService {
     model,
   }: {
     messages: ChatMessage[];
-    model?: string;
+    model: string;
     tools?: string[];
   }): OpenAIChatMessage[] => {
     // handle content type for vision model
@@ -125,7 +138,11 @@ class ChatService {
 
       if (imageList.length === 0) return m.content;
 
-      if (!isVisionModel(model)) {
+      const canUploadFile = modelProviderSelectors.modelEnabledUpload(model)(
+        useGlobalStore.getState(),
+      );
+
+      if (!canUploadFile) {
         return m.content;
       }
 
@@ -156,11 +173,14 @@ class ChatService {
 
     return produce(postMessages, (draft) => {
       if (!tools || tools.length === 0) return;
+      const hasFC = modelProviderSelectors.modelEnabledFunctionCall(model)(
+        useGlobalStore.getState(),
+      );
+      if (!hasFC) return;
 
       const systemMessage = draft.find((i) => i.role === 'system');
 
       const toolsSystemRoles = toolSelectors.enabledSystemRoles(tools)(useToolStore.getState());
-
       if (!toolsSystemRoles) return;
 
       if (systemMessage) {

package/src/store/chat/slices/message/action.ts

@@ -5,7 +5,6 @@ import { template } from 'lodash-es';
 import useSWR, { SWRResponse, mutate } from 'swr';
 import { StateCreator } from 'zustand/vanilla';
 
-import { GPT4_VISION_MODEL_DEFAULT_MAX_TOKENS } from '@/const/llm';
 import { LOADING_FLAT, isFunctionMessageAtStart, testFunctionMessageAtEnd } from '@/const/message';
 import { CreateMessageParams } from '@/database/models/message';
 import { chatService } from '@/services/chat';
@@ -255,13 +254,14 @@ export const chatMessage: StateCreator<
   coreProcessMessage: async (messages, userMessageId) => {
     const { fetchAIChatMessage, triggerFunctionCall, refreshMessages, activeTopicId } = get();
 
-    const { model } = getAgentConfig();
+    const { model, provider } = getAgentConfig();
 
     // 1. Add an empty message to place the AI response
     const assistantMessage: CreateMessageParams = {
       role: 'assistant',
       content: LOADING_FLAT,
       fromModel: model,
+      fromProvider: provider,
 
       parentId: userMessageId,
       sessionId: get().activeId,
@@ -288,13 +288,14 @@ export const chatMessage: StateCreator<
         const functionMessage: CreateMessageParams = {
           role: 'function',
           content: functionCallContent,
-          extra: {
-            fromModel: model,
-          },
+          fromModel: model,
+          fromProvider: provider,
+
           parentId: userMessageId,
           sessionId: get().activeId,
           topicId: activeTopicId,
         };
+
         functionId = await messageService.create(functionMessage);
       }
 
@@ -368,7 +369,8 @@ export const chatMessage: StateCreator<
     if (config.model === 'gpt-4-vision-preview') {
       /* eslint-disable unicorn/no-lonely-if */
       if (!config.params.max_tokens)
-        config.params.max_tokens = GPT4_VISION_MODEL_DEFAULT_MAX_TOKENS;
+        // refs: https://github.com/lobehub/lobe-chat/issues/837
+        config.params.max_tokens = 2048;
     }
 
     const fetcher = () =>
@@ -376,6 +378,7 @@ export const chatMessage: StateCreator<
         {
           messages: preprocessMsgs,
           model: config.model,
+          provider: config.provider,
           ...config.params,
           plugins: config.plugins,
         },

package/src/store/global/slices/common/action.ts

@@ -1,5 +1,6 @@
 import { gt } from 'semver';
 import useSWR, { SWRResponse, mutate } from 'swr';
+import { DeepPartial } from 'utility-types';
 import type { StateCreator } from 'zustand/vanilla';
 
 import { INBOX_SESSION_ID } from '@/const/session';
@@ -8,7 +9,7 @@ import { CURRENT_VERSION } from '@/const/version';
 import { globalService } from '@/services/global';
 import { UserConfig, userService } from '@/services/user';
 import type { GlobalStore } from '@/store/global';
-import type { GlobalServerConfig } from '@/types/settings';
+import type { GlobalServerConfig, GlobalSettings } from '@/types/settings';
 import { merge } from '@/utils/merge';
 import { setNamespace } from '@/utils/storeDebug';
 
@@ -63,7 +64,12 @@ export const createCommonSlice: StateCreator<
     useSWR<GlobalServerConfig>('fetchGlobalConfig', globalService.getGlobalConfig, {
       onSuccess: (data) => {
         if (data) {
-          const defaultSettings = merge(get().defaultSettings, { defaultAgent: data.defaultAgent });
+          const serverSettings: DeepPartial<GlobalSettings> = {
+            defaultAgent: data.defaultAgent,
+            languageModel: data.languageModel,
+          };
+
+          const defaultSettings = merge(get().defaultSettings, serverSettings);
           set({ defaultSettings, serverConfig: data }, false, n('initGlobalConfig'));
         }
       },

package/src/store/global/slices/settings/action.test.ts

@@ -67,14 +67,14 @@ describe('SettingsAction', () => {
     });
   });
 
-  describe('setOpenAIConfig', () => {
+  describe('setModelProviderConfig', () => {
     it('should set OpenAI configuration', async () => {
       const { result } = renderHook(() => useGlobalStore());
       const openAIConfig: Partial<OpenAIConfig> = { OPENAI_API_KEY: 'test-key' };
 
       // Perform the action
       await act(async () => {
-        await result.current.setOpenAIConfig(openAIConfig);
+        await result.current.setModelProviderConfig('openAI', openAIConfig);
       });
 
       // Assert that updateUserSettings was called with the correct OpenAI configuration

package/src/store/global/slices/settings/action.ts

@@ -7,7 +7,7 @@ import { userService } from '@/services/user';
 import type { GlobalStore } from '@/store/global';
 import { SettingsTabs } from '@/store/global/initialState';
 import { LobeAgentSettings } from '@/types/session';
-import type { GlobalSettings, OpenAIConfig } from '@/types/settings';
+import { GlobalLLMConfig, GlobalLLMProviderKey, GlobalSettings } from '@/types/settings';
 import { difference } from '@/utils/difference';
 import { merge } from '@/utils/merge';
 
@@ -17,10 +17,14 @@ import { merge } from '@/utils/merge';
 export interface SettingsAction {
   importAppSettings: (settings: GlobalSettings) => Promise<void>;
   resetSettings: () => Promise<void>;
-  setOpenAIConfig: (config: Partial<OpenAIConfig>) => Promise<void>;
+  setModelProviderConfig: <T extends GlobalLLMProviderKey>(
+    provider: T,
+    config: Partial<GlobalLLMConfig[T]>,
+  ) => Promise<void>;
   setSettings: (settings: DeepPartial<GlobalSettings>) => Promise<void>;
   switchSettingTabs: (tab: SettingsTabs) => void;
   switchThemeMode: (themeMode: ThemeMode) => Promise<void>;
+  toggleProviderEnabled: (provider: GlobalLLMProviderKey, enabled: boolean) => Promise<void>;
   updateDefaultAgent: (agent: DeepPartial<LobeAgentSettings>) => Promise<void>;
 }
 
@@ -41,8 +45,8 @@ export const createSettingsSlice: StateCreator<
     await userService.resetUserSettings();
     await get().refreshUserConfig();
   },
-  setOpenAIConfig: async (config) => {
-    await get().setSettings({ languageModel: { openAI: config } });
+  setModelProviderConfig: async (provider, config) => {
+    await get().setSettings({ languageModel: { [provider]: config } });
   },
   setSettings: async (settings) => {
     const { settings: prevSetting, defaultSettings } = get();
@@ -63,6 +67,9 @@ export const createSettingsSlice: StateCreator<
   switchThemeMode: async (themeMode) => {
     await get().setSettings({ themeMode });
   },
+  toggleProviderEnabled: async (provider, enabled) => {
+    await get().setSettings({ languageModel: { [provider]: { enabled } } });
+  },
   updateDefaultAgent: async (defaultAgent) => {
     await get().setSettings({ defaultAgent });
   },

package/src/store/global/slices/settings/selectors/__snapshots__/modelProvider.test.ts.snap

@@ -3,98 +3,30 @@
 exports[`modelProviderSelectors > CUSTOM_MODELS > custom deletion, addition, and renaming of models 1`] = `
 [
   {
-    "displayName": "llama",
-    "name": "llama",
-  },
-  {
-    "displayName": "claude-2",
-    "name": "claude-2",
-  },
-  {
-    "displayName": "gpt-4-32k",
-    "name": "gpt-4-0125-preview",
-  },
-]
-`;
-
-exports[`modelProviderSelectors > CUSTOM_MODELS > duplicate naming model 1`] = `
-[
-  {
-    "displayName": "gpt-3.5-turbo",
-    "name": "gpt-3.5-turbo",
-  },
-  {
-    "displayName": "gpt-3.5-turbo-1106",
-    "name": "gpt-3.5-turbo-1106",
-  },
-  {
-    "displayName": "gpt-3.5-turbo-16k",
-    "name": "gpt-3.5-turbo-16k",
-  },
-  {
-    "displayName": "gpt-4",
-    "name": "gpt-4",
-  },
-  {
-    "displayName": "gpt-4-32k",
-    "name": "gpt-4-32k",
-  },
-  {
-    "displayName": "gpt-4-vision-preview",
-    "name": "gpt-4-vision-preview",
-  },
-  {
-    "displayName": "gpt-4-32k",
-    "name": "gpt-4-0125-preview",
-  },
-]
-`;
-
-exports[`modelProviderSelectors > CUSTOM_MODELS > only add the model 1`] = `
-[
-  {
-    "displayName": "gpt-3.5-turbo",
-    "name": "gpt-3.5-turbo",
-  },
-  {
-    "displayName": "gpt-3.5-turbo-1106",
-    "name": "gpt-3.5-turbo-1106",
-  },
-  {
-    "displayName": "gpt-3.5-turbo-16k",
-    "name": "gpt-3.5-turbo-16k",
-  },
-  {
-    "displayName": "gpt-4",
-    "name": "gpt-4",
-  },
-  {
-    "displayName": "gpt-4-32k",
-    "name": "gpt-4-32k",
-  },
-  {
-    "displayName": "gpt-4-0125-preview",
-    "name": "gpt-4-0125-preview",
-  },
-  {
-    "displayName": "gpt-4-vision-preview",
-    "name": "gpt-4-vision-preview",
-  },
-  {
-    "displayName": "model1",
-    "name": "model1",
-  },
-  {
-    "displayName": "model2",
-    "name": "model2",
-  },
-  {
-    "displayName": "model3",
-    "name": "model3",
-  },
-  {
-    "displayName": "model4",
-    "name": "model4",
+    "chatModels": [
+      {
+        "displayName": "llama",
+        "functionCall": true,
+        "id": "llama",
+        "isCustom": true,
+        "vision": true,
+      },
+      {
+        "displayName": "claude-2",
+        "functionCall": true,
+        "id": "claude-2",
+        "isCustom": true,
+        "vision": true,
+      },
+      {
+        "displayName": "gpt-4-32k",
+        "functionCall": true,
+        "id": "gpt-4-0125-preview",
+        "tokens": 128000,
+      },
+    ],
+    "enabled": true,
+    "id": "openai",
   },
 ]
 `;

package/src/store/global/slices/settings/selectors/__snapshots__/selectors.test.ts.snap

@@ -81,6 +81,7 @@ exports[`settingsSelectors > defaultAgent > should merge DEFAULT_AGENT and s.set
       "top_p": 1,
     },
     "plugins": [],
+    "provider": "openai",
     "systemRole": "user",
     "tts": {
       "showAllLocaleVoice": false,
@@ -112,6 +113,7 @@ exports[`settingsSelectors > defaultAgentConfig > should merge DEFAULT_AGENT_CON
     "top_p": 1,
   },
   "plugins": [],
+  "provider": "openai",
   "systemRole": "user",
   "tts": {
     "showAllLocaleVoice": false,

package/src/store/global/slices/settings/selectors/modelProvider.test.ts

@@ -1,28 +1,30 @@
-import { GlobalStore } from '../../../store';
+import { describe, expect, it } from 'vitest';
+
+import { DEFAULT_SETTINGS } from '@/const/settings';
+import { agentSelectors } from '@/store/session/slices/agent';
+import { merge } from '@/utils/merge';
+
+import { GlobalStore, useGlobalStore } from '../../../store';
+import { initialSettingsState } from '../initialState';
 import { modelProviderSelectors } from './modelProvider';
 
 describe('modelProviderSelectors', () => {
   describe('CUSTOM_MODELS', () => {
     it('custom deletion, addition, and renaming of models', () => {
-      const s = {
+      const s = merge(initialSettingsState, {
         serverConfig: {
           customModelName:
             '-all,+llama,+claude-2，-gpt-3.5-turbo,gpt-4-0125-preview=gpt-4-turbo,gpt-4-0125-preview=gpt-4-32k',
         },
-        settings: {
-          languageModel: {
-            openAI: {},
-          },
-        },
-      } as unknown as GlobalStore;
+      }) as unknown as GlobalStore;
 
-      const result = modelProviderSelectors.modelList(s);
+      const result = modelProviderSelectors.modelSelectList(s).filter((r) => r.enabled);
 
       expect(result).toMatchSnapshot();
     });
 
     it('duplicate naming model', () => {
-      const s = {
+      const s = merge(initialSettingsState, {
         serverConfig: {},
         settings: {
           languageModel: {
@@ -31,57 +33,48 @@ describe('modelProviderSelectors', () => {
             },
           },
         },
-      } as unknown as GlobalStore;
+      }) as unknown as GlobalStore;
 
-      const result = modelProviderSelectors.modelList(s);
+      const result = modelProviderSelectors.modelSelectList(s).filter((r) => r.enabled);
 
-      expect(result).toMatchSnapshot();
+      expect(result[0].chatModels.find((s) => s.id === 'gpt-4-0125-preview')?.displayName).toEqual(
+        'gpt-4-32k',
+      );
     });
 
     it('should delete model', () => {
-      const s = {
-        serverConfig: {
-          customModelName: '-gpt-4',
-        },
+      const s = merge(initialSettingsState, {
+        serverConfig: { customModelName: '-gpt-4' },
+      }) as unknown as GlobalStore;
+
+      const result = modelProviderSelectors.modelSelectList(s).filter((r) => r.enabled);
+
+      expect(result.find((r) => r.id === 'gpt-4')).toBeUndefined();
+    });
+
+    it('show the hidden model', () => {
+      const s = merge(initialSettingsState, {
+        serverConfig: {},
         settings: {
           languageModel: {
-            openAI: {},
+            openAI: {
+              customModelName: '+gpt-4-1106-preview',
+            },
           },
         },
-      } as unknown as GlobalStore;
+      }) as unknown as GlobalStore;
 
-      const result = modelProviderSelectors.modelList(s);
+      const result = modelProviderSelectors.modelSelectList(s).filter((r) => r.enabled);
 
-      expect(result).toEqual([
-        {
-          displayName: 'gpt-3.5-turbo',
-          name: 'gpt-3.5-turbo',
-        },
-        {
-          displayName: 'gpt-3.5-turbo-1106',
-          name: 'gpt-3.5-turbo-1106',
-        },
-        {
-          displayName: 'gpt-3.5-turbo-16k',
-          name: 'gpt-3.5-turbo-16k',
-        },
-        {
-          displayName: 'gpt-4-32k',
-          name: 'gpt-4-32k',
-        },
-        {
-          displayName: 'gpt-4-0125-preview',
-          name: 'gpt-4-0125-preview',
-        },
-        {
-          displayName: 'gpt-4-vision-preview',
-          name: 'gpt-4-vision-preview',
-        },
-      ]);
+      expect(result[0].chatModels.find((o) => o.id === 'gpt-4-1106-preview')).toEqual({
+        functionCall: true,
+        id: 'gpt-4-1106-preview',
+        tokens: 128000,
+      });
     });
 
     it('only add the model', () => {
-      const s = {
+      const s = merge(initialSettingsState, {
         serverConfig: {},
         settings: {
           languageModel: {
@@ -90,11 +83,95 @@ describe('modelProviderSelectors', () => {
             },
           },
         },
-      } as unknown as GlobalStore;
+      }) as unknown as GlobalStore;
 
-      const result = modelProviderSelectors.modelList(s);
+      const result = modelProviderSelectors.modelSelectList(s).filter((r) => r.enabled);
 
-      expect(result).toMatchSnapshot();
+      expect(result[0].chatModels).toContainEqual({
+        displayName: 'model1',
+        functionCall: true,
+        id: 'model1',
+        isCustom: true,
+        vision: true,
+      });
+      expect(result[0].chatModels).toContainEqual({
+        displayName: 'model2',
+        functionCall: true,
+        id: 'model2',
+        isCustom: true,
+        vision: true,
+      });
+      expect(result[0].chatModels).toContainEqual({
+        displayName: 'model3',
+        functionCall: true,
+        id: 'model3',
+        isCustom: true,
+        vision: true,
+      });
+      expect(result[0].chatModels).toContainEqual({
+        displayName: 'model4',
+        functionCall: true,
+        id: 'model4',
+        isCustom: true,
+        vision: true,
+      });
+    });
+  });
+
+  describe('modelEnabledVision', () => {
+    it('should return true if the model has vision ability', () => {
+      const hasAbility = modelProviderSelectors.modelEnabledVision('gpt-4-vision-preview')(
+        useGlobalStore.getState(),
+      );
+      expect(hasAbility).toBeTruthy();
+    });
+
+    it('should return false if the model does not have vision ability', () => {
+      const hasAbility = modelProviderSelectors.modelEnabledVision('some-other-model')(
+        useGlobalStore.getState(),
+      );
+
+      expect(hasAbility).toBeFalsy();
+    });
+
+    it('should return false if the model include vision in id', () => {
+      const hasAbility = modelProviderSelectors.modelEnabledVision('some-other-model-vision')(
+        useGlobalStore.getState(),
+      );
+
+      expect(hasAbility).toBeTruthy();
+    });
+  });
+
+  describe('modelEnabledFiles', () => {
+    it('should return false if the model does not have file ability', () => {
+      const enabledFiles = modelProviderSelectors.modelEnabledFiles('gpt-4-vision-preview')(
+        useGlobalStore.getState(),
+      );
+      expect(enabledFiles).toBeFalsy();
+    });
+
+    it('should return true if the model has file ability', () => {
+      const enabledFiles = modelProviderSelectors.modelEnabledFiles('gpt-4-all')(
+        useGlobalStore.getState(),
+      );
+      expect(enabledFiles).toBeTruthy();
+    });
+  });
+
+  describe('modelHasMaxToken', () => {
+    it('should return true if the model is in the list of models that show tokens', () => {
+      const show = modelProviderSelectors.modelHasMaxToken('gpt-3.5-turbo')(
+        useGlobalStore.getState(),
+      );
+      expect(show).toBeTruthy();
+    });
+
+    it('should return false if the model is not in the list of models that show tokens', () => {
+      const show = modelProviderSelectors.modelHasMaxToken('some-other-model')(
+        useGlobalStore.getState(),
+      );
+      expect(show).toBe(false);
     });
   });
 });