@lobehub/lobehub 2.0.0-next.15 → 2.0.0-next.17

package/packages/utils/src/_deprecated/parseModels.test.ts

@@ -1,287 +0,0 @@
-import { ChatModelCard } from '@lobechat/types';
-import { describe, expect, it } from 'vitest';
-
-import { LOBE_DEFAULT_MODEL_LIST, OpenAIProviderCard } from '@/config/modelProviders';
-
-import { parseModelString, transformToChatModelCards } from './parseModels';
-
-describe('parseModelString', () => {
-  it('custom deletion, addition, and renaming of models', () => {
-    const result = parseModelString(
-      '-all,+llama,+claude-2，-gpt-3.5-turbo,gpt-4-1106-preview=gpt-4-turbo,gpt-4-1106-preview=gpt-4-32k',
-    );
-
-    expect(result).toMatchSnapshot();
-  });
-
-  it('duplicate naming model', () => {
-    const result = parseModelString('gpt-4-1106-preview=gpt-4-turbo，gpt-4-1106-preview=gpt-4-32k');
-    expect(result).toMatchSnapshot();
-  });
-
-  it('only add the model', () => {
-    const result = parseModelString('model1,model2,model3，model4');
-
-    expect(result).toMatchSnapshot();
-  });
-
-  it('empty string model', () => {
-    const result = parseModelString('gpt-4-1106-preview=gpt-4-turbo,,  ,\n  ，+claude-2');
-    expect(result).toMatchSnapshot();
-  });
-
-  describe('extension capabilities', () => {
-    it('with token', () => {
-      const result = parseModelString('chatglm-6b=ChatGLM 6B<4096>');
-
-      expect(result.add[0]).toEqual({
-        displayName: 'ChatGLM 6B',
-        id: 'chatglm-6b',
-        contextWindowTokens: 4096,
-      });
-    });
-
-    it('token and function calling', () => {
-      const result = parseModelString('spark-v3.5=讯飞星火 v3.5<8192:fc>');
-
-      expect(result.add[0]).toEqual({
-        displayName: '讯飞星火 v3.5',
-        functionCall: true,
-        id: 'spark-v3.5',
-        contextWindowTokens: 8192,
-      });
-    });
-
-    it('token and reasoning', () => {
-      const result = parseModelString('deepseek-r1=Deepseek R1<65536:reasoning>');
-
-      expect(result.add[0]).toEqual({
-        displayName: 'Deepseek R1',
-        reasoning: true,
-        id: 'deepseek-r1',
-        contextWindowTokens: 65_536,
-      });
-    });
-
-    it('multi models', () => {
-      const result = parseModelString(
-        'gemini-1.5-flash-latest=Gemini 1.5 Flash<16000:vision>,gpt-4-all=ChatGPT Plus<128000:fc:vision:file>',
-      );
-
-      expect(result.add).toEqual([
-        {
-          displayName: 'Gemini 1.5 Flash',
-          vision: true,
-          id: 'gemini-1.5-flash-latest',
-          contextWindowTokens: 16000,
-        },
-        {
-          displayName: 'ChatGPT Plus',
-          vision: true,
-          functionCall: true,
-          files: true,
-          id: 'gpt-4-all',
-          contextWindowTokens: 128000,
-        },
-      ]);
-    });
-
-    it('should have file with builtin models like gpt-4-0125-preview', () => {
-      const result = parseModelString(
-        '-all,+gpt-4-0125-preview=ChatGPT-4<128000:fc:file>,+gpt-4-turbo-2024-04-09=ChatGPT-4 Vision<128000:fc:vision:file>',
-      );
-      expect(result.add).toEqual([
-        {
-          displayName: 'ChatGPT-4',
-          files: true,
-          functionCall: true,
-          id: 'gpt-4-0125-preview',
-          contextWindowTokens: 128000,
-        },
-        {
-          displayName: 'ChatGPT-4 Vision',
-          files: true,
-          functionCall: true,
-          id: 'gpt-4-turbo-2024-04-09',
-          contextWindowTokens: 128000,
-          vision: true,
-        },
-      ]);
-    });
-
-    it('should handle empty extension capability value', () => {
-      const result = parseModelString('model1<1024:>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024 });
-    });
-
-    it('should handle empty extension capability name', () => {
-      const result = parseModelString('model1<1024::file>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024, files: true });
-    });
-
-    it('should handle duplicate extension capabilities', () => {
-      const result = parseModelString('model1<1024:vision:vision>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024, vision: true });
-    });
-
-    it('should handle case-sensitive extension capability names', () => {
-      const result = parseModelString('model1<1024:VISION:FC:file>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024, files: true });
-    });
-
-    it('should handle case-sensitive extension capability values', () => {
-      const result = parseModelString('model1<1024:vision:Fc:File>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024, vision: true });
-    });
-
-    it('should handle empty angle brackets', () => {
-      const result = parseModelString('model1<>');
-      expect(result.add[0]).toEqual({ id: 'model1' });
-    });
-
-    it('should handle not close angle brackets', () => {
-      const result = parseModelString('model1<,model2');
-      expect(result.add).toEqual([{ id: 'model1' }, { id: 'model2' }]);
-    });
-
-    it('should handle multi close angle brackets', () => {
-      const result = parseModelString('model1<>>,model2');
-      expect(result.add).toEqual([{ id: 'model1' }, { id: 'model2' }]);
-    });
-
-    it('should handle only colon inside angle brackets', () => {
-      const result = parseModelString('model1<:>');
-      expect(result.add[0]).toEqual({ id: 'model1' });
-    });
-
-    it('should handle only non-digit characters inside angle brackets', () => {
-      const result = parseModelString('model1<abc>');
-      expect(result.add[0]).toEqual({ id: 'model1' });
-    });
-
-    it('should handle non-digit characters followed by digits inside angle brackets', () => {
-      const result = parseModelString('model1<abc123>');
-      expect(result.add[0]).toEqual({ id: 'model1' });
-    });
-
-    it('should handle digits followed by non-colon characters inside angle brackets', () => {
-      const result = parseModelString('model1<1024abc>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024 });
-    });
-
-    it('should handle digits followed by multiple colons inside angle brackets', () => {
-      const result = parseModelString('model1<1024::>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024 });
-    });
-
-    it('should handle digits followed by a colon and non-letter characters inside angle brackets', () => {
-      const result = parseModelString('model1<1024:123>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024 });
-    });
-
-    it('should handle digits followed by a colon and spaces inside angle brackets', () => {
-      const result = parseModelString('model1<1024: vision>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024 });
-    });
-
-    it('should handle digits followed by multiple colons and spaces inside angle brackets', () => {
-      const result = parseModelString('model1<1024: : vision>');
-      expect(result.add[0]).toEqual({ id: 'model1', contextWindowTokens: 1024 });
-    });
-  });
-
-  describe('deployment name', () => {
-    it('should have same deployment name as id', () => {
-      const result = parseModelString('model1=Model 1', true);
-      expect(result.add[0]).toEqual({
-        id: 'model1',
-        displayName: 'Model 1',
-        deploymentName: 'model1',
-      });
-    });
-
-    it('should have diff deployment name as id', () => {
-      const result = parseModelString('gpt-35-turbo->my-deploy=GPT 3.5 Turbo', true);
-      expect(result.add[0]).toEqual({
-        id: 'gpt-35-turbo',
-        displayName: 'GPT 3.5 Turbo',
-        deploymentName: 'my-deploy',
-      });
-    });
-  });
-});
-
-describe('transformToChatModelCards', () => {
-  const defaultChatModels: ChatModelCard[] = [
-    { id: 'model1', displayName: 'Model 1', enabled: true },
-    { id: 'model2', displayName: 'Model 2', enabled: false },
-  ];
-
-  it('should return undefined when modelString is empty', () => {
-    const result = transformToChatModelCards({
-      modelString: '',
-      defaultChatModels,
-    });
-    expect(result).toBeUndefined();
-  });
-
-  it('should remove all models when removeAll is true', () => {
-    const result = transformToChatModelCards({
-      modelString: '-all',
-      defaultChatModels,
-    });
-    expect(result).toEqual([]);
-  });
-
-  it('should remove specified models', () => {
-    const result = transformToChatModelCards({
-      modelString: '-model1',
-      defaultChatModels,
-    });
-    expect(result).toEqual([{ id: 'model2', displayName: 'Model 2', enabled: false }]);
-  });
-
-  it('should add a new known model', () => {
-    const knownModel = LOBE_DEFAULT_MODEL_LIST[0];
-    const result = transformToChatModelCards({
-      modelString: `${knownModel.id}`,
-      defaultChatModels,
-    });
-    expect(result).toContainEqual({
-      ...knownModel,
-      displayName: knownModel.displayName || knownModel.id,
-      enabled: true,
-    });
-  });
-
-  it('should update an existing known model', () => {
-    const knownModel = LOBE_DEFAULT_MODEL_LIST[0];
-    const result = transformToChatModelCards({
-      modelString: `+${knownModel.id}=Updated Model`,
-      defaultChatModels: [knownModel],
-    });
-    expect(result![0]).toEqual({ ...knownModel, displayName: 'Updated Model', enabled: true });
-  });
-
-  it('should add a new custom model', () => {
-    const result = transformToChatModelCards({
-      modelString: '+custom_model=Custom Model',
-      defaultChatModels,
-    });
-    expect(result).toContainEqual({
-      id: 'custom_model',
-      displayName: 'Custom Model',
-      enabled: true,
-    });
-  });
-
-  it('should have file with builtin models like gpt-4-0125-preview', () => {
-    const result = transformToChatModelCards({
-      modelString:
-        '-all,+gpt-4-0125-preview=ChatGPT-4<128000:fc:file>,+gpt-4-turbo-2024-04-09=ChatGPT-4 Vision<128000:fc:vision:file>',
-      defaultChatModels: OpenAIProviderCard.chatModels,
-    });
-
-    expect(result).toMatchSnapshot();
-  });
-});

package/packages/utils/src/_deprecated/parseModels.ts

@@ -1,165 +0,0 @@
-import { ChatModelCard } from '@lobechat/types';
-import { produce } from 'immer';
-
-import { LOBE_DEFAULT_MODEL_LIST } from '@/config/modelProviders';
-
-/**
- * Parse model string to add or remove models.
- */
-export const parseModelString = (modelString: string = '', withDeploymentName = false) => {
-  let models: ChatModelCard[] = [];
-  let removeAll = false;
-  const removedModels: string[] = [];
-  const modelNames = modelString.split(/[,，]/).filter(Boolean);
-
-  for (const item of modelNames) {
-    const disable = item.startsWith('-');
-    const nameConfig = item.startsWith('+') || item.startsWith('-') ? item.slice(1) : item;
-    const [idAndDisplayName, ...capabilities] = nameConfig.split('<');
-    let [id, displayName] = idAndDisplayName.split('=');
-
-    let deploymentName: string | undefined;
-
-    if (withDeploymentName) {
-      [id, deploymentName] = id.split('->');
-      if (!deploymentName) deploymentName = id;
-    }
-
-    if (disable) {
-      // Disable all models.
-      if (id === 'all') {
-        removeAll = true;
-      }
-      removedModels.push(id);
-      continue;
-    }
-
-    // remove empty model name
-    if (!item.trim().length) {
-      continue;
-    }
-
-    // Remove duplicate model entries.
-    const existingIndex = models.findIndex(({ id: n }) => n === id);
-    if (existingIndex !== -1) {
-      models.splice(existingIndex, 1);
-    }
-
-    const model: ChatModelCard = {
-      displayName: displayName || undefined,
-      id,
-    };
-
-    if (deploymentName) {
-      model.deploymentName = deploymentName;
-    }
-
-    if (capabilities.length > 0) {
-      const [maxTokenStr, ...capabilityList] = capabilities[0].replace('>', '').split(':');
-      model.contextWindowTokens = parseInt(maxTokenStr, 10) || undefined;
-
-      for (const capability of capabilityList) {
-        switch (capability) {
-          case 'reasoning': {
-            model.reasoning = true;
-            break;
-          }
-          case 'vision': {
-            model.vision = true;
-            break;
-          }
-          case 'fc': {
-            model.functionCall = true;
-            break;
-          }
-          case 'file': {
-            model.files = true;
-            break;
-          }
-          default: {
-            console.warn(`Unknown capability: ${capability}`);
-          }
-        }
-      }
-    }
-
-    models.push(model);
-  }
-
-  return {
-    add: models,
-    removeAll,
-    removed: removedModels,
-  };
-};
-
-/**
- * Extract a special method to process chatModels
- */
-export const transformToChatModelCards = ({
-  modelString = '',
-  defaultChatModels,
-  withDeploymentName = false,
-}: {
-  defaultChatModels: ChatModelCard[];
-  modelString?: string;
-  withDeploymentName?: boolean;
-}): ChatModelCard[] | undefined => {
-  if (!modelString) return undefined;
-
-  const modelConfig = parseModelString(modelString, withDeploymentName);
-  let chatModels = modelConfig.removeAll ? [] : defaultChatModels;
-
-  // 处理移除逻辑
-  if (!modelConfig.removeAll) {
-    chatModels = chatModels.filter((m) => !modelConfig.removed.includes(m.id));
-  }
-
-  return produce(chatModels, (draft) => {
-    // 处理添加或替换逻辑
-    for (const toAddModel of modelConfig.add) {
-      // first try to find the model in LOBE_DEFAULT_MODEL_LIST to confirm if it is a known model
-      const knownModel = LOBE_DEFAULT_MODEL_LIST.find((model) => model.id === toAddModel.id);
-
-      // if the model is known, update it based on the known model
-      if (knownModel) {
-        const index = draft.findIndex((model) => model.id === toAddModel.id);
-        const modelInList = draft[index];
-
-        // if the model is already in chatModels, update it
-        if (modelInList) {
-          draft[index] = {
-            ...modelInList,
-            ...toAddModel,
-            displayName: toAddModel.displayName || modelInList.displayName || modelInList.id,
-            enabled: true,
-          };
-        } else {
-          // if the model is not in chatModels, add it
-          draft.push({
-            ...knownModel,
-            ...toAddModel,
-            displayName: toAddModel.displayName || knownModel.displayName || knownModel.id,
-            enabled: true,
-          });
-        }
-      } else {
-        // if the model is not in LOBE_DEFAULT_MODEL_LIST, add it as a new custom model
-        draft.push({
-          ...toAddModel,
-          displayName: toAddModel.displayName || toAddModel.id,
-          enabled: true,
-        });
-      }
-    }
-  });
-};
-
-export const extractEnabledModels = (modelString: string = '', withDeploymentName = false) => {
-  const modelConfig = parseModelString(modelString, withDeploymentName);
-  const list = modelConfig.add.map((m) => m.id);
-
-  if (list.length === 0) return;
-
-  return list;
-};

package/src/hooks/_header.ts

@@ -1,23 +0,0 @@
-import { LOBE_CHAT_ACCESS_CODE, OPENAI_API_KEY_HEADER_KEY, OPENAI_END_POINT } from '@/const/fetch';
-import { useUserStore } from '@/store/user';
-import { keyVaultsConfigSelectors } from '@/store/user/selectors';
-
-/**
- * TODO: Need to be removed after tts refactor
- * @deprecated
- */
-// eslint-disable-next-line no-undef
-export const createHeaderWithOpenAI = (header?: HeadersInit): HeadersInit => {
-  const openai = keyVaultsConfigSelectors.openAIConfig(useUserStore.getState());
-
-  const apiKey = openai.apiKey || '';
-  const endpoint = openai.baseURL || '';
-
-  // eslint-disable-next-line no-undef
-  return {
-    ...header,
-    [LOBE_CHAT_ACCESS_CODE]: keyVaultsConfigSelectors.password(useUserStore.getState()),
-    [OPENAI_API_KEY_HEADER_KEY]: apiKey,
-    [OPENAI_END_POINT]: endpoint,
-  };
-};

package/src/server/globalConfig/_deprecated.test.ts

@@ -1,92 +0,0 @@
-import { describe, expect, it, vi } from 'vitest';
-
-import { genServerLLMConfig } from './_deprecated';
-
-// Mock ModelProvider enum
-vi.mock('model-bank', () => ({
-  ModelProvider: {
-    Azure: 'azure',
-    Bedrock: 'bedrock',
-    Ollama: 'ollama',
-  },
-}));
-
-// Mock ProviderCards
-vi.mock('@/config/modelProviders', () => ({
-  azureProviderCard: {
-    chatModels: [],
-  },
-  bedrockProviderCard: {
-    chatModels: ['bedrockModel1', 'bedrockModel2'],
-  },
-  ollamaProviderCard: {
-    chatModels: ['ollamaModel1', 'ollamaModel2'],
-  },
-}));
-
-// Mock LLM config
-vi.mock('@/envs/llm', () => ({
-  getLLMConfig: () => ({
-    ENABLED_AZURE_OPENAI: true,
-    ENABLED_AWS_BEDROCK: true,
-    ENABLED_OLLAMA: true,
-    AZURE_MODEL_LIST: 'azureModels',
-    AWS_BEDROCK_MODEL_LIST: 'bedrockModels',
-    OLLAMA_MODEL_LIST: 'ollamaModels',
-    OLLAMA_PROXY_URL: '',
-  }),
-}));
-
-// Mock parse models utils
-vi.mock('@/utils/_deprecated/parseModels', () => ({
-  extractEnabledModels: (modelString: string, withDeploymentName?: boolean) => {
-    // Returns different format if withDeploymentName is true
-    return withDeploymentName ? [`${modelString}_withDeployment`] : [modelString];
-  },
-  transformToChatModelCards: ({ defaultChatModels, modelString, withDeploymentName }: any) => {
-    // Simulate transformation based on withDeploymentName
-    return withDeploymentName ? [`${modelString}_transformed`] : defaultChatModels;
-  },
-}));
-
-describe('genServerLLMConfig', () => {
-  it('should generate correct LLM config for Azure, Bedrock, and Ollama', () => {
-    vi.stubEnv('AZURE_MODEL_LIST', 'azureModels');
-    vi.stubEnv('AWS_BEDROCK_MODEL_LIST', 'bedrockModels');
-    vi.stubEnv('OLLAMA_MODEL_LIST', 'ollamaModels');
-
-    const specificConfig = {
-      azure: {
-        enabledKey: 'ENABLED_AZURE_OPENAI',
-        withDeploymentName: true,
-      },
-      bedrock: {
-        enabledKey: 'ENABLED_AWS_BEDROCK',
-        modelListKey: 'AWS_BEDROCK_MODEL_LIST',
-      },
-      ollama: {
-        fetchOnClient: !process.env.OLLAMA_PROXY_URL,
-      },
-    };
-    const config = genServerLLMConfig(specificConfig);
-
-    expect(config.azure).toEqual({
-      enabled: true,
-      enabledModels: ['azureModels_withDeployment'],
-      serverModelCards: ['azureModels_transformed'],
-    });
-
-    expect(config.bedrock).toEqual({
-      enabled: true,
-      enabledModels: ['bedrockModels'],
-      serverModelCards: ['bedrockModel1', 'bedrockModel2'],
-    });
-
-    expect(config.ollama).toEqual({
-      enabled: true,
-      enabledModels: ['ollamaModels'],
-      fetchOnClient: true,
-      serverModelCards: ['ollamaModel1', 'ollamaModel2'],
-    });
-  });
-});

package/src/server/globalConfig/_deprecated.ts

@@ -1,41 +0,0 @@
-import { ModelProvider } from 'model-bank';
-
-import * as ProviderCards from '@/config/modelProviders';
-import { getLLMConfig } from '@/envs/llm';
-import { ModelProviderCard } from '@/types/llm';
-import { extractEnabledModels, transformToChatModelCards } from '@/utils/_deprecated/parseModels';
-
-export const genServerLLMConfig = (specificConfig: Record<any, any>) => {
-  const llmConfig = getLLMConfig() as Record<string, any>;
-
-  return Object.values(ModelProvider).reduce(
-    (config, provider) => {
-      const providerUpperCase = provider.toUpperCase();
-      const providerCard = ProviderCards[
-        `${provider}ProviderCard` as keyof typeof ProviderCards
-      ] as ModelProviderCard;
-      const providerConfig = specificConfig[provider as keyof typeof specificConfig] || {};
-      const providerModelList =
-        process.env[providerConfig.modelListKey ?? `${providerUpperCase}_MODEL_LIST`];
-
-      config[provider] = {
-        enabled: llmConfig[providerConfig.enabledKey || `ENABLED_${providerUpperCase}`],
-        enabledModels: extractEnabledModels(
-          providerModelList,
-          providerConfig.withDeploymentName || false,
-        ),
-        serverModelCards: transformToChatModelCards({
-          defaultChatModels: (providerCard as ModelProviderCard)?.chatModels || [],
-          modelString: providerModelList,
-          withDeploymentName: providerConfig.withDeploymentName || false,
-        }),
-        ...(providerConfig.fetchOnClient !== undefined && {
-          fetchOnClient: providerConfig.fetchOnClient,
-        }),
-      };
-
-      return config;
-    },
-    {} as Record<ModelProvider, any>,
-  );
-};

package/src/store/global/actions/clientDb.ts

@@ -1,67 +0,0 @@
-import { SWRResponse } from 'swr';
-import type { StateCreator } from 'zustand/vanilla';
-
-import { useOnlyFetchOnceSWR } from '@/libs/swr';
-import type { GlobalStore } from '@/store/global';
-import { DatabaseLoadingState, OnStageChange } from '@/types/clientDB';
-
-type InitClientDBParams = { onStateChange: OnStageChange };
-/**
- * 设置操作
- */
-export interface GlobalClientDBAction {
-  initializeClientDB: (params?: InitClientDBParams) => Promise<void>;
-  markPgliteEnabled: () => void;
-  useInitClientDB: (params?: InitClientDBParams) => SWRResponse;
-}
-
-export const clientDBSlice: StateCreator<
-  GlobalStore,
-  [['zustand/devtools', never]],
-  [],
-  GlobalClientDBAction
-> = (set, get) => ({
-  initializeClientDB: async (params) => {
-    // if the db has started initialized or not error, just skip.
-    if (
-      get().initClientDBStage !== DatabaseLoadingState.Idle &&
-      get().initClientDBStage !== DatabaseLoadingState.Error
-    )
-      return;
-
-    const { initializeDB } = await import('@/database/client/db');
-    await initializeDB({
-      onError: ({ error, migrationsSQL, migrationTableItems }) => {
-        set({
-          initClientDBError: error,
-          initClientDBMigrations: {
-            sqls: migrationsSQL,
-            tableRecords: migrationTableItems,
-          },
-        });
-      },
-      onProgress: (data) => {
-        set({ initClientDBProcess: data });
-      },
-      onStateChange: (state) => {
-        set({ initClientDBStage: state });
-        params?.onStateChange?.(state);
-      },
-    });
-  },
-  markPgliteEnabled: async () => {
-    get().updateSystemStatus({ isEnablePglite: true });
-
-    if (navigator.storage && !!navigator.storage.persist) {
-      // 1. Check if persistent permission has been obtained
-      const isPersisted = await navigator.storage.persisted();
-
-      // 2. If the persistent permission has not been obtained, request permission
-      if (!isPersisted) {
-        await navigator.storage.persist();
-      }
-    }
-  },
-  useInitClientDB: (params) =>
-    useOnlyFetchOnceSWR('initClientDB', () => get().initializeClientDB(params)),
-});

package/src/store/user/slices/modelList/__snapshots__/action.test.ts.snap

@@ -1,12 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`LLMSettingsSliceAction > refreshModelProviderList > visible 1`] = `
-{
-  "contextWindowTokens": 4096,
-  "description": "LLaVA 是结合视觉编码器和 Vicuna 的多模态模型，用于强大的视觉和语言理解。",
-  "displayName": "LLaVA 7B",
-  "enabled": true,
-  "id": "llava",
-  "vision": true,
-}
-`;