@lobehub/chat 0.146.2 → 0.147.0

package/src/app/api/chat/[provider]/route.ts

@@ -1,40 +1,23 @@
 import { getPreferredRegion } from '@/app/api/config';
 import { createErrorResponse } from '@/app/api/errorResponse';
-import { LOBE_CHAT_AUTH_HEADER, OAUTH_AUTHORIZED } from '@/const/auth';
-import { AgentRuntimeError, ChatCompletionErrorPayload } from '@/libs/agent-runtime';
+import { ChatCompletionErrorPayload } from '@/libs/agent-runtime';
 import { ChatErrorType } from '@/types/fetch';
 import { ChatStreamPayload } from '@/types/openai/chat';
 import { getTracePayload } from '@/utils/trace';
 
-import { checkAuthMethod, getJWTPayload } from '../auth';
-import AgentRuntime from './agentRuntime';
+import AgentRuntime from '../agentRuntime';
+import { checkAuth } from '../auth';
 
 export const runtime = 'edge';
 
 export const preferredRegion = getPreferredRegion();
 
-export const POST = async (req: Request, { params }: { params: { provider: string } }) => {
+export const POST = checkAuth(async (req: Request, { params, jwtPayload }) => {
   const { provider } = params;
 
   try {
     // ============  1. init chat model   ============ //
-
-    // get Authorization from header
-    const authorization = req.headers.get(LOBE_CHAT_AUTH_HEADER);
-    const oauthAuthorized = !!req.headers.get(OAUTH_AUTHORIZED);
-
-    if (!authorization) throw AgentRuntimeError.createError(ChatErrorType.Unauthorized);
-
-    // check the Auth With payload
-    const jwtPayload = await getJWTPayload(authorization);
-    checkAuthMethod(jwtPayload.accessCode, jwtPayload.apiKey, oauthAuthorized);
-
-    const body = await req.clone().json();
-    const agentRuntime = await AgentRuntime.initializeWithUserPayload(provider, jwtPayload, {
-      apiVersion: jwtPayload.azureApiVersion,
-      model: body.model,
-      useAzure: jwtPayload.useAzure,
-    });
+    const agentRuntime = await AgentRuntime.initializeWithUserPayload(provider, jwtPayload);
 
     // ============  2. create chat completion   ============ //
 
@@ -60,4 +43,4 @@ export const POST = async (req: Request, { params }: { params: { provider: strin
 
     return createErrorResponse(errorType, { error, ...res, provider });
   }
-};
+});

package/src/app/api/chat/{[provider]/agentRuntime.test.ts → agentRuntime.test.ts}

@@ -18,6 +18,7 @@ import {
   LobeOpenAI,
   LobeOpenRouterAI,
   LobePerplexityAI,
+  LobeRuntimeAI,
   LobeTogetherAI,
   LobeZhipuAI,
   ModelProvider,
@@ -70,33 +71,32 @@ describe('AgentRuntime', () => {
         const jwtPayload: JWTPayload = {
           apiKey: 'user-azure-key',
           endpoint: 'user-azure-endpoint',
-          useAzure: true,
+          azureApiVersion: '2024-02-01',
         };
         const runtime = await AgentRuntime.initializeWithUserPayload(
-          ModelProvider.OpenAI,
+          ModelProvider.Azure,
           jwtPayload,
         );
 
         expect(runtime).toBeInstanceOf(AgentRuntime);
-        expect(runtime['_runtime']).toBeInstanceOf(LobeOpenAI);
+        expect(runtime['_runtime']).toBeInstanceOf(LobeAzureOpenAI);
         expect(runtime['_runtime'].baseURL).toBe('user-azure-endpoint');
       });
       it('should initialize with azureOpenAIParams correctly', async () => {
-        const jwtPayload: JWTPayload = { apiKey: 'user-openai-key', endpoint: 'user-endpoint' };
-        const azureOpenAIParams = {
-          apiVersion: 'custom-version',
-          model: 'custom-model',
-          useAzure: true,
+        const jwtPayload: JWTPayload = {
+          apiKey: 'user-openai-key',
+          endpoint: 'user-endpoint',
+          azureApiVersion: 'custom-version',
         };
+
         const runtime = await AgentRuntime.initializeWithUserPayload(
-          ModelProvider.OpenAI,
+          ModelProvider.Azure,
           jwtPayload,
-          azureOpenAIParams,
         );
 
         expect(runtime).toBeInstanceOf(AgentRuntime);
-        const openAIRuntime = runtime['_runtime'] as LobeOpenAI;
-        expect(openAIRuntime).toBeInstanceOf(LobeOpenAI);
+        const openAIRuntime = runtime['_runtime'] as LobeRuntimeAI;
+        expect(openAIRuntime).toBeInstanceOf(LobeAzureOpenAI);
       });
 
       it('should initialize with AzureAI correctly', async () => {

package/src/app/api/chat/{[provider]/agentRuntime.ts → agentRuntime.ts}

@@ -28,13 +28,7 @@ import {
 } from '@/libs/agent-runtime';
 import { TraceClient } from '@/libs/traces';
 
-import apiKeyManager from '../apiKeyManager';
-
-interface AzureOpenAIParams {
-  apiVersion?: string;
-  model: string;
-  useAzure?: boolean;
-}
+import apiKeyManager from './apiKeyManager';
 
 export interface AgentChatOptions {
   enableTrace?: boolean;
@@ -112,18 +106,18 @@ class AgentRuntime {
     });
   }
 
-  static async initializeWithUserPayload(
-    provider: string,
-    payload: JWTPayload,
-    azureOpenAI?: AzureOpenAIParams,
-  ) {
+  async models() {
+    return this._runtime.models?.();
+  }
+
+  static async initializeWithUserPayload(provider: string, payload: JWTPayload) {
     let runtimeModel: LobeRuntimeAI;
 
     switch (provider) {
       default:
       case 'oneapi':
       case ModelProvider.OpenAI: {
-        runtimeModel = this.initOpenAI(payload, azureOpenAI);
+        runtimeModel = this.initOpenAI(payload);
         break;
       }
 
@@ -196,27 +190,14 @@ class AgentRuntime {
     return new AgentRuntime(runtimeModel);
   }
 
-  private static initOpenAI(payload: JWTPayload, azureOpenAI?: AzureOpenAIParams) {
-    const { OPENAI_API_KEY, OPENAI_PROXY_URL, AZURE_API_VERSION, AZURE_API_KEY, USE_AZURE_OPENAI } =
-      getServerConfig();
+  private static initOpenAI(payload: JWTPayload) {
+    const { OPENAI_API_KEY, OPENAI_PROXY_URL } = getServerConfig();
     const openaiApiKey = payload?.apiKey || OPENAI_API_KEY;
     const baseURL = payload?.endpoint || OPENAI_PROXY_URL;
 
-    const azureApiKey = payload.apiKey || AZURE_API_KEY;
-    const useAzure = azureOpenAI?.useAzure || USE_AZURE_OPENAI;
-    const apiVersion = azureOpenAI?.apiVersion || AZURE_API_VERSION;
-
-    const apiKey = apiKeyManager.pick(useAzure ? azureApiKey : openaiApiKey);
+    const apiKey = apiKeyManager.pick(openaiApiKey);
 
-    return new LobeOpenAI({
-      apiKey,
-      azureOptions: {
-        apiVersion,
-        model: azureOpenAI?.model,
-      },
-      baseURL,
-      useAzure,
-    });
+    return new LobeOpenAI({ apiKey, baseURL });
   }
 
   private static initAzureOpenAI(payload: JWTPayload) {

package/src/app/api/chat/auth/index.ts

@@ -0,0 +1,42 @@
+import { createErrorResponse } from '@/app/api/errorResponse';
+import { JWTPayload, LOBE_CHAT_AUTH_HEADER, OAUTH_AUTHORIZED } from '@/const/auth';
+import { AgentRuntimeError, ChatCompletionErrorPayload } from '@/libs/agent-runtime';
+import { ChatErrorType } from '@/types/fetch';
+
+import { checkAuthMethod, getJWTPayload } from './utils';
+
+type RequestOptions = { params: { provider: string } };
+
+export type RequestHandler = (
+  req: Request,
+  options: RequestOptions & { jwtPayload: JWTPayload },
+) => Promise<Response>;
+
+export const checkAuth =
+  (handler: RequestHandler) => async (req: Request, options: RequestOptions) => {
+    let jwtPayload: JWTPayload;
+
+    try {
+      // get Authorization from header
+      const authorization = req.headers.get(LOBE_CHAT_AUTH_HEADER);
+      const oauthAuthorized = !!req.headers.get(OAUTH_AUTHORIZED);
+
+      if (!authorization) throw AgentRuntimeError.createError(ChatErrorType.Unauthorized);
+
+      // check the Auth With payload
+      jwtPayload = await getJWTPayload(authorization);
+      checkAuthMethod(jwtPayload.accessCode, jwtPayload.apiKey, oauthAuthorized);
+    } catch (e) {
+      const {
+        errorType = ChatErrorType.InternalServerError,
+        error: errorContent,
+        ...res
+      } = e as ChatCompletionErrorPayload;
+
+      const error = errorContent || e;
+
+      return createErrorResponse(errorType, { error, ...res, provider: options.params?.provider });
+    }
+
+    return handler(req, { ...options, jwtPayload });
+  };

package/src/app/api/chat/models/[provider]/route.ts

@@ -0,0 +1,45 @@
+import { NextResponse } from 'next/server';
+
+import { getPreferredRegion } from '@/app/api/config';
+import { createErrorResponse } from '@/app/api/errorResponse';
+import { ChatCompletionErrorPayload, ModelProvider } from '@/libs/agent-runtime';
+import { ChatErrorType } from '@/types/fetch';
+
+import AgentRuntime from '../../agentRuntime';
+import { checkAuth } from '../../auth';
+
+export const runtime = 'edge';
+
+export const preferredRegion = getPreferredRegion();
+
+const noNeedAPIKey = (provider: string) =>
+  [ModelProvider.OpenRouter, ModelProvider.TogetherAI].includes(provider as any);
+
+export const GET = checkAuth(async (req, { params, jwtPayload }) => {
+  const { provider } = params;
+
+  try {
+    const hasDefaultApiKey = jwtPayload.apiKey || 'dont-need-api-key-for-model-list';
+
+    const agentRuntime = await AgentRuntime.initializeWithUserPayload(provider, {
+      ...jwtPayload,
+      apiKey: noNeedAPIKey(provider) ? hasDefaultApiKey : jwtPayload.apiKey,
+    });
+
+    const list = await agentRuntime.models();
+
+    return NextResponse.json(list);
+  } catch (e) {
+    const {
+      errorType = ChatErrorType.InternalServerError,
+      error: errorContent,
+      ...res
+    } = e as ChatCompletionErrorPayload;
+
+    const error = errorContent || e;
+    // track the error at server side
+    console.error(`Route: [${provider}] ${errorType}:`, error);
+
+    return createErrorResponse(errorType, { error, ...res, provider });
+  }
+});

package/src/app/api/config/__snapshots__/route.test.ts.snap

@@ -0,0 +1,127 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`GET /api/config > Model Provider env > CUSTOM_MODELS > custom deletion, addition, and renaming of models 1`] = `
+[
+  {
+    "displayName": "llama",
+    "enabled": true,
+    "functionCall": true,
+    "id": "llama",
+    "vision": true,
+  },
+  {
+    "displayName": "claude-2",
+    "enabled": true,
+    "functionCall": true,
+    "id": "claude-2",
+    "vision": true,
+  },
+  {
+    "displayName": "gpt-4-32k",
+    "enabled": true,
+    "functionCall": true,
+    "id": "gpt-4-0125-preview",
+    "tokens": 128000,
+  },
+]
+`;
+
+exports[`GET /api/config > Model Provider env > OPENAI_MODEL_LIST > custom deletion, addition, and renaming of models 1`] = `
+[
+  {
+    "displayName": "llama",
+    "enabled": true,
+    "functionCall": true,
+    "id": "llama",
+    "vision": true,
+  },
+  {
+    "displayName": "claude-2",
+    "enabled": true,
+    "functionCall": true,
+    "id": "claude-2",
+    "vision": true,
+  },
+  {
+    "displayName": "gpt-4-32k",
+    "enabled": true,
+    "functionCall": true,
+    "id": "gpt-4-0125-preview",
+    "tokens": 128000,
+  },
+]
+`;
+
+exports[`GET /api/config > Model Provider env > OPENAI_MODEL_LIST > should work correct with gpt-4 1`] = `
+[
+  {
+    "displayName": "GPT-3.5 Turbo (1106)",
+    "enabled": true,
+    "functionCall": true,
+    "id": "gpt-3.5-turbo-1106",
+    "tokens": 16385,
+  },
+  {
+    "description": "GPT 3.5 Turbo，适用于各种文本生成和理解任务",
+    "displayName": "GPT-3.5 Turbo",
+    "enabled": true,
+    "functionCall": true,
+    "id": "gpt-3.5-turbo",
+    "tokens": 16385,
+  },
+  {
+    "displayName": "GPT-3.5 Turbo 16K",
+    "enabled": true,
+    "id": "gpt-3.5-turbo-16k",
+    "tokens": 16385,
+  },
+  {
+    "displayName": "GPT-4",
+    "enabled": true,
+    "functionCall": true,
+    "id": "gpt-4",
+    "tokens": 8192,
+  },
+  {
+    "displayName": "GPT-4 32K",
+    "enabled": true,
+    "functionCall": true,
+    "id": "gpt-4-32k",
+    "tokens": 32768,
+  },
+  {
+    "displayName": "GPT-4 Turbo Preview (1106)",
+    "enabled": true,
+    "functionCall": true,
+    "id": "gpt-4-1106-preview",
+    "tokens": 128000,
+  },
+  {
+    "description": "GPT-4 视觉预览版，支持视觉任务",
+    "displayName": "GPT-4 Turbo Vision Preview",
+    "enabled": true,
+    "id": "gpt-4-vision-preview",
+    "tokens": 128000,
+    "vision": true,
+  },
+]
+`;
+
+exports[`GET /api/config > Model Provider env > OPENROUTER_MODEL_LIST > custom deletion, addition, and renaming of models 1`] = `
+[
+  {
+    "displayName": "google/gemma-7b-it",
+    "enabled": true,
+    "functionCall": true,
+    "id": "google/gemma-7b-it",
+    "vision": true,
+  },
+  {
+    "displayName": "Mistral-7B-Instruct",
+    "enabled": true,
+    "functionCall": true,
+    "id": "mistralai/mistral-7b-instruct",
+    "vision": true,
+  },
+]
+`;

package/src/app/api/config/route.test.ts

@@ -0,0 +1,170 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { GlobalServerConfig } from '@/types/serverConfig';
+
+import { GET } from './route';
+
+beforeEach(() => {
+  vi.resetAllMocks();
+});
+
+describe('GET /api/config', () => {
+  describe('Model Provider env', () => {
+    describe('OPENAI_MODEL_LIST', () => {
+      it('custom deletion, addition, and renaming of models', async () => {
+        process.env.OPENAI_MODEL_LIST =
+          '-all,+llama,+claude-2，-gpt-3.5-turbo,gpt-4-0125-preview=gpt-4-turbo,gpt-4-0125-preview=gpt-4-32k';
+
+        const response = await GET();
+
+        // Assert
+        expect(response).toBeInstanceOf(Response);
+        expect(response.status).toBe(200);
+
+        const jsonResponse: GlobalServerConfig = await response.json();
+
+        const result = jsonResponse.languageModel?.openai?.serverModelCards;
+
+        expect(result).toMatchSnapshot();
+        process.env.OPENAI_MODEL_LIST = '';
+      });
+
+      it('should work correct with gpt-4', async () => {
+        process.env.OPENAI_MODEL_LIST =
+          '-all,+gpt-3.5-turbo-1106,+gpt-3.5-turbo,+gpt-3.5-turbo-16k,+gpt-4,+gpt-4-32k,+gpt-4-1106-preview,+gpt-4-vision-preview';
+
+        const response = await GET();
+        const jsonResponse: GlobalServerConfig = await response.json();
+
+        const result = jsonResponse.languageModel?.openai?.serverModelCards;
+
+        expect(result).toMatchSnapshot();
+
+        process.env.OPENAI_MODEL_LIST = '';
+      });
+
+      it('duplicate naming model', async () => {
+        process.env.OPENAI_MODEL_LIST =
+          'gpt-4-0125-preview=gpt-4-turbo，gpt-4-0125-preview=gpt-4-32k';
+
+        const response = await GET();
+        const jsonResponse: GlobalServerConfig = await response.json();
+
+        const result = jsonResponse.languageModel?.openai?.serverModelCards;
+
+        expect(result?.find((s) => s.id === 'gpt-4-0125-preview')?.displayName).toEqual(
+          'gpt-4-32k',
+        );
+
+        process.env.OPENAI_MODEL_LIST = '';
+      });
+
+      it('should delete model', async () => {
+        process.env.OPENAI_MODEL_LIST = '-gpt-4';
+
+        const res = await GET();
+        const data: GlobalServerConfig = await res.json();
+
+        const result = data.languageModel?.openai?.serverModelCards;
+
+        expect(result?.find((r) => r.id === 'gpt-4')).toBeUndefined();
+
+        process.env.OPENAI_MODEL_LIST = '';
+      });
+
+      it('show the hidden model', async () => {
+        process.env.OPENAI_MODEL_LIST = '+gpt-4-1106-preview';
+
+        const res = await GET();
+        const data: GlobalServerConfig = await res.json();
+
+        const result = data.languageModel?.openai?.serverModelCards;
+
+        expect(result?.find((o) => o.id === 'gpt-4-1106-preview')).toEqual({
+          displayName: 'GPT-4 Turbo Preview (1106)',
+          functionCall: true,
+          enabled: true,
+          id: 'gpt-4-1106-preview',
+          tokens: 128000,
+        });
+
+        process.env.OPENAI_MODEL_LIST = '';
+      });
+
+      it('only add the model', async () => {
+        process.env.OPENAI_MODEL_LIST = 'model1,model2,model3，model4';
+
+        const res = await GET();
+        const data: GlobalServerConfig = await res.json();
+
+        const result = data.languageModel?.openai?.serverModelCards;
+
+        expect(result).toContainEqual({
+          displayName: 'model1',
+          functionCall: true,
+          id: 'model1',
+          enabled: true,
+          vision: true,
+        });
+        expect(result).toContainEqual({
+          displayName: 'model2',
+          functionCall: true,
+          enabled: true,
+          id: 'model2',
+          vision: true,
+        });
+        expect(result).toContainEqual({
+          displayName: 'model3',
+          enabled: true,
+          functionCall: true,
+          id: 'model3',
+          vision: true,
+        });
+        expect(result).toContainEqual({
+          displayName: 'model4',
+          functionCall: true,
+          enabled: true,
+          id: 'model4',
+          vision: true,
+        });
+
+        process.env.OPENAI_MODEL_LIST = '';
+      });
+    });
+
+    describe('CUSTOM_MODELS', () => {
+      it('custom deletion, addition, and renaming of models', async () => {
+        process.env.CUSTOM_MODELS =
+          '-all,+llama,+claude-2，-gpt-3.5-turbo,gpt-4-0125-preview=gpt-4-turbo,gpt-4-0125-preview=gpt-4-32k';
+
+        const response = await GET();
+
+        // Assert
+        expect(response).toBeInstanceOf(Response);
+        expect(response.status).toBe(200);
+
+        const jsonResponse: GlobalServerConfig = await response.json();
+
+        const result = jsonResponse.languageModel?.openai?.serverModelCards;
+
+        expect(result).toMatchSnapshot();
+      });
+    });
+
+    describe('OPENROUTER_MODEL_LIST', () => {
+      it('custom deletion, addition, and renaming of models', async () => {
+        process.env.OPENROUTER_MODEL_LIST =
+          '-all,+google/gemma-7b-it,+mistralai/mistral-7b-instruct=Mistral-7B-Instruct';
+
+        const res = await GET();
+        const data: GlobalServerConfig = await res.json();
+
+        const result = data.languageModel?.openrouter?.serverModelCards;
+
+        expect(result).toMatchSnapshot();
+
+        process.env.OPENROUTER_MODEL_LIST = '';
+      });
+    });
+  });
+});

package/src/app/api/config/route.ts

@@ -1,5 +1,11 @@
+import {
+  OllamaProviderCard,
+  OpenRouterProviderCard,
+  TogetherAIProviderCard,
+} from '@/config/modelProviders';
 import { getServerConfig } from '@/config/server';
-import { GlobalServerConfig } from '@/types/settings';
+import { GlobalServerConfig } from '@/types/serverConfig';
+import { transformToChatModelCards } from '@/utils/parseModels';
 
 import { parseAgentConfig } from './parseDefaultAgent';
 
@@ -11,27 +17,32 @@ export const runtime = 'edge';
 export const GET = async () => {
   const {
     ENABLE_LANGFUSE,
-    CUSTOM_MODELS,
+    ENABLE_OAUTH_SSO,
+
+    DEFAULT_AGENT_CONFIG,
+    OPENAI_MODEL_LIST,
+
     ENABLED_MOONSHOT,
     ENABLED_ZHIPU,
     ENABLED_AWS_BEDROCK,
     ENABLED_GOOGLE,
     ENABLED_GROQ,
-    ENABLE_OAUTH_SSO,
-    ENABLE_OLLAMA,
     ENABLED_PERPLEXITY,
     ENABLED_ANTHROPIC,
     ENABLED_MISTRAL,
+
+    ENABLE_OLLAMA,
+    OLLAMA_MODEL_LIST,
+
     ENABLED_OPENROUTER,
+    OPENROUTER_MODEL_LIST,
+
     ENABLED_ZEROONE,
     ENABLED_TOGETHERAI,
-    DEFAULT_AGENT_CONFIG,
-    OLLAMA_CUSTOM_MODELS,
-    OPENROUTER_CUSTOM_MODELS,
+    TOGETHERAI_MODEL_LIST,
   } = getServerConfig();
 
   const config: GlobalServerConfig = {
-    customModelName: CUSTOM_MODELS,
     defaultAgent: {
       config: parseAgentConfig(DEFAULT_AGENT_CONFIG),
     },
@@ -44,10 +55,34 @@ export const GET = async () => {
       groq: { enabled: ENABLED_GROQ },
       mistral: { enabled: ENABLED_MISTRAL },
       moonshot: { enabled: ENABLED_MOONSHOT },
-      ollama: { customModelName: OLLAMA_CUSTOM_MODELS, enabled: ENABLE_OLLAMA },
-      openrouter: { customModelName: OPENROUTER_CUSTOM_MODELS, enabled: ENABLED_OPENROUTER },
+
+      ollama: {
+        enabled: ENABLE_OLLAMA,
+        serverModelCards: transformToChatModelCards(
+          OLLAMA_MODEL_LIST,
+          OllamaProviderCard.chatModels,
+        ),
+      },
+      openai: {
+        serverModelCards: transformToChatModelCards(OPENAI_MODEL_LIST),
+      },
+      openrouter: {
+        enabled: ENABLED_OPENROUTER,
+        serverModelCards: transformToChatModelCards(
+          OPENROUTER_MODEL_LIST,
+          OpenRouterProviderCard.chatModels,
+        ),
+      },
       perplexity: { enabled: ENABLED_PERPLEXITY },
-      togetherai: { enabled: ENABLED_TOGETHERAI },
+
+      togetherai: {
+        enabled: ENABLED_TOGETHERAI,
+        serverModelCards: transformToChatModelCards(
+          TOGETHERAI_MODEL_LIST,
+          TogetherAIProviderCard.chatModels,
+        ),
+      },
+
       zeroone: { enabled: ENABLED_ZEROONE },
       zhipu: { enabled: ENABLED_ZHIPU },
     },

package/src/app/api/plugin/gateway/route.ts

@@ -1,7 +1,7 @@
 import { PluginRequestPayload } from '@lobehub/chat-plugin-sdk';
 import { createGatewayOnEdgeRuntime } from '@lobehub/chat-plugins-gateway';
 
-import { getJWTPayload } from '@/app/api/chat/auth';
+import { getJWTPayload } from '@/app/api/chat/auth/utils';
 import { createErrorResponse } from '@/app/api/errorResponse';
 import { getServerConfig } from '@/config/server';
 import { LOBE_CHAT_AUTH_HEADER, OAUTH_AUTHORIZED } from '@/const/auth';