@lobehub/lobehub 2.0.0-next.310 → 2.0.0-next.311

package/packages/model-runtime/src/core/RouterRuntime/createRuntime.ts

@@ -1,11 +1,14 @@
 /**
  * @see https://github.com/lobehub/lobe-chat/discussions/6563
  */
+import type { GoogleGenAIOptions } from '@google/genai';
 import type { ChatModelCard } from '@lobechat/types';
+import debug from 'debug';
 import OpenAI, { ClientOptions } from 'openai';
 import { Stream } from 'openai/streaming';
 
 import { LobeOpenAI } from '../../providers/openai';
+import { LobeVertexAI } from '../../providers/vertexai';
 import {
   CreateImagePayload,
   CreateImageResponse,
@@ -20,19 +23,15 @@ import {
   ChatStreamPayload,
   EmbeddingsOptions,
   EmbeddingsPayload,
-  TextToImagePayload,
   TextToSpeechPayload,
 } from '../../types';
 import { postProcessModelList } from '../../utils/postProcessModelList';
+import { safeParseJSON } from '../../utils/safeParseJSON';
 import { LobeRuntimeAI } from '../BaseAI';
 import { CreateImageOptions, CustomClientOptions } from '../openaiCompatibleFactory';
 import { baseRuntimeMap } from './baseRuntimeMap';
 
-export interface RuntimeItem {
-  id: string;
-  models?: string[];
-  runtime: LobeRuntimeAI;
-}
+const log = debug('lobe-model-runtime:router-runtime');
 
 interface ProviderIniOptions extends Record<string, any> {
   accessKeyId?: string;
@@ -46,12 +45,25 @@ interface ProviderIniOptions extends Record<string, any> {
   sessionToken?: string;
 }
 
+/**
+ * Router option item used for inference.
+ * When `options` is an array, items are tried in order for chat fallback.
+ * `apiType` allows switching provider when falling back.
+ */
+interface RouterOptionItem extends ProviderIniOptions {
+  apiType?: keyof typeof baseRuntimeMap;
+  id?: string;
+  remark?: string;
+}
+
+type RouterOptions = RouterOptionItem | RouterOptionItem[];
+
 export type RuntimeClass = typeof LobeOpenAI;
 
 interface RouterInstance {
   apiType: keyof typeof baseRuntimeMap;
   models?: string[];
-  options: ProviderIniOptions;
+  options: RouterOptions;
   runtime?: RuntimeClass;
 }
 
@@ -121,6 +133,14 @@ export interface CreateRouterRuntimeOptions<T extends Record<string, any> = any>
   routers: Routers;
 }
 
+const formatErrorMessage = (error: unknown): string => {
+  if (error instanceof Error) {
+    return error.name ? `${error.name}: ${error.message}` : error.message;
+  }
+
+  return String(error);
+};
+
 export const createRouterRuntime = ({
   id,
   routers,
@@ -163,61 +183,200 @@ export const createRouterRuntime = ({
       return resolvedRouters;
     }
 
-    /**
-     * Create runtime for inference requests (chat, generateObject, etc.)
-     * Finds the router that matches the model, or uses the last router as fallback
-     */
-    private async createRuntimeForInference(model: string): Promise<RuntimeItem> {
+    private async resolveMatchedRouter(model: string): Promise<RouterInstance> {
       const resolvedRouters = await this.resolveRouters(model);
-
-      const matchedRouter =
+      return (
         resolvedRouters.find((router) => {
           if (router.models && router.models.length > 0) {
             return router.models.includes(model);
           }
           return false;
-        }) ?? resolvedRouters.at(-1)!;
+        }) ?? resolvedRouters.at(-1)!
+      );
+    }
+
+    private normalizeRouterOptions(router: RouterInstance): RouterOptionItem[] {
+      const routerOptions = Array.isArray(router.options) ? router.options : [router.options];
+
+      if (routerOptions.length === 0 || routerOptions.some((optionItem) => !optionItem)) {
+        throw new Error('empty provider options');
+      }
+
+      return routerOptions;
+    }
+
+    /**
+     * Build a runtime instance for a specific option item.
+     * Option items can override apiType to switch providers for fallback.
+     */
+    private createRuntimeFromOption(
+      router: RouterInstance,
+      optionItem: RouterOptionItem,
+    ): {
+      channelId?: string;
+      id: keyof typeof baseRuntimeMap;
+      remark?: string;
+      runtime: LobeRuntimeAI;
+    } {
+      const { apiType: optionApiType, id: channelId, remark, ...optionOverrides } = optionItem;
+      const resolvedApiType = optionApiType ?? router.apiType;
+      const finalOptions = { ...this._params, ...this._options, ...optionOverrides };
+
+      /**
+       * Vertex AI uses GoogleGenAI credentials flow rather than API keys.
+       * Accept JSON credentials in apiKey for compatibility with server config.
+       */
+      if (resolvedApiType === 'vertexai') {
+        const { apiKey, googleAuthOptions, project, location, ...restOptions } = finalOptions;
+        const credentials = safeParseJSON<Record<string, any>>(apiKey);
+        const vertexOptions: GoogleGenAIOptions = {
+          ...(restOptions as GoogleGenAIOptions),
+          vertexai: true,
+        };
+
+        if (googleAuthOptions) {
+          vertexOptions.googleAuthOptions = googleAuthOptions;
+        } else if (credentials) {
+          vertexOptions.googleAuthOptions = { credentials };
+        }
+
+        if (project) vertexOptions.project = project;
+        if (location) vertexOptions.location = location as GoogleGenAIOptions['location'];
+
+        return {
+          channelId,
+          id: resolvedApiType,
+          remark,
+          runtime: LobeVertexAI.initFromVertexAI(vertexOptions),
+        };
+      }
 
       const providerAI =
-        matchedRouter.runtime ?? baseRuntimeMap[matchedRouter.apiType] ?? LobeOpenAI;
-      const finalOptions = { ...this._params, ...this._options, ...matchedRouter.options };
+        resolvedApiType === router.apiType
+          ? (router.runtime ?? baseRuntimeMap[resolvedApiType] ?? LobeOpenAI)
+          : (baseRuntimeMap[resolvedApiType] ?? LobeOpenAI);
       const runtime: LobeRuntimeAI = new providerAI({ ...finalOptions, id: this._id });
 
       return {
-        id: matchedRouter.apiType,
-        models: matchedRouter.models,
+        channelId,
+        id: resolvedApiType,
+        remark,
         runtime,
       };
     }
 
-    /**
-     * Create all runtimes for listing models
-     */
-    private async createRuntimes(): Promise<RuntimeItem[]> {
+    private async runWithFallback<T>(
+      model: string,
+      requestHandler: (runtime: LobeRuntimeAI) => Promise<T>,
+    ): Promise<T> {
+      const matchedRouter = await this.resolveMatchedRouter(model);
+      const routerOptions = this.normalizeRouterOptions(matchedRouter);
+      const totalOptions = routerOptions.length;
+
+      log(
+        'resolve router for model=%s apiType=%s options=%d',
+        model,
+        matchedRouter.apiType,
+        totalOptions,
+      );
+
+      let lastError: unknown;
+
+      for (const [index, optionItem] of routerOptions.entries()) {
+        const attempt = index + 1;
+        const {
+          channelId,
+          id: resolvedApiType,
+          remark,
+          runtime,
+        } = this.createRuntimeFromOption(matchedRouter, optionItem);
+
+        try {
+          const result = await requestHandler(runtime);
+
+          if (totalOptions > 1 && attempt > 1) {
+            log(
+              'fallback success for model=%s attempt=%d/%d apiType=%s channelId=%s remark=%s',
+              model,
+              attempt,
+              totalOptions,
+              resolvedApiType,
+              channelId ?? '',
+              remark ?? '',
+            );
+          }
+
+          return result;
+        } catch (error) {
+          lastError = error;
+
+          const message = formatErrorMessage(error);
+          if (attempt < totalOptions) {
+            log(
+              'attempt failed, fallback to next: model=%s attempt=%d/%d apiType=%s channelId=%s remark=%s error=%s',
+              model,
+              attempt,
+              totalOptions,
+              resolvedApiType,
+              channelId ?? '',
+              remark ?? '',
+              message,
+            );
+          } else {
+            log(
+              'attempt failed, no more fallbacks: model=%s attempt=%d/%d apiType=%s channelId=%s remark=%s error=%s',
+              model,
+              attempt,
+              totalOptions,
+              resolvedApiType,
+              channelId ?? '',
+              remark ?? '',
+              message,
+            );
+          }
+        }
+      }
+
+      throw lastError ?? new Error('empty provider options');
+    }
+
+    async models() {
       const resolvedRouters = await this.resolveRouters();
-      return resolvedRouters.map((router) => {
-        const providerAI = router.runtime ?? baseRuntimeMap[router.apiType] ?? LobeOpenAI;
-        const finalOptions = { ...this._params, ...this._options, ...router.options };
-        const runtime: LobeRuntimeAI = new providerAI({ ...finalOptions, id: this._id });
+      const runtimes = resolvedRouters.map((router) => {
+        const routerOptions = this.normalizeRouterOptions(router);
+        const { id: resolvedApiType, runtime } = this.createRuntimeFromOption(
+          router,
+          routerOptions[0],
+        );
 
         return {
-          id: router.apiType,
+          id: resolvedApiType,
           models: router.models,
           runtime,
         };
       });
-    }
 
-    // Check if it can match a specific model, otherwise default to using the last runtime
-    async getRuntimeByModel(model: string) {
-      const runtimeItem = await this.createRuntimeForInference(model);
-      return runtimeItem.runtime;
+      if (modelsOption && typeof modelsOption === 'function') {
+        // If it's a functional configuration, use the last runtime's client to call the function
+        const lastRuntime = runtimes.at(-1)?.runtime;
+        if (lastRuntime && 'client' in lastRuntime) {
+          const modelList = await modelsOption({ client: (lastRuntime as any).client });
+          return await postProcessModelList(modelList);
+        }
+      }
+
+      return runtimes.at(-1)?.runtime.models?.();
     }
 
+    /**
+     * Try router options in order for chat requests.
+     * When options is an array, fall back to the next item on failure.
+     */
     async chat(payload: ChatStreamPayload, options?: ChatMethodOptions) {
       try {
-        const runtime = await this.getRuntimeByModel(payload.model);
-        return await runtime.chat!(payload, options);
+        return await this.runWithFallback(payload.model, (runtime) =>
+          runtime.chat!(payload, options),
+        );
       } catch (e) {
         if (params.chatCompletion?.handleError) {
           const error = params.chatCompletion.handleError(e, this._options);
@@ -231,44 +390,26 @@ export const createRouterRuntime = ({
       }
     }
 
-    async generateObject(payload: GenerateObjectPayload, options?: GenerateObjectOptions) {
-      const runtime = await this.getRuntimeByModel(payload.model);
-      return runtime.generateObject!(payload, options);
-    }
-
     async createImage(payload: CreateImagePayload) {
-      const runtime = await this.getRuntimeByModel(payload.model);
-      return runtime.createImage!(payload);
+      return this.runWithFallback(payload.model, (runtime) => runtime.createImage!(payload));
     }
 
-    async textToImage(payload: TextToImagePayload) {
-      const runtime = await this.getRuntimeByModel(payload.model);
-      return runtime.textToImage!(payload);
-    }
-
-    async models() {
-      if (modelsOption && typeof modelsOption === 'function') {
-        const runtimes = await this.createRuntimes();
-        // If it's a functional configuration, use the last runtime's client to call the function
-        const lastRuntime = runtimes.at(-1)?.runtime;
-        if (lastRuntime && 'client' in lastRuntime) {
-          const modelList = await modelsOption({ client: (lastRuntime as any).client });
-          return await postProcessModelList(modelList);
-        }
-      }
-
-      const runtimes = await this.createRuntimes();
-      return runtimes.at(-1)?.runtime.models?.();
+    async generateObject(payload: GenerateObjectPayload, options?: GenerateObjectOptions) {
+      return this.runWithFallback(payload.model, (runtime) =>
+        runtime.generateObject!(payload, options),
+      );
     }
 
     async embeddings(payload: EmbeddingsPayload, options?: EmbeddingsOptions) {
-      const runtime = await this.getRuntimeByModel(payload.model);
-      return runtime.embeddings!(payload, options);
+      return this.runWithFallback(payload.model, (runtime) =>
+        runtime.embeddings!(payload, options),
+      );
     }
 
     async textToSpeech(payload: TextToSpeechPayload, options?: EmbeddingsOptions) {
-      const runtime = await this.getRuntimeByModel(payload.model);
-      return runtime.textToSpeech!(payload, options);
+      return this.runWithFallback(payload.model, (runtime) =>
+        runtime.textToSpeech!(payload, options),
+      );
     }
   };
 };

package/packages/model-runtime/src/core/openaiCompatibleFactory/index.ts

@@ -19,7 +19,6 @@ import {
   EmbeddingsPayload,
   GenerateObjectOptions,
   GenerateObjectPayload,
-  TextToImagePayload,
   TextToSpeechOptions,
   TextToSpeechPayload,
 } from '../../types';
@@ -736,19 +735,6 @@ export const createOpenAICompatibleRuntime = <T extends Record<string, any> = an
       }
     }
 
-    async textToImage(payload: TextToImagePayload) {
-      const log = debug(`${this.logPrefix}:textToImage`);
-      log('textToImage called with prompt length: %d', payload.prompt?.length || 0);
-
-      try {
-        const res = await this.client.images.generate(payload);
-        log('generated %d images', res.data?.length || 0);
-        return (res.data || []).map((o) => o.url) as string[];
-      } catch (error) {
-        throw this.handleError(error);
-      }
-    }
-
     async textToSpeech(payload: TextToSpeechPayload, options?: TextToSpeechOptions) {
       const log = debug(`${this.logPrefix}:textToSpeech`);
       log(

package/packages/model-runtime/src/providers/aihubmix/index.test.ts

@@ -1,5 +1,4 @@
 // @vitest-environment node
-import { ModelProvider } from 'model-bank';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { LobeAiHubMixAI } from './index';
@@ -25,9 +24,7 @@ describe('LobeAiHubMixAI', () => {
 
   describe('chat', () => {
     it('should support chat method', async () => {
-      vi.spyOn(instance as any, 'getRuntimeByModel').mockResolvedValue({
-        chat: vi.fn().mockResolvedValue(new Response()),
-      });
+      vi.spyOn(instance as any, 'runWithFallback').mockResolvedValue(new Response());
 
       const payload = {
         messages: [{ content: 'Hello', role: 'user' as const }],
@@ -49,24 +46,21 @@ describe('LobeAiHubMixAI', () => {
         },
       };
 
-      vi.spyOn(instance as any, 'getRuntimeByModel').mockResolvedValue({
-        models: async () => {
-          try {
-            await mockClient.models.list();
-          } catch (error) {
-            console.warn(
-              'Failed to fetch AiHubMix models. Please ensure your AiHubMix API key is valid:',
-              error,
-            );
-            return [];
-          }
-        },
-      });
+      class MockRuntime {
+        client = mockClient;
+      }
 
       // The models method should return empty array on error
-      const models = await (instance as any)
-        .getRuntimeByModel('test-model')
-        .then((r: any) => r.models());
+      vi.spyOn(instance as any, 'resolveRouters').mockResolvedValue([
+        {
+          apiType: 'openai',
+          models: [],
+          options: {},
+          runtime: MockRuntime,
+        },
+      ]);
+
+      const models = await instance.models();
       expect(models).toEqual([]);
     });
   });

package/packages/model-runtime/src/types/index.ts

@@ -4,7 +4,6 @@ export * from './error';
 export * from './image';
 export * from './model';
 export * from './structureOutput';
-export * from './textToImage';
 export * from './toolsCalling';
 export * from './tts';
 export * from './type';

package/packages/model-runtime/src/utils/createError.test.ts

@@ -72,24 +72,4 @@ describe('AgentRuntimeError', () => {
       expect(result).toEqual(errorPayload);
     });
   });
-
-  describe('textToImage', () => {
-    it('should return the same error object', () => {
-      const error = { message: 'Text to image failed', code: 'GENERATION_ERROR' };
-      const result = AgentRuntimeError.textToImage(error);
-
-      expect(result).toBe(error);
-      expect(result).toEqual(error);
-    });
-
-    it('should handle any type of error', () => {
-      const stringError = 'String error';
-      const numberError = 404;
-      const arrayError = ['error1', 'error2'];
-
-      expect(AgentRuntimeError.textToImage(stringError)).toBe(stringError);
-      expect(AgentRuntimeError.textToImage(numberError)).toBe(numberError);
-      expect(AgentRuntimeError.textToImage(arrayError)).toBe(arrayError);
-    });
-  });
 });

package/packages/model-runtime/src/utils/createError.ts

@@ -12,5 +12,4 @@ export const AgentRuntimeError = {
     error?: any,
   ): AgentInitErrorPayload => ({ error, errorType }),
   createImage: (error: CreateImageErrorPayload): CreateImageErrorPayload => error,
-  textToImage: (error: any): any => error,
 };

package/src/app/(backend)/market/agent/[[...segments]]/route.ts

@@ -1,7 +1,6 @@
-import { MarketSDK } from '@lobehub/market-sdk';
 import { type NextRequest, NextResponse } from 'next/server';
 
-import { getTrustedClientTokenForSession } from '@/libs/trusted-client';
+import { MarketService } from '@/server/services/market';
 
 type RouteContext = {
   params: Promise<{
@@ -9,18 +8,6 @@ type RouteContext = {
   }>;
 };
 
-const MARKET_BASE_URL = process.env.NEXT_PUBLIC_MARKET_BASE_URL || 'https://market.lobehub.com';
-
-const extractAccessToken = (req: NextRequest) => {
-  const authorization = req.headers.get('authorization');
-  if (!authorization) return undefined;
-
-  const [scheme, token] = authorization.split(' ');
-  if (scheme?.toLowerCase() !== 'bearer' || !token) return undefined;
-
-  return token;
-};
-
 const methodNotAllowed = (methods: string[]) =>
   NextResponse.json(
     {
@@ -55,14 +42,8 @@ const notFound = (reason: string) =>
   );
 
 const handleAgent = async (req: NextRequest, segments: string[]) => {
-  const accessToken = extractAccessToken(req);
-  const trustedClientToken = await getTrustedClientTokenForSession();
-
-  const market = new MarketSDK({
-    accessToken,
-    baseURL: MARKET_BASE_URL,
-    trustedClientToken,
-  });
+  const marketService = await MarketService.createFromRequest(req);
+  const market = marketService.market;
 
   if (segments.length === 0) {
     return notFound('Missing agent action.');
@@ -94,17 +75,6 @@ const handleAgent = async (req: NextRequest, segments: string[]) => {
   if (action === 'own') {
     if (req.method !== 'GET') return methodNotAllowed(['GET']);
 
-    if (!accessToken) {
-      return NextResponse.json(
-        {
-          error: 'unauthorized',
-          message: 'Authentication required to get own agents',
-          status: 'error',
-        },
-        { status: 401 },
-      );
-    }
-
     try {
       // Parse query parameters from the request URL
       const url = new URL(req.url);

package/src/app/(backend)/market/oidc/[[...segments]]/route.ts

@@ -1,15 +1,15 @@
-import { MarketSDK } from '@lobehub/market-sdk';
 import { type NextRequest, NextResponse } from 'next/server';
 
 import { getTrustedClientTokenForSession } from '@/libs/trusted-client';
+import { MarketService } from '@/server/services/market';
+
+const MARKET_BASE_URL = process.env.NEXT_PUBLIC_MARKET_BASE_URL || 'https://market.lobehub.com';
 
 type RouteContext = {
   params: Promise<{
     segments?: string[];
   }>;
 };
-
-const MARKET_BASE_URL = process.env.NEXT_PUBLIC_MARKET_BASE_URL || 'https://market.lobehub.com';
 const ALLOWED_ENDPOINTS = new Set(['handoff', 'token', 'userinfo']);
 
 const ensureEndpoint = (segments?: string[]) => {
@@ -44,9 +44,8 @@ const methodNotAllowed = (allowed: string[]) =>
   );
 
 const handleProxy = async (req: NextRequest, context: RouteContext) => {
-  const market = new MarketSDK({
-    baseURL: MARKET_BASE_URL,
-  });
+  const marketService = new MarketService();
+  const market = marketService.market;
 
   const { segments } = await context.params;
   const endpointResult = ensureEndpoint(segments);