@lobehub/chat 1.92.3 → 1.93.0

package/src/database/migrations/meta/_journal.json

@@ -175,6 +175,13 @@
       "when": 1749301573666,
       "tag": "0024_add_rbac_tables",
       "breakpoints": true
+    },
+    {
+      "idx": 25,
+      "version": "7",
+      "when": 1749309388370,
+      "tag": "0025_add_provider_config",
+      "breakpoints": true
     }
   ],
   "version": "6"

package/src/database/models/__tests__/aiProvider.test.ts

@@ -361,12 +361,14 @@ describe('AiProviderModel', () => {
       const config = await aiProviderModel.getAiProviderRuntimeConfig(mockDecryptor);
 
       expect(config.provider1).toEqual({
+        config: {},
         fetchOnClient: true,
         keyVaults: { decryptedKey: 'value' },
         settings: { setting1: true },
       });
 
       expect(config.provider2).toEqual({
+        config: {},
         fetchOnClient: undefined,
         keyVaults: {},
         settings: {},

package/src/database/models/aiProvider.ts

@@ -119,6 +119,7 @@ export class AiProviderModel {
 
     const commonFields = {
       checkModel: value.checkModel,
+      config: value.config,
       fetchOnClient: value.fetchOnClient,
       keyVaults,
     };
@@ -129,11 +130,10 @@ export class AiProviderModel {
         ...commonFields,
         id,
         source: this.getProviderSource(id),
-        updatedAt: new Date(),
         userId: this.userId,
       })
       .onConflictDoUpdate({
-        set: { ...commonFields, updatedAt: new Date() },
+        set: commonFields,
         target: [aiProviders.id, aiProviders.userId],
       });
   };
@@ -184,6 +184,7 @@ export class AiProviderModel {
     const query = this.db
       .select({
         checkModel: aiProviders.checkModel,
+        config: aiProviders.config,
         description: aiProviders.description,
         enabled: aiProviders.enabled,
         fetchOnClient: aiProviders.fetchOnClient,
@@ -236,6 +237,7 @@ export class AiProviderModel {
   getAiProviderRuntimeConfig = async (decryptor?: DecryptUserKeyVaults) => {
     const result = await this.db
       .select({
+        config: aiProviders.config,
         fetchOnClient: aiProviders.fetchOnClient,
         id: aiProviders.id,
         keyVaults: aiProviders.keyVaults,
@@ -262,6 +264,7 @@ export class AiProviderModel {
       }
 
       runtimeConfig[item.id] = {
+        config: item.config || {},
         fetchOnClient: typeof item.fetchOnClient === 'boolean' ? item.fetchOnClient : undefined,
         keyVaults,
         settings: !!builtin ? merge(builtin.settings, userSettings) : userSettings,

package/src/database/repositories/tableViewer/index.test.ts

@@ -23,7 +23,7 @@ describe('TableViewerRepo', () => {
     it('should return all tables with counts', async () => {
       const result = await repo.getAllTables();
 
-      expect(result.length).toEqual(51);
+      expect(result.length).toEqual(55);
       expect(result[0]).toEqual({ name: 'agents', count: 0, type: 'BASE TABLE' });
     });
 

package/src/database/schemas/_helpers.ts

@@ -3,7 +3,11 @@ import { timestamp } from 'drizzle-orm/pg-core';
 export const timestamptz = (name: string) => timestamp(name, { withTimezone: true });
 
 export const createdAt = () => timestamptz('created_at').notNull().defaultNow();
-export const updatedAt = () => timestamptz('updated_at').notNull().defaultNow();
+export const updatedAt = () =>
+  timestamptz('updated_at')
+    .notNull()
+    .defaultNow()
+    .$onUpdate(() => new Date());
 export const accessedAt = () => timestamptz('accessed_at').notNull().defaultNow();
 
 // columns.helpers.ts

package/src/database/schemas/aiInfra.ts

@@ -3,7 +3,7 @@ import { boolean, integer, jsonb, pgTable, primaryKey, text, varchar } from 'dri
 
 import { timestamps } from '@/database/schemas/_helpers';
 import { users } from '@/database/schemas/user';
-import { AiProviderSettings } from '@/types/aiProvider';
+import { AiProviderConfig, AiProviderSettings } from '@/types/aiProvider';
 
 export const aiProviders = pgTable(
   'ai_providers',
@@ -29,6 +29,10 @@ export const aiProviders = pgTable(
       .$defaultFn(() => ({}))
       .$type<AiProviderSettings>(),
 
+    config: jsonb('config')
+      .$defaultFn(() => ({}))
+      .$type<AiProviderConfig>(),
+
     ...timestamps,
   },
   (table) => [primaryKey({ columns: [table.id, table.userId] })],

package/src/libs/model-runtime/openai/index.ts

@@ -1,7 +1,7 @@
-import { ModelProvider } from '../types';
+import { ChatStreamPayload, ModelProvider } from '../types';
+import { processMultiProviderModelList } from '../utils/modelParse';
 import { createOpenAICompatibleRuntime } from '../utils/openaiCompatibleFactory';
 import { pruneReasoningPayload } from '../utils/openaiHelpers';
-import { processMultiProviderModelList } from '../utils/modelParse';
 
 export interface OpenAIModelCard {
   id: string;
@@ -15,6 +15,10 @@ export const LobeOpenAI = createOpenAICompatibleRuntime({
     handlePayload: (payload) => {
       const { model } = payload;
 
+      if (model === 'o1-pro') {
+        return { ...payload, apiMode: 'responses' } as ChatStreamPayload;
+      }
+
       if (prunePrefixes.some((prefix) => model.startsWith(prefix))) {
         return pruneReasoningPayload(payload) as any;
       }
@@ -42,6 +46,7 @@ export const LobeOpenAI = createOpenAICompatibleRuntime({
   },
   debug: {
     chatCompletion: () => process.env.DEBUG_OPENAI_CHAT_COMPLETION === '1',
+    responses: () => process.env.DEBUG_OPENAI_RESPONSES === '1',
   },
   models: async ({ client }) => {
     const modelsPage = (await client.models.list()) as any;
@@ -51,4 +56,18 @@ export const LobeOpenAI = createOpenAICompatibleRuntime({
     return processMultiProviderModelList(modelList);
   },
   provider: ModelProvider.OpenAI,
+  responses: {
+    handlePayload: (payload: ChatStreamPayload) => {
+      const { model } = payload;
+      if (prunePrefixes.some((prefix) => model.startsWith(prefix))) {
+        if (!payload.reasoning) {
+          payload.reasoning = { summary: 'auto' };
+        } else {
+          payload.reasoning.summary = 'auto';
+        }
+      }
+
+      return { ...payload, stream: payload.stream ?? true };
+    },
+  },
 });

package/src/libs/model-runtime/types/chat.ts

@@ -67,6 +67,7 @@ export interface OpenAIChatMessage {
  * @title Chat Stream Payload
  */
 export interface ChatStreamPayload {
+  apiMode?: 'chatCompletion' | 'responses';
   /**
    * 开启上下文缓存
    */
@@ -96,21 +97,17 @@ export interface ChatStreamPayload {
    * @title 返回的文本数量
    */
   n?: number;
-  /**
-   * 开启的插件列表
-   */
-  plugins?: string[];
   /**
    * @title 控制生成文本中的惩罚系数，用于减少主题的变化
    * @default 0
    */
   presence_penalty?: number;
-
-  /**
-   * @default openai
-   */
   provider?: string;
-  responseMode?: 'streamText' | 'json';
+  reasoning?: {
+    effort?: string;
+    summary?: string;
+  };
+  responseMode?: 'stream' | 'json';
   /**
    * @title 是否开启流式请求
    * @default true

package/src/libs/model-runtime/utils/openaiCompatibleFactory/index.ts

@@ -8,9 +8,11 @@ import type { ChatModelCard } from '@/types/llm';
 
 import { LobeRuntimeAI } from '../../BaseAI';
 import { AgentRuntimeErrorType, ILobeAgentRuntimeErrorType } from '../../error';
-import type {
+import {
   ChatCompletionErrorPayload,
+  ChatCompletionTool,
   ChatMethodOptions,
+  ChatStreamCallbacks,
   ChatStreamPayload,
   Embeddings,
   EmbeddingsOptions,
@@ -20,14 +22,13 @@ import type {
   TextToSpeechOptions,
   TextToSpeechPayload,
 } from '../../types';
-import { ChatStreamCallbacks } from '../../types';
 import { AgentRuntimeError } from '../createError';
 import { debugResponse, debugStream } from '../debugStream';
 import { desensitizeUrl } from '../desensitizeUrl';
 import { handleOpenAIError } from '../handleOpenAIError';
-import { convertOpenAIMessages } from '../openaiHelpers';
+import { convertOpenAIMessages, convertOpenAIResponseInputs } from '../openaiHelpers';
 import { StreamingResponse } from '../response';
-import { OpenAIStream, OpenAIStreamOptions } from '../streams';
+import { OpenAIResponsesStream, OpenAIStream, OpenAIStreamOptions } from '../streams';
 
 // the model contains the following keywords is not a chat model, so we should filter them out
 export const CHAT_MODELS_BLOCK_LIST = [
@@ -83,6 +84,7 @@ interface OpenAICompatibleFactoryOptions<T extends Record<string, any> = any> {
   customClient?: CustomClientOptions<T>;
   debug?: {
     chatCompletion: () => boolean;
+    responses?: () => boolean;
   };
   errorType?: {
     bizError: ILobeAgentRuntimeErrorType;
@@ -94,6 +96,12 @@ interface OpenAICompatibleFactoryOptions<T extends Record<string, any> = any> {
         transformModel?: (model: OpenAI.Model) => ChatModelCard;
       };
   provider: string;
+  responses?: {
+    handlePayload?: (
+      payload: ChatStreamPayload,
+      options: ConstructorOptions<T>,
+    ) => ChatStreamPayload;
+  };
 }
 
 /**
@@ -160,6 +168,7 @@ export const createOpenAICompatibleRuntime = <T extends Record<string, any> = an
   chatCompletion,
   models,
   customClient,
+  responses,
 }: OpenAICompatibleFactoryOptions<T>) => {
   const ErrorType = {
     bizError: errorType?.bizError || AgentRuntimeErrorType.ProviderBizError,
@@ -199,7 +208,15 @@ export const createOpenAICompatibleRuntime = <T extends Record<string, any> = an
       this.id = options.id || provider;
     }
 
-    async chat({ responseMode, ...payload }: ChatStreamPayload, options?: ChatMethodOptions) {
+    async chat(
+      { responseMode, apiMode, ...payload }: ChatStreamPayload,
+      options?: ChatMethodOptions,
+    ) {
+      // new openai Response API
+      if (apiMode === 'responses') {
+        return this.handleResponseAPIMode(payload, options);
+      }
+
       try {
         const inputStartAt = Date.now();
         const postPayload = chatCompletion?.handlePayload
@@ -454,5 +471,62 @@ export const createOpenAICompatibleRuntime = <T extends Record<string, any> = an
         provider: this.id as ModelProvider,
       });
     }
+
+    private async handleResponseAPIMode(
+      payload: ChatStreamPayload,
+      options?: ChatMethodOptions,
+    ): Promise<Response> {
+      const inputStartAt = Date.now();
+
+      const { messages, ...res } = responses?.handlePayload
+        ? (responses?.handlePayload(payload, this._options) as ChatStreamPayload)
+        : payload;
+
+      // remove penalty params
+      delete res.frequency_penalty;
+      delete res.presence_penalty;
+
+      const input = await convertOpenAIResponseInputs(messages as any);
+
+      const postPayload = {
+        ...res,
+        input,
+        store: false,
+        tools: payload.tools?.map((tool) => this.convertChatCompletionToolToResponseTool(tool)),
+      } as OpenAI.Responses.ResponseCreateParamsStreaming;
+
+      if (debug?.responses?.()) {
+        console.log('[requestPayload]');
+        console.log(JSON.stringify(postPayload), '\n');
+      }
+
+      const response = await this.client.responses.create(postPayload, {
+        headers: options?.requestHeaders,
+        signal: options?.signal,
+      });
+
+      const [prod, useForDebug] = response.tee();
+
+      if (debug?.responses?.()) {
+        const useForDebugStream =
+          useForDebug instanceof ReadableStream ? useForDebug : useForDebug.toReadableStream();
+
+        debugStream(useForDebugStream).catch(console.error);
+      }
+
+      const streamOptions: OpenAIStreamOptions = {
+        bizErrorTypeTransformer: chatCompletion?.handleStreamBizErrorType,
+        callbacks: options?.callback,
+        provider: this.id,
+      };
+
+      return StreamingResponse(OpenAIResponsesStream(prod, { ...streamOptions, inputStartAt }), {
+        headers: options?.headers,
+      });
+    }
+
+    private convertChatCompletionToolToResponseTool = (tool: ChatCompletionTool) => {
+      return { type: tool.type, ...tool.function };
+    };
   };
 };

package/src/libs/model-runtime/utils/openaiHelpers.test.ts

@@ -3,7 +3,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { imageUrlToBase64 } from '@/utils/imageToBase64';
 
-import { convertMessageContent, convertOpenAIMessages } from './openaiHelpers';
+import {
+  convertMessageContent,
+  convertOpenAIMessages,
+  convertOpenAIResponseInputs,
+} from './openaiHelpers';
 import { parseDataUri } from './uriParser';
 
 // 模拟依赖
@@ -144,3 +148,143 @@ describe('convertOpenAIMessages', () => {
     expect(Promise.all).toHaveBeenCalledTimes(2); // 一次用于消息数组，一次用于内容数组
   });
 });
+
+describe('convertOpenAIResponseInputs', () => {
+  it('应该正确转换普通文本消息', async () => {
+    const messages: OpenAI.ChatCompletionMessageParam[] = [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there!' },
+    ];
+
+    const result = await convertOpenAIResponseInputs(messages);
+
+    expect(result).toEqual([
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there!' },
+    ]);
+  });
+
+  it('应该正确转换带有工具调用的消息', async () => {
+    const messages: OpenAI.ChatCompletionMessageParam[] = [
+      {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'call_123',
+            type: 'function',
+            function: {
+              name: 'test_function',
+              arguments: '{"key": "value"}',
+            },
+          },
+        ],
+      },
+    ];
+
+    const result = await convertOpenAIResponseInputs(messages);
+
+    expect(result).toEqual([
+      {
+        arguments: 'test_function',
+        call_id: 'call_123',
+        name: 'test_function',
+        type: 'function_call',
+      },
+    ]);
+  });
+
+  it('应该正确转换工具响应消息', async () => {
+    const messages: OpenAI.ChatCompletionMessageParam[] = [
+      {
+        role: 'tool',
+        content: 'Function result',
+        tool_call_id: 'call_123',
+      },
+    ];
+
+    const result = await convertOpenAIResponseInputs(messages);
+
+    expect(result).toEqual([
+      {
+        call_id: 'call_123',
+        output: 'Function result',
+        type: 'function_call_output',
+      },
+    ]);
+  });
+
+  it('应该正确转换包含图片的消息', async () => {
+    const messages: OpenAI.ChatCompletionMessageParam[] = [
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Here is an image' },
+          {
+            type: 'image_url',
+            image_url: {
+              url: 'data:image/jpeg;base64,test123',
+            },
+          },
+        ],
+      },
+    ];
+
+    const result = await convertOpenAIResponseInputs(messages);
+
+    expect(result).toEqual([
+      {
+        role: 'user',
+        content: [
+          { type: 'input_text', text: 'Here is an image' },
+          {
+            type: 'input_image',
+            image_url: 'data:image/jpeg;base64,test123',
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('应该正确处理混合类型的消息序列', async () => {
+    const messages: OpenAI.ChatCompletionMessageParam[] = [
+      { role: 'user', content: 'I need help with a function' },
+      {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'call_456',
+            type: 'function',
+            function: {
+              name: 'get_data',
+              arguments: '{}',
+            },
+          },
+        ],
+      },
+      {
+        role: 'tool',
+        content: '{"result": "success"}',
+        tool_call_id: 'call_456',
+      },
+    ];
+
+    const result = await convertOpenAIResponseInputs(messages);
+
+    expect(result).toEqual([
+      { role: 'user', content: 'I need help with a function' },
+      {
+        arguments: 'get_data',
+        call_id: 'call_456',
+        name: 'get_data',
+        type: 'function_call',
+      },
+      {
+        call_id: 'call_456',
+        output: '{"result": "success"}',
+        type: 'function_call_output',
+      },
+    ]);
+  });
+});

package/src/libs/model-runtime/utils/openaiHelpers.ts

@@ -41,6 +41,65 @@ export const convertOpenAIMessages = async (messages: OpenAI.ChatCompletionMessa
   )) as OpenAI.ChatCompletionMessageParam[];
 };
 
+export const convertOpenAIResponseInputs = async (
+  messages: OpenAI.ChatCompletionMessageParam[],
+) => {
+  let input: OpenAI.Responses.ResponseInputItem[] = [];
+  await Promise.all(
+    messages.map(async (message) => {
+      // if message is assistant messages with tool calls , transform it to function type item
+      if (message.role === 'assistant' && message.tool_calls && message.tool_calls?.length > 0) {
+        message.tool_calls?.forEach((tool) => {
+          input.push({
+            arguments: tool.function.name,
+            call_id: tool.id,
+            name: tool.function.name,
+            type: 'function_call',
+          });
+        });
+
+        return;
+      }
+
+      if (message.role === 'tool') {
+        input.push({
+          call_id: message.tool_call_id,
+          output: message.content,
+          type: 'function_call_output',
+        } as OpenAI.Responses.ResponseFunctionToolCallOutputItem);
+
+        return;
+      }
+
+      // default item
+      // also need handle image
+      const item = {
+        ...message,
+        content:
+          typeof message.content === 'string'
+            ? message.content
+            : await Promise.all(
+                (message.content || []).map(async (c) => {
+                  if (c.type === 'text') {
+                    return { ...c, type: 'input_text' };
+                  }
+
+                  const image = await convertMessageContent(c as OpenAI.ChatCompletionContentPart);
+                  return {
+                    image_url: (image as OpenAI.ChatCompletionContentPartImage).image_url?.url,
+                    type: 'input_image',
+                  };
+                }),
+              ),
+      } as OpenAI.Responses.ResponseInputItem;
+
+      input.push(item);
+    }),
+  );
+
+  return input;
+};
+
 export const pruneReasoningPayload = (payload: ChatStreamPayload) => {
   return {
     ...payload,