@nocobase/plugin-ai 1.9.33 → 1.9.34

package/dist/node_modules/@langchain/openai/dist/chat_models.js

@@ -4,7 +4,7 @@ import { ChatGenerationChunk, } from "@langchain/core/outputs";
 import { getEnvironmentVariable } from "@langchain/core/utils/env";
 import { BaseChatModel, } from "@langchain/core/language_models/chat_models";
 import { isOpenAITool, } from "@langchain/core/language_models/base";
-import { RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
+import { RunnableLambda, RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
 import { JsonOutputParser, StructuredOutputParser, } from "@langchain/core/output_parsers";
 import { JsonOutputKeyToolsParser, convertLangChainToolCallToOpenAI, makeInvalidToolCall, parseToolCall, } from "@langchain/core/output_parsers/openai_tools";
 import { zodToJsonSchema } from "zod-to-json-schema";
@@ -64,11 +64,11 @@ export function _convertMessagesToOpenAIParams(messages, model) {
         }
         if (message.additional_kwargs.function_call != null) {
             completionParam.function_call = message.additional_kwargs.function_call;
-            completionParam.content = null;
+            completionParam.content = "";
         }
         if (isAIMessage(message) && !!message.tool_calls?.length) {
             completionParam.tool_calls = message.tool_calls.map(convertLangChainToolCallToOpenAI);
-            completionParam.content = null;
+            completionParam.content = "";
         }
         else {
             if (message.additional_kwargs.tool_calls != null) {
@@ -92,6 +92,354 @@ export function _convertMessagesToOpenAIParams(messages, model) {
         return completionParam;
     });
 }
+const _FUNCTION_CALL_IDS_MAP_KEY = "__openai_function_call_ids__";
+function _convertMessagesToOpenAIResponsesParams(messages, model) {
+    return messages.flatMap((lcMsg) => {
+        let role = messageToOpenAIRole(lcMsg);
+        if (role === "system" && isReasoningModel(model))
+            role = "developer";
+        if (role === "function") {
+            throw new Error("Function messages are not supported in Responses API");
+        }
+        if (role === "tool") {
+            const toolMessage = lcMsg;
+            // Handle computer call output
+            if (toolMessage.additional_kwargs?.type === "computer_call_output") {
+                const output = (() => {
+                    if (typeof toolMessage.content === "string") {
+                        return {
+                            type: "computer_screenshot",
+                            image_url: toolMessage.content,
+                        };
+                    }
+                    if (Array.isArray(toolMessage.content)) {
+                        const oaiScreenshot = toolMessage.content.find((i) => i.type === "computer_screenshot");
+                        if (oaiScreenshot)
+                            return oaiScreenshot;
+                        const lcImage = toolMessage.content.find((i) => i.type === "image_url");
+                        if (lcImage) {
+                            return {
+                                type: "computer_screenshot",
+                                image_url: typeof lcImage.image_url === "string"
+                                    ? lcImage.image_url
+                                    : lcImage.image_url.url,
+                            };
+                        }
+                    }
+                    throw new Error("Invalid computer call output");
+                })();
+                return {
+                    type: "computer_call_output",
+                    output,
+                    call_id: toolMessage.tool_call_id,
+                };
+            }
+            return {
+                type: "function_call_output",
+                call_id: toolMessage.tool_call_id,
+                id: toolMessage.id,
+                output: typeof toolMessage.content !== "string"
+                    ? JSON.stringify(toolMessage.content)
+                    : toolMessage.content,
+            };
+        }
+        if (role === "assistant") {
+            const input = [];
+            // reasoning items
+            if (lcMsg.additional_kwargs.reasoning != null) {
+                const isReasoningItem = (item) => typeof item === "object" &&
+                    item != null &&
+                    "type" in item &&
+                    item.type === "reasoning";
+                if (isReasoningItem(lcMsg.additional_kwargs.reasoning)) {
+                    input.push(lcMsg.additional_kwargs.reasoning);
+                }
+            }
+            // ai content
+            let { content } = lcMsg;
+            if (lcMsg.additional_kwargs.refusal != null) {
+                if (typeof content === "string") {
+                    content = [{ type: "output_text", text: content, annotations: [] }];
+                }
+                content = [
+                    ...content,
+                    { type: "refusal", refusal: lcMsg.additional_kwargs.refusal },
+                ];
+            }
+            input.push({
+                type: "message",
+                role: "assistant",
+                content: typeof content === "string"
+                    ? content
+                    : content.flatMap((item) => {
+                        if (item.type === "text") {
+                            return {
+                                type: "output_text",
+                                text: item.text,
+                                // @ts-expect-error TODO: add types for `annotations`
+                                annotations: item.annotations ?? [],
+                            };
+                        }
+                        if (item.type === "output_text" || item.type === "refusal") {
+                            return item;
+                        }
+                        return [];
+                    }),
+            });
+            // function tool calls and computer use tool calls
+            const functionCallIds = 
+            // eslint-disable-next-line @typescript-eslint/no-use-before-define
+            lcMsg.additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY];
+            if (isAIMessage(lcMsg) && !!lcMsg.tool_calls?.length) {
+                input.push(...lcMsg.tool_calls.map((toolCall) => ({
+                    type: "function_call",
+                    name: toolCall.name,
+                    arguments: JSON.stringify(toolCall.args),
+                    call_id: toolCall.id,
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
+                })));
+            }
+            else if (lcMsg.additional_kwargs.tool_calls != null) {
+                input.push(...lcMsg.additional_kwargs.tool_calls.map((toolCall) => ({
+                    type: "function_call",
+                    name: toolCall.function.name,
+                    call_id: toolCall.id,
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
+                    arguments: toolCall.function.arguments,
+                })));
+            }
+            if (lcMsg.additional_kwargs.tool_outputs != null) {
+                const toolOutputs = lcMsg.additional_kwargs
+                    .tool_outputs;
+                const computerCalls = toolOutputs?.filter((item) => item.type === "computer_call");
+                if (computerCalls.length > 0)
+                    input.push(...computerCalls);
+            }
+            return input;
+        }
+        if (role === "user") {
+            return {
+                type: "message",
+                role: "user",
+                content: typeof lcMsg.content === "string"
+                    ? lcMsg.content
+                    : lcMsg.content.flatMap((item) => {
+                        if (item.type === "text") {
+                            return { type: "input_text", text: item.text };
+                        }
+                        if (item.type === "image_url") {
+                            const image_url = typeof item.image_url === "string"
+                                ? item.image_url
+                                : item.image_url.url;
+                            const detail = typeof item.image_url === "string"
+                                ? "auto"
+                                : item.image_url.detail;
+                            return { type: "input_image", image_url, detail };
+                        }
+                        if (item.type === "input_text" ||
+                            item.type === "input_image" ||
+                            item.type === "input_file") {
+                            return item;
+                        }
+                        return [];
+                    }),
+            };
+        }
+        return [];
+    });
+}
+function _convertOpenAIResponsesMessageToBaseMessage(response) {
+    if (response.error) {
+        // TODO: add support for `addLangChainErrorFields`
+        const error = new Error(response.error.message);
+        error.name = response.error.code;
+        throw error;
+    }
+    const content = [];
+    const tool_calls = [];
+    const invalid_tool_calls = [];
+    const response_metadata = {
+        model: response.model,
+        created_at: response.created_at,
+        id: response.id,
+        incomplete_details: response.incomplete_details,
+        metadata: response.metadata,
+        object: response.object,
+        status: response.status,
+        user: response.user,
+        // for compatibility with chat completion calls.
+        model_name: response.model,
+    };
+    const additional_kwargs = {};
+    for (const item of response.output) {
+        if (item.type === "message") {
+            content.push(...item.content.flatMap((part) => {
+                if (part.type === "output_text") {
+                    if ("parsed" in part && part.parsed != null) {
+                        additional_kwargs.parsed = part.parsed;
+                    }
+                    return {
+                        type: "text",
+                        text: part.text,
+                        annotations: part.annotations,
+                    };
+                }
+                if (part.type === "refusal") {
+                    additional_kwargs.refusal = part.refusal;
+                    return [];
+                }
+                return part;
+            }));
+        }
+        else if (item.type === "function_call") {
+            const fnAdapter = {
+                function: { name: item.name, arguments: item.arguments },
+                id: item.call_id,
+            };
+            try {
+                tool_calls.push(parseToolCall(fnAdapter, { returnId: true }));
+            }
+            catch (e) {
+                let errMessage;
+                if (typeof e === "object" &&
+                    e != null &&
+                    "message" in e &&
+                    typeof e.message === "string") {
+                    errMessage = e.message;
+                }
+                invalid_tool_calls.push(makeInvalidToolCall(fnAdapter, errMessage));
+            }
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] ??= {};
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
+        }
+        else if (item.type === "reasoning") {
+            additional_kwargs.reasoning = item;
+        }
+        else {
+            additional_kwargs.tool_outputs ??= [];
+            additional_kwargs.tool_outputs.push(item);
+        }
+    }
+    return new AIMessage({
+        id: response.id,
+        content,
+        tool_calls,
+        invalid_tool_calls,
+        usage_metadata: response.usage,
+        additional_kwargs,
+        response_metadata,
+    });
+}
+function _convertOpenAIResponsesDeltaToBaseMessageChunk(chunk) {
+    const content = [];
+    let generationInfo = {};
+    let usage_metadata;
+    const tool_call_chunks = [];
+    const response_metadata = {};
+    const additional_kwargs = {};
+    let id;
+    if (chunk.type === "response.output_text.delta") {
+        content.push({
+            type: "text",
+            text: chunk.delta,
+            index: chunk.content_index,
+        });
+    }
+    else if (chunk.type === "response.output_text.annotation.added") {
+        content.push({
+            type: "text",
+            text: "",
+            annotations: [chunk.annotation],
+            index: chunk.content_index,
+        });
+    }
+    else if (chunk.type === "response.output_item.added" &&
+        chunk.item.type === "message") {
+        id = chunk.item.id;
+    }
+    else if (chunk.type === "response.output_item.added" &&
+        chunk.item.type === "function_call") {
+        tool_call_chunks.push({
+            type: "tool_call_chunk",
+            name: chunk.item.name,
+            args: chunk.item.arguments,
+            id: chunk.item.id,
+            index: chunk.output_index,
+        });
+        additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] = {
+            [chunk.item.call_id]: chunk.item.id,
+        };
+    }
+    else if (chunk.type === "response.output_item.done" &&
+        (chunk.item.type === "web_search_call" ||
+            chunk.item.type === "file_search_call" ||
+            chunk.item.type === "computer_call")) {
+        additional_kwargs.tool_outputs = [chunk.item];
+    }
+    else if (chunk.type === "response.created") {
+        response_metadata.id = chunk.response.id;
+        response_metadata.model_name = chunk.response.model;
+        response_metadata.model = chunk.response.model;
+    }
+    else if (chunk.type === "response.completed") {
+        const msg = _convertOpenAIResponsesMessageToBaseMessage(chunk.response);
+        usage_metadata = chunk.response.usage;
+        if (chunk.response.text?.format?.type === "json_schema") {
+            additional_kwargs.parsed ??= JSON.parse(msg.text);
+        }
+        for (const [key, value] of Object.entries(chunk.response)) {
+            if (key !== "id")
+                response_metadata[key] = value;
+        }
+    }
+    else if (chunk.type === "response.function_call_arguments.delta") {
+        tool_call_chunks.push({
+            type: "tool_call_chunk",
+            args: chunk.delta,
+            index: chunk.output_index,
+        });
+    }
+    else if (chunk.type === "response.web_search_call.completed" ||
+        chunk.type === "response.file_search_call.completed") {
+        generationInfo = {
+            tool_outputs: {
+                id: chunk.item_id,
+                type: chunk.type.replace("response.", "").replace(".completed", ""),
+                status: "completed",
+            },
+        };
+    }
+    else if (chunk.type === "response.refusal.done") {
+        additional_kwargs.refusal = chunk.refusal;
+    }
+    else {
+        return null;
+    }
+    return new ChatGenerationChunk({
+        // Legacy reasons, `onLLMNewToken` should pulls this out
+        text: content.map((part) => part.text).join(""),
+        message: new AIMessageChunk({
+            id,
+            content,
+            tool_call_chunks,
+            usage_metadata,
+            additional_kwargs,
+            response_metadata,
+        }),
+        generationInfo,
+    });
+}
+function isBuiltInTool(tool) {
+    return "type" in tool && tool.type !== "function";
+}
+function isBuiltInToolChoice(tool_choice) {
+    return (tool_choice != null &&
+        typeof tool_choice === "object" &&
+        "type" in tool_choice &&
+        tool_choice.type !== "function");
+}
 function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
     if (isOpenAITool(tool)) {
         if (fields?.strict !== undefined) {
@@ -899,6 +1247,12 @@ export class ChatOpenAI extends BaseChatModel {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "useResponsesApi", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
         this.openAIApiKey =
             fields?.apiKey ??
                 fields?.openAIApiKey ??
@@ -921,13 +1275,14 @@ export class ChatOpenAI extends BaseChatModel {
         this.n = fields?.n ?? this.n;
         this.logitBias = fields?.logitBias;
         this.stop = fields?.stopSequences ?? fields?.stop;
-        this.stopSequences = this?.stop;
+        this.stopSequences = this.stop;
         this.user = fields?.user;
         this.__includeRawResponse = fields?.__includeRawResponse;
         this.audio = fields?.audio;
         this.modalities = fields?.modalities;
         this.reasoningEffort = fields?.reasoningEffort;
         this.maxTokens = fields?.maxCompletionTokens ?? fields?.maxTokens;
+        this.useResponsesApi = fields?.useResponsesApi ?? this.useResponsesApi;
         if (this.model === "o1") {
             this.disableStreaming = true;
         }
@@ -965,7 +1320,9 @@ export class ChatOpenAI extends BaseChatModel {
             strict = this.supportsStrictToolCalling;
         }
         return this.bind({
-            tools: tools.map((tool) => _convertChatOpenAIToolTypeToOpenAITool(tool, { strict })),
+            tools: tools.map((tool) => isBuiltInTool(tool)
+                ? tool
+                : _convertChatOpenAIToolTypeToOpenAITool(tool, { strict })),
             ...kwargs,
         });
     }
@@ -991,6 +1348,80 @@ export class ChatOpenAI extends BaseChatModel {
         else if (this.supportsStrictToolCalling !== undefined) {
             strict = this.supportsStrictToolCalling;
         }
+        if (this._useResponseApi(options)) {
+            const params = {
+                model: this.model,
+                temperature: this.temperature,
+                top_p: this.topP,
+                user: this.user,
+                // if include_usage is set or streamUsage then stream must be set to true.
+                stream: this.streaming,
+                previous_response_id: options?.previous_response_id,
+                truncation: options?.truncation,
+                include: options?.include,
+                tools: options?.tools?.length
+                    ? options.tools
+                        .map((tool) => {
+                        if (isBuiltInTool(tool)) {
+                            return tool;
+                        }
+                        else if (isOpenAITool(tool)) {
+                            return {
+                                type: "function",
+                                name: tool.function.name,
+                                parameters: tool.function.parameters,
+                                description: tool.function.description,
+                                strict,
+                            };
+                        }
+                        else {
+                            return null;
+                        }
+                    })
+                        .filter((tool) => tool !== null)
+                    : undefined,
+                tool_choice: isBuiltInToolChoice(options?.tool_choice)
+                    ? options?.tool_choice
+                    : (() => {
+                        const formatted = formatToOpenAIToolChoice(options?.tool_choice);
+                        if (typeof formatted === "object" && "type" in formatted) {
+                            return { type: "function", name: formatted.function.name };
+                        }
+                        else {
+                            return undefined;
+                        }
+                    })(),
+                text: (() => {
+                    if (options?.text)
+                        return options.text;
+                    const format = this.createResponseFormat(options?.response_format);
+                    if (format?.type === "json_schema") {
+                        if (format.json_schema.schema != null) {
+                            return {
+                                format: {
+                                    type: "json_schema",
+                                    schema: format.json_schema.schema,
+                                    description: format.json_schema.description,
+                                    name: format.json_schema.name,
+                                    strict: format.json_schema.strict,
+                                },
+                            };
+                        }
+                        return undefined;
+                    }
+                    return { format };
+                })(),
+                parallel_tool_calls: options?.parallel_tool_calls,
+                max_output_tokens: this.maxTokens === -1 ? undefined : this.maxTokens,
+                ...this.modelKwargs,
+            };
+            const reasoningEffort = options?.reasoning_effort ?? this.reasoningEffort;
+            if (reasoningEffort !== undefined) {
+                params.reasoning = { effort: reasoningEffort };
+            }
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            return params;
+        }
         let streamOptionsConfig = {};
         if (options?.stream_options !== undefined) {
             streamOptionsConfig = { stream_options: options.stream_options };
@@ -1044,6 +1475,7 @@ export class ChatOpenAI extends BaseChatModel {
         else {
             params.max_tokens = this.maxTokens === -1 ? undefined : this.maxTokens;
         }
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         return params;
     }
     _convertOpenAIChatCompletionMessageToBaseMessage(message, rawResponse) {
@@ -1187,6 +1619,20 @@ export class ChatOpenAI extends BaseChatModel {
         };
     }
     async *_streamResponseChunks(messages, options, runManager) {
+        if (this._useResponseApi(options)) {
+            const streamIterable = await this.responseApiWithRetry({
+                ...this.invocationParams(options, { streaming: true }),
+                input: _convertMessagesToOpenAIResponsesParams(messages, this.model),
+                stream: true,
+            }, options);
+            for await (const data of streamIterable) {
+                const chunk = _convertOpenAIResponsesDeltaToBaseMessageChunk(data);
+                if (chunk == null)
+                    continue;
+                yield chunk;
+            }
+            return;
+        }
         const messagesMapped = _convertMessagesToOpenAIParams(messages, this.model);
         const params = {
             ...this.invocationParams(options, {
@@ -1291,7 +1737,60 @@ export class ChatOpenAI extends BaseChatModel {
         return this._identifyingParams();
     }
     /** @ignore */
+    async _responseApiGenerate(messages, options, runManager) {
+        const invocationParams = this.invocationParams(options);
+        if (invocationParams.stream) {
+            const stream = this._streamResponseChunks(messages, options, runManager);
+            let finalChunk;
+            for await (const chunk of stream) {
+                chunk.message.response_metadata = {
+                    ...chunk.generationInfo,
+                    ...chunk.message.response_metadata,
+                };
+                finalChunk = finalChunk?.concat(chunk) ?? chunk;
+            }
+            return {
+                generations: finalChunk ? [finalChunk] : [],
+                llmOutput: {
+                    estimatedTokenUsage: finalChunk?.message
+                        ?.usage_metadata,
+                },
+            };
+        }
+        const input = _convertMessagesToOpenAIResponsesParams(messages, this.model);
+        const data = await this.responseApiWithRetry({ input, ...invocationParams }, { signal: options?.signal, ...options?.options });
+        return {
+            generations: [
+                {
+                    text: data.output_text,
+                    message: _convertOpenAIResponsesMessageToBaseMessage(data),
+                },
+            ],
+            llmOutput: {
+                id: data.id,
+                estimatedTokenUsage: data.usage
+                    ? {
+                        promptTokens: data.usage.input_tokens,
+                        completionTokens: data.usage.output_tokens,
+                        totalTokens: data.usage.total_tokens,
+                    }
+                    : undefined,
+            },
+        };
+    }
+    _useResponseApi(options) {
+        const usesBuiltInTools = options?.tools?.some(isBuiltInTool);
+        const hasResponsesOnlyKwargs = options?.previous_response_id != null ||
+            options?.text != null ||
+            options?.truncation != null ||
+            options?.include != null;
+        return this.useResponsesApi || usesBuiltInTools || hasResponsesOnlyKwargs;
+    }
+    /** @ignore */
     async _generate(messages, options, runManager) {
+        if (this._useResponseApi(options)) {
+            return this._responseApiGenerate(messages, options, runManager);
+        }
         const usageMetadata = {};
         const params = this.invocationParams(options);
         const messagesMapped = _convertMessagesToOpenAIParams(messages, this.model);
@@ -1529,6 +2028,22 @@ export class ChatOpenAI extends BaseChatModel {
             }
         });
     }
+    async responseApiWithRetry(request, options) {
+        return this.caller.call(async () => {
+            const requestOptions = this._getClientOptions(options);
+            try {
+                // use parse if dealing with json_schema
+                if (request.text?.format?.type === "json_schema" && !request.stream) {
+                    return await this.client.responses.parse(request, requestOptions);
+                }
+                return await this.client.responses.create(request, requestOptions);
+            }
+            catch (e) {
+                const error = wrapOpenAIClientError(e);
+                throw error;
+            }
+        });
+    }
     /**
      * Call the beta chat completions parse endpoint. This should only be called if
      * response_format is set to "json_object".
@@ -1651,7 +2166,13 @@ export class ChatOpenAI extends BaseChatModel {
                 },
             });
             if (isZodSchema(schema)) {
-                outputParser = StructuredOutputParser.fromZodSchema(schema);
+                const altParser = StructuredOutputParser.fromZodSchema(schema);
+                outputParser = RunnableLambda.from((aiMessage) => {
+                    if ("parsed" in aiMessage.additional_kwargs) {
+                        return aiMessage.additional_kwargs.parsed;
+                    }
+                    return altParser;
+                });
             }
             else {
                 outputParser = new JsonOutputParser();
@@ -1739,12 +2260,7 @@ export class ChatOpenAI extends BaseChatModel {
         const parsedWithFallback = parserAssign.withFallbacks({
             fallbacks: [parserNone],
         });
-        return RunnableSequence.from([
-            {
-                raw: llm,
-            },
-            parsedWithFallback,
-        ]);
+        return RunnableSequence.from([{ raw: llm }, parsedWithFallback]);
     }
 }
 function isZodSchema(

package/dist/node_modules/@langchain/openai/dist/llms.cjs

@@ -235,7 +235,7 @@ class OpenAI extends llms_1.BaseLLM {
         this.bestOf = fields?.bestOf ?? this.bestOf;
         this.logitBias = fields?.logitBias;
         this.stop = fields?.stopSequences ?? fields?.stop;
-        this.stopSequences = fields?.stopSequences;
+        this.stopSequences = this.stop;
         this.user = fields?.user;
         this.streaming = fields?.streaming ?? false;
         if (this.streaming && this.bestOf && this.bestOf > 1) {

package/dist/node_modules/@langchain/openai/dist/llms.js

@@ -232,7 +232,7 @@ export class OpenAI extends BaseLLM {
         this.bestOf = fields?.bestOf ?? this.bestOf;
         this.logitBias = fields?.logitBias;
         this.stop = fields?.stopSequences ?? fields?.stop;
-        this.stopSequences = fields?.stopSequences;
+        this.stopSequences = this.stop;
         this.user = fields?.user;
         this.streaming = fields?.streaming ?? false;
         if (this.streaming && this.bestOf && this.bestOf > 1) {

package/dist/node_modules/@langchain/openai/dist/types.d.ts

@@ -146,7 +146,7 @@ export interface OpenAIChatInput extends OpenAIBaseInput {
      * Constrains effort on reasoning for reasoning models. Currently supported values are low, medium, and high.
      * Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response.
      */
-    reasoningEffort?: OpenAIClient.ChatCompletionReasoningEffort;
+    reasoningEffort?: OpenAIClient.Chat.ChatCompletionReasoningEffort;
 }
 export declare interface AzureOpenAIInput {
     /**