@librechat/agents 2.4.322 → 3.0.0-rc2

package/dist/esm/llm/openai/utils/index.mjs

@@ -0,0 +1,666 @@
+import { isAIMessage, isDataContentBlock, convertToProviderContentBlock, AIMessageChunk, ChatMessage, parseBase64DataUrl, parseMimeType, AIMessage } from '@langchain/core/messages';
+import { ChatGenerationChunk } from '@langchain/core/outputs';
+import { convertLangChainToolCallToOpenAI, parseToolCall, makeInvalidToolCall } from '@langchain/core/output_parsers/openai_tools';
+
+function extractGenericMessageCustomRole(message) {
+    if (message.role !== 'system' &&
+        message.role !== 'developer' &&
+        message.role !== 'assistant' &&
+        message.role !== 'user' &&
+        message.role !== 'function' &&
+        message.role !== 'tool') {
+        console.warn(`Unknown message role: ${message.role}`);
+    }
+    return message.role;
+}
+function messageToOpenAIRole(message) {
+    const type = message._getType();
+    switch (type) {
+        case 'system':
+            return 'system';
+        case 'ai':
+            return 'assistant';
+        case 'human':
+            return 'user';
+        case 'function':
+            return 'function';
+        case 'tool':
+            return 'tool';
+        case 'generic': {
+            if (!ChatMessage.isInstance(message))
+                throw new Error('Invalid generic chat message');
+            return extractGenericMessageCustomRole(message);
+        }
+        default:
+            throw new Error(`Unknown message type: ${type}`);
+    }
+}
+const completionsApiContentBlockConverter = {
+    providerName: 'ChatOpenAI',
+    fromStandardTextBlock(block) {
+        return { type: 'text', text: block.text };
+    },
+    fromStandardImageBlock(block) {
+        if (block.source_type === 'url') {
+            return {
+                type: 'image_url',
+                image_url: {
+                    url: block.url,
+                    ...(block.metadata?.detail
+                        ? { detail: block.metadata.detail }
+                        : {}),
+                },
+            };
+        }
+        if (block.source_type === 'base64') {
+            const url = `data:${block.mime_type ?? ''};base64,${block.data}`;
+            return {
+                type: 'image_url',
+                image_url: {
+                    url,
+                    ...(block.metadata?.detail
+                        ? { detail: block.metadata.detail }
+                        : {}),
+                },
+            };
+        }
+        throw new Error(`Image content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
+    },
+    fromStandardAudioBlock(block) {
+        if (block.source_type === 'url') {
+            const data = parseBase64DataUrl({ dataUrl: block.url });
+            if (!data) {
+                throw new Error(`URL audio blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
+            }
+            const rawMimeType = data.mime_type || block.mime_type || '';
+            let mimeType;
+            try {
+                mimeType = parseMimeType(rawMimeType);
+            }
+            catch {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            if (mimeType.type !== 'audio' ||
+                (mimeType.subtype !== 'wav' && mimeType.subtype !== 'mp3')) {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            return {
+                type: 'input_audio',
+                input_audio: {
+                    format: mimeType.subtype,
+                    data: data.data,
+                },
+            };
+        }
+        if (block.source_type === 'base64') {
+            let mimeType;
+            try {
+                mimeType = parseMimeType(block.mime_type ?? '');
+            }
+            catch {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            if (mimeType.type !== 'audio' ||
+                (mimeType.subtype !== 'wav' && mimeType.subtype !== 'mp3')) {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            return {
+                type: 'input_audio',
+                input_audio: {
+                    format: mimeType.subtype,
+                    data: block.data,
+                },
+            };
+        }
+        throw new Error(`Audio content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
+    },
+    fromStandardFileBlock(block) {
+        if (block.source_type === 'url') {
+            const data = parseBase64DataUrl({ dataUrl: block.url });
+            if (!data) {
+                throw new Error(`URL file blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
+            }
+            return {
+                type: 'file',
+                file: {
+                    file_data: block.url, // formatted as base64 data URL
+                    ...(block.metadata?.filename || block.metadata?.name
+                        ? {
+                            filename: (block.metadata.filename ||
+                                block.metadata.name),
+                        }
+                        : {}),
+                },
+            };
+        }
+        if (block.source_type === 'base64') {
+            return {
+                type: 'file',
+                file: {
+                    file_data: `data:${block.mime_type ?? ''};base64,${block.data}`,
+                    ...(block.metadata?.filename ||
+                        block.metadata?.name ||
+                        block.metadata?.title
+                        ? {
+                            filename: (block.metadata.filename ||
+                                block.metadata.name ||
+                                block.metadata.title),
+                        }
+                        : {}),
+                },
+            };
+        }
+        if (block.source_type === 'id') {
+            return {
+                type: 'file',
+                file: {
+                    file_id: block.id,
+                },
+            };
+        }
+        throw new Error(`File content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
+    },
+};
+// Used in LangSmith, export is important here
+function _convertMessagesToOpenAIParams(messages, model) {
+    // TODO: Function messages do not support array content, fix cast
+    return messages.flatMap((message) => {
+        let role = messageToOpenAIRole(message);
+        if (role === 'system' && isReasoningModel(model)) {
+            role = 'developer';
+        }
+        const content = typeof message.content === 'string'
+            ? message.content
+            : message.content.map((m) => {
+                if (isDataContentBlock(m)) {
+                    return convertToProviderContentBlock(m, completionsApiContentBlockConverter);
+                }
+                return m;
+            });
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const completionParam = {
+            role,
+            content,
+        };
+        if (message.name != null) {
+            completionParam.name = message.name;
+        }
+        if (message.additional_kwargs.function_call != null) {
+            completionParam.function_call = message.additional_kwargs.function_call;
+            completionParam.content = '';
+        }
+        if (isAIMessage(message) && !!message.tool_calls?.length) {
+            completionParam.tool_calls = message.tool_calls.map(convertLangChainToolCallToOpenAI);
+            completionParam.content = '';
+        }
+        else {
+            if (message.additional_kwargs.tool_calls != null) {
+                completionParam.tool_calls = message.additional_kwargs.tool_calls;
+            }
+            if (message.tool_call_id != null) {
+                completionParam.tool_call_id = message.tool_call_id;
+            }
+        }
+        if (message.additional_kwargs.audio &&
+            typeof message.additional_kwargs.audio === 'object' &&
+            'id' in message.additional_kwargs.audio) {
+            const audioMessage = {
+                role: 'assistant',
+                audio: {
+                    id: message.additional_kwargs.audio.id,
+                },
+            };
+            return [completionParam, audioMessage];
+        }
+        return completionParam;
+    });
+}
+const _FUNCTION_CALL_IDS_MAP_KEY = '__openai_function_call_ids__';
+function _convertReasoningSummaryToOpenAIResponsesParams(reasoning) {
+    // combine summary parts that have the the same index and then remove the indexes
+    const summary = (reasoning.summary.length > 1
+        ? reasoning.summary.reduce((acc, curr) => {
+            const last = acc.at(-1);
+            if (last.index === curr.index) {
+                last.text += curr.text;
+            }
+            else {
+                acc.push(curr);
+            }
+            return acc;
+        }, [{ ...reasoning.summary[0] }])
+        : reasoning.summary).map((s) => Object.fromEntries(Object.entries(s).filter(([k]) => k !== 'index')));
+    return {
+        ...reasoning,
+        summary,
+    };
+}
+function _convertMessagesToOpenAIResponsesParams(messages, model, zdrEnabled) {
+    return messages.flatMap((lcMsg) => {
+        const additional_kwargs = lcMsg.additional_kwargs;
+        let role = messageToOpenAIRole(lcMsg);
+        if (role === 'system' && isReasoningModel(model))
+            role = 'developer';
+        if (role === 'function') {
+            throw new Error('Function messages are not supported in Responses API');
+        }
+        if (role === 'tool') {
+            const toolMessage = lcMsg;
+            // Handle computer call output
+            if (additional_kwargs.type === 'computer_call_output') {
+                const output = (() => {
+                    if (typeof toolMessage.content === 'string') {
+                        return {
+                            type: 'computer_screenshot',
+                            image_url: toolMessage.content,
+                        };
+                    }
+                    if (Array.isArray(toolMessage.content)) {
+                        const oaiScreenshot = toolMessage.content.find((i) => i.type === 'computer_screenshot');
+                        if (oaiScreenshot)
+                            return oaiScreenshot;
+                        const lcImage = toolMessage.content.find((i) => i.type === 'image_url');
+                        if (lcImage) {
+                            return {
+                                type: 'computer_screenshot',
+                                image_url: typeof lcImage.image_url === 'string'
+                                    ? lcImage.image_url
+                                    : lcImage.image_url.url,
+                            };
+                        }
+                    }
+                    throw new Error('Invalid computer call output');
+                })();
+                return {
+                    type: 'computer_call_output',
+                    output,
+                    call_id: toolMessage.tool_call_id,
+                };
+            }
+            return {
+                type: 'function_call_output',
+                call_id: toolMessage.tool_call_id,
+                id: toolMessage.id?.startsWith('fc_') ? toolMessage.id : undefined,
+                output: typeof toolMessage.content !== 'string'
+                    ? JSON.stringify(toolMessage.content)
+                    : toolMessage.content,
+            };
+        }
+        if (role === 'assistant') {
+            // if we have the original response items, just reuse them
+            if (!zdrEnabled &&
+                lcMsg.response_metadata.output != null &&
+                Array.isArray(lcMsg.response_metadata.output) &&
+                lcMsg.response_metadata.output.length > 0 &&
+                lcMsg.response_metadata.output.every((item) => 'type' in item)) {
+                return lcMsg.response_metadata.output;
+            }
+            // otherwise, try to reconstruct the response from what we have
+            const input = [];
+            // reasoning items
+            if (additional_kwargs.reasoning && !zdrEnabled) {
+                const reasoningItem = _convertReasoningSummaryToOpenAIResponsesParams(additional_kwargs.reasoning);
+                input.push(reasoningItem);
+            }
+            // ai content
+            let { content } = lcMsg;
+            if (additional_kwargs.refusal) {
+                if (typeof content === 'string') {
+                    content = [{ type: 'output_text', text: content, annotations: [] }];
+                }
+                content = [
+                    ...content,
+                    { type: 'refusal', refusal: additional_kwargs.refusal },
+                ];
+            }
+            input.push({
+                type: 'message',
+                role: 'assistant',
+                ...(lcMsg.id && !zdrEnabled ? { id: lcMsg.id } : {}),
+                content: typeof content === 'string'
+                    ? content
+                    : content.flatMap((item) => {
+                        if (item.type === 'text') {
+                            return {
+                                type: 'output_text',
+                                text: item.text,
+                                // @ts-expect-error TODO: add types for `annotations`
+                                annotations: item.annotations ?? [],
+                            };
+                        }
+                        if (item.type === 'output_text' || item.type === 'refusal') {
+                            return item;
+                        }
+                        return [];
+                    }),
+            });
+            const functionCallIds = additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY];
+            if (isAIMessage(lcMsg) && !!lcMsg.tool_calls?.length) {
+                input.push(...lcMsg.tool_calls.map((toolCall) => ({
+                    type: 'function_call',
+                    name: toolCall.name,
+                    arguments: JSON.stringify(toolCall.args),
+                    call_id: toolCall.id,
+                    ...(zdrEnabled ? { id: functionCallIds?.[toolCall.id] } : {}),
+                })));
+            }
+            else if (additional_kwargs.tool_calls) {
+                input.push(...additional_kwargs.tool_calls.map((toolCall) => ({
+                    type: 'function_call',
+                    name: toolCall.function.name,
+                    call_id: toolCall.id,
+                    arguments: toolCall.function.arguments,
+                    ...(zdrEnabled ? { id: functionCallIds?.[toolCall.id] } : {}),
+                })));
+            }
+            const toolOutputs = (lcMsg.response_metadata.output?.length ?? 0) > 0
+                ? lcMsg.response_metadata.output
+                : additional_kwargs.tool_outputs;
+            const fallthroughCallTypes = [
+                'computer_call',
+                /** @ts-ignore */
+                'mcp_call',
+                /** @ts-ignore */
+                'code_interpreter_call',
+                /** @ts-ignore */
+                'image_generation_call',
+            ];
+            if (toolOutputs != null) {
+                const castToolOutputs = toolOutputs;
+                const fallthroughCalls = castToolOutputs.filter((item) => fallthroughCallTypes.includes(item.type));
+                if (fallthroughCalls.length > 0)
+                    input.push(...fallthroughCalls);
+            }
+            return input;
+        }
+        if (role === 'user' || role === 'system' || role === 'developer') {
+            if (typeof lcMsg.content === 'string') {
+                return { type: 'message', role, content: lcMsg.content };
+            }
+            const messages = [];
+            const content = lcMsg.content.flatMap((item) => {
+                if (item.type === 'mcp_approval_response') {
+                    messages.push({
+                        // @ts-ignore
+                        type: 'mcp_approval_response',
+                        approval_request_id: item.approval_request_id,
+                        approve: item.approve,
+                    });
+                }
+                if (isDataContentBlock(item)) {
+                    return convertToProviderContentBlock(item, completionsApiContentBlockConverter);
+                }
+                if (item.type === 'text') {
+                    return {
+                        type: 'input_text',
+                        text: item.text,
+                    };
+                }
+                if (item.type === 'image_url') {
+                    return {
+                        type: 'input_image',
+                        image_url: typeof item.image_url === 'string'
+                            ? item.image_url
+                            : item.image_url.url,
+                        detail: typeof item.image_url === 'string'
+                            ? 'auto'
+                            : item.image_url.detail,
+                    };
+                }
+                if (item.type === 'input_text' ||
+                    item.type === 'input_image' ||
+                    item.type === 'input_file') {
+                    return item;
+                }
+                return [];
+            });
+            if (content.length > 0) {
+                messages.push({ type: 'message', role, content });
+            }
+            return messages;
+        }
+        console.warn(`Unsupported role found when converting to OpenAI Responses API: ${role}`);
+        return [];
+    });
+}
+function isReasoningModel(model) {
+    return model != null && model !== '' && /\b(o\d|gpt-[5-9])\b/i.test(model);
+}
+function _convertOpenAIResponsesMessageToBaseMessage(response) {
+    if (response.error) {
+        // TODO: add support for `addLangChainErrorFields`
+        const error = new Error(response.error.message);
+        error.name = response.error.code;
+        throw error;
+    }
+    let messageId;
+    const content = [];
+    const tool_calls = [];
+    const invalid_tool_calls = [];
+    const response_metadata = {
+        model: response.model,
+        created_at: response.created_at,
+        id: response.id,
+        incomplete_details: response.incomplete_details,
+        metadata: response.metadata,
+        object: response.object,
+        status: response.status,
+        user: response.user,
+        service_tier: response.service_tier,
+        // for compatibility with chat completion calls.
+        model_name: response.model,
+    };
+    const additional_kwargs = {};
+    for (const item of response.output) {
+        if (item.type === 'message') {
+            messageId = item.id;
+            content.push(...item.content.flatMap((part) => {
+                if (part.type === 'output_text') {
+                    if ('parsed' in part && part.parsed != null) {
+                        additional_kwargs.parsed = part.parsed;
+                    }
+                    return {
+                        type: 'text',
+                        text: part.text,
+                        annotations: part.annotations,
+                    };
+                }
+                if (part.type === 'refusal') {
+                    additional_kwargs.refusal = part.refusal;
+                    return [];
+                }
+                return part;
+            }));
+        }
+        else if (item.type === 'function_call') {
+            const fnAdapter = {
+                function: { name: item.name, arguments: item.arguments },
+                id: item.call_id,
+            };
+            try {
+                tool_calls.push(parseToolCall(fnAdapter, { returnId: true }));
+            }
+            catch (e) {
+                let errMessage;
+                if (typeof e === 'object' &&
+                    e != null &&
+                    'message' in e &&
+                    typeof e.message === 'string') {
+                    errMessage = e.message;
+                }
+                invalid_tool_calls.push(makeInvalidToolCall(fnAdapter, errMessage));
+            }
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] ??= {};
+            if (item.id) {
+                additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
+            }
+        }
+        else if (item.type === 'reasoning') {
+            additional_kwargs.reasoning = item;
+        }
+        else {
+            additional_kwargs.tool_outputs ??= [];
+            additional_kwargs.tool_outputs.push(item);
+        }
+    }
+    return new AIMessage({
+        id: messageId,
+        content,
+        tool_calls,
+        invalid_tool_calls,
+        usage_metadata: response.usage,
+        additional_kwargs,
+        response_metadata,
+    });
+}
+function _convertOpenAIResponsesDeltaToBaseMessageChunk(chunk) {
+    const content = [];
+    let generationInfo = {};
+    let usage_metadata;
+    const tool_call_chunks = [];
+    const response_metadata = {};
+    const additional_kwargs = {};
+    let id;
+    if (chunk.type === 'response.output_text.delta') {
+        content.push({
+            type: 'text',
+            text: chunk.delta,
+            index: chunk.content_index,
+        });
+        /** @ts-ignore */
+    }
+    else if (chunk.type === 'response.output_text_annotation.added') {
+        content.push({
+            type: 'text',
+            text: '',
+            /** @ts-ignore */
+            annotations: [chunk.annotation],
+            /** @ts-ignore */
+            index: chunk.content_index,
+        });
+    }
+    else if (chunk.type === 'response.output_item.added' &&
+        chunk.item.type === 'message') {
+        id = chunk.item.id;
+    }
+    else if (chunk.type === 'response.output_item.added' &&
+        chunk.item.type === 'function_call') {
+        tool_call_chunks.push({
+            type: 'tool_call_chunk',
+            name: chunk.item.name,
+            args: chunk.item.arguments,
+            id: chunk.item.call_id,
+            index: chunk.output_index,
+        });
+        additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] = {
+            [chunk.item.call_id]: chunk.item.id,
+        };
+    }
+    else if (chunk.type === 'response.output_item.done' &&
+        [
+            'web_search_call',
+            'file_search_call',
+            'computer_call',
+            'code_interpreter_call',
+            'mcp_call',
+            'mcp_list_tools',
+            'mcp_approval_request',
+            'image_generation_call',
+        ].includes(chunk.item.type)) {
+        additional_kwargs.tool_outputs = [chunk.item];
+    }
+    else if (chunk.type === 'response.created') {
+        response_metadata.id = chunk.response.id;
+        response_metadata.model_name = chunk.response.model;
+        response_metadata.model = chunk.response.model;
+    }
+    else if (chunk.type === 'response.completed') {
+        const msg = _convertOpenAIResponsesMessageToBaseMessage(chunk.response);
+        usage_metadata = chunk.response.usage;
+        if (chunk.response.text?.format?.type === 'json_schema') {
+            additional_kwargs.parsed ??= JSON.parse(msg.text);
+        }
+        for (const [key, value] of Object.entries(chunk.response)) {
+            if (key !== 'id')
+                response_metadata[key] = value;
+        }
+    }
+    else if (chunk.type === 'response.function_call_arguments.delta') {
+        tool_call_chunks.push({
+            type: 'tool_call_chunk',
+            args: chunk.delta,
+            index: chunk.output_index,
+        });
+    }
+    else if (chunk.type === 'response.web_search_call.completed' ||
+        chunk.type === 'response.file_search_call.completed') {
+        generationInfo = {
+            tool_outputs: {
+                id: chunk.item_id,
+                type: chunk.type.replace('response.', '').replace('.completed', ''),
+                status: 'completed',
+            },
+        };
+    }
+    else if (chunk.type === 'response.refusal.done') {
+        additional_kwargs.refusal = chunk.refusal;
+    }
+    else if (chunk.type === 'response.output_item.added' &&
+        'item' in chunk &&
+        chunk.item.type === 'reasoning') {
+        const summary = chunk
+            .item.summary
+            ? chunk.item.summary.map((s, index) => ({
+                ...s,
+                index,
+            }))
+            : undefined;
+        additional_kwargs.reasoning = {
+            // We only capture ID in the first chunk or else the concatenated result of all chunks will
+            // have an ID field that is repeated once per chunk. There is special handling for the `type`
+            // field that prevents this, however.
+            id: chunk.item.id,
+            type: chunk.item.type,
+            ...(summary ? { summary } : {}),
+        };
+    }
+    else if (chunk.type === 'response.reasoning_summary_part.added') {
+        additional_kwargs.reasoning = {
+            type: 'reasoning',
+            summary: [{ ...chunk.part, index: chunk.summary_index }],
+        };
+    }
+    else if (chunk.type === 'response.reasoning_summary_text.delta') {
+        additional_kwargs.reasoning = {
+            type: 'reasoning',
+            summary: [
+                { text: chunk.delta, type: 'summary_text', index: chunk.summary_index },
+            ],
+        };
+        /** @ts-ignore */
+    }
+    else if (chunk.type === 'response.image_generation_call.partial_image') {
+        // noop/fixme: retaining partial images in a message chunk means that _all_
+        // partial images get kept in history, so we don't do anything here.
+        return null;
+    }
+    else {
+        return null;
+    }
+    return new ChatGenerationChunk({
+        // Legacy reasons, `onLLMNewToken` should pulls this out
+        text: content.map((part) => part.text).join(''),
+        message: new AIMessageChunk({
+            id,
+            content,
+            tool_call_chunks,
+            usage_metadata,
+            additional_kwargs,
+            response_metadata,
+        }),
+        generationInfo,
+    });
+}
+
+export { _convertMessagesToOpenAIParams, _convertMessagesToOpenAIResponsesParams, _convertOpenAIResponsesDeltaToBaseMessageChunk, isReasoningModel, messageToOpenAIRole };
+//# sourceMappingURL=index.mjs.map