@agentscope-ai/agentscope 0.0.2

package/src/formatter/ollama-chat-formatter.ts

@@ -0,0 +1,67 @@
+import { FormatterBase } from './base';
+import { Msg, getContentBlocks, getTextContent } from '../message';
+
+/**
+ * Format AgentScope message objects into Ollama Chat message format.
+ * Ollama expects simple string content, not the multimodal array format.
+ */
+export class OllamaChatFormatter extends FormatterBase {
+    // eslint-disable-next-line jsdoc/require-returns
+    /**
+     * Format messages for Ollama API
+     * @param root0
+     * @param root0.msgs
+     */
+    async format({ msgs }: { msgs: Array<Msg> }): Promise<Record<string, unknown>[]> {
+        const formattedMsgs: Array<Record<string, unknown>> = [];
+
+        for (const msg of msgs) {
+            const formattedMsg: {
+                role: string;
+                content: string;
+                tool_calls?: {
+                    function: {
+                        name: string;
+                        arguments: Record<string, unknown>;
+                    };
+                }[];
+            } = {
+                role: msg.role,
+                content: '',
+            };
+
+            // Extract text content
+            const textContent = getTextContent(msg);
+            if (textContent) {
+                formattedMsg.content = textContent;
+            }
+
+            // Handle tool calls
+            const toolCalls = getContentBlocks(msg, 'tool_call');
+            if (toolCalls.length > 0) {
+                formattedMsg.tool_calls = toolCalls.map(toolCall => ({
+                    function: {
+                        name: toolCall.name,
+                        arguments: JSON.parse(toolCall.input),
+                    },
+                }));
+            }
+
+            // Handle tool results
+            const toolResults = getContentBlocks(msg, 'tool_result');
+            for (const toolResult of toolResults) {
+                const resultText = this.convertToolOutputToString(toolResult.output, false);
+                formattedMsgs.push({
+                    role: 'tool',
+                    content: resultText.text,
+                });
+            }
+
+            if (formattedMsg.content || formattedMsg.tool_calls) {
+                formattedMsgs.push(formattedMsg);
+            }
+        }
+
+        return formattedMsgs;
+    }
+}

package/src/formatter/openai-chat-formatter.test.ts

@@ -0,0 +1,263 @@
+import { createMsg } from '../message';
+import { OpenAIChatFormatter } from './openai-chat-formatter';
+
+describe('OpenAIChatFormatter', () => {
+    test('format textual messages', async () => {
+        const msgs = [
+            createMsg({
+                name: 'system',
+                content: [
+                    { id: crypto.randomUUID(), type: 'text', text: 'You are a helpful assistant.' },
+                ],
+                role: 'system',
+            }),
+            createMsg({
+                name: 'user',
+                content: [{ id: crypto.randomUUID(), type: 'text', text: 'Hello, how are you?' }],
+                role: 'user',
+            }),
+            createMsg({
+                name: 'assistant',
+                content: [{ id: crypto.randomUUID(), type: 'text', text: 'I am fine, thank you!' }],
+                role: 'assistant',
+            }),
+        ];
+
+        const formatter = new OpenAIChatFormatter();
+        const res = await formatter.format({ msgs });
+        expect(res).toEqual([
+            {
+                role: 'system',
+                name: 'system',
+                content: [{ type: 'text', text: 'You are a helpful assistant.' }],
+            },
+            {
+                role: 'user',
+                name: 'user',
+                content: [{ type: 'text', text: 'Hello, how are you?' }],
+            },
+            {
+                role: 'assistant',
+                name: 'assistant',
+                content: [{ type: 'text', text: 'I am fine, thank you!' }],
+            },
+        ]);
+    });
+
+    test('format tool messages', async () => {
+        const msgs = [
+            createMsg({
+                name: 'assistant',
+                content: [
+                    {
+                        type: 'tool_call',
+                        id: '1',
+                        name: 'google_search',
+                        input: '{"query": "example1"}',
+                    },
+                    {
+                        type: 'tool_call',
+                        id: '2',
+                        name: 'bing_search',
+                        input: '{"query": "example2"}',
+                    },
+                    {
+                        type: 'tool_result',
+                        id: '1',
+                        name: 'google_search',
+                        output: 'Google search result for example1',
+                        state: 'success',
+                    },
+                    {
+                        type: 'tool_result',
+                        id: '2',
+                        name: 'bing_search',
+                        output: 'Bing search result for example2',
+                        state: 'success',
+                    },
+                ],
+                role: 'assistant',
+            }),
+        ];
+
+        const formatter = new OpenAIChatFormatter();
+        const res = await formatter.format({ msgs });
+        expect(res).toEqual([
+            {
+                role: 'assistant',
+                name: 'assistant',
+                content: null,
+                tool_calls: [
+                    {
+                        id: '1',
+                        type: 'function',
+                        function: {
+                            name: 'google_search',
+                            arguments: '{"query": "example1"}',
+                        },
+                    },
+                    {
+                        id: '2',
+                        type: 'function',
+                        function: {
+                            name: 'bing_search',
+                            arguments: '{"query": "example2"}',
+                        },
+                    },
+                ],
+            },
+            {
+                role: 'tool',
+                tool_call_id: '1',
+                name: 'google_search',
+                content: 'Google search result for example1',
+            },
+            {
+                role: 'tool',
+                tool_call_id: '2',
+                name: 'bing_search',
+                content: 'Bing search result for example2',
+            },
+        ]);
+    });
+
+    test('format multimodal messages', async () => {
+        const msgs = [
+            createMsg({
+                name: 'user',
+                content: [
+                    { id: crypto.randomUUID(), type: 'text', text: 'Please see the image below.' },
+                    {
+                        id: crypto.randomUUID(),
+                        type: 'data',
+                        source: {
+                            type: 'url',
+                            url: 'https://example.com/image.png',
+                            mediaType: 'image/png',
+                        },
+                    },
+                    {
+                        id: crypto.randomUUID(),
+                        type: 'data',
+                        source: { type: 'base64', data: 'xxx', mediaType: 'audio/mp3' },
+                    },
+                ],
+                role: 'user',
+            }),
+            createMsg({
+                name: 'assistant',
+                content: [
+                    {
+                        id: crypto.randomUUID(),
+                        type: 'data',
+                        source: {
+                            type: 'base64',
+                            data: 'assistant-audio',
+                            mediaType: 'audio/mp3',
+                        },
+                    },
+                ],
+                role: 'assistant',
+            }),
+        ];
+
+        const formatter = new OpenAIChatFormatter();
+        const res = await formatter.format({ msgs });
+        expect(res).toEqual([
+            {
+                role: 'user',
+                name: 'user',
+                content: [
+                    { type: 'text', text: 'Please see the image below.' },
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: 'https://example.com/image.png',
+                        },
+                    },
+                    {
+                        type: 'input_audio',
+                        input_audio: {
+                            data: 'xxx',
+                            format: 'mp3',
+                        },
+                    },
+                ],
+            },
+        ]);
+    });
+
+    test('format tool result with promoted multimodal blocks', async () => {
+        const mockRandom = jest.spyOn(Math, 'random');
+        mockRandom.mockReturnValueOnce(0.123456789);
+
+        const msgs = [
+            createMsg({
+                name: 'assistant',
+                content: [
+                    { type: 'tool_call', id: '1', name: 'google_search', input: '{"query": "A"}' },
+                    {
+                        type: 'tool_result',
+                        id: '1',
+                        name: 'google_search',
+                        output: [
+                            { type: 'text', text: 'content 1', id: crypto.randomUUID() },
+                            {
+                                type: 'data',
+                                source: { type: 'base64', data: 'img64', mediaType: 'image/png' },
+                                id: crypto.randomUUID(),
+                            },
+                        ],
+                        state: 'success',
+                    },
+                ],
+                role: 'assistant',
+            }),
+        ];
+
+        const formatter = new OpenAIChatFormatter({
+            promoteMultimodalToolResult: { image: true },
+        });
+        const res = await formatter.format({ msgs });
+        mockRandom.mockRestore();
+
+        expect(res).toEqual([
+            {
+                role: 'assistant',
+                name: 'assistant',
+                content: null,
+                tool_calls: [
+                    {
+                        id: '1',
+                        type: 'function',
+                        function: { name: 'google_search', arguments: '{"query": "A"}' },
+                    },
+                ],
+            },
+            {
+                role: 'tool',
+                tool_call_id: '1',
+                name: 'google_search',
+                content:
+                    "content 1\n<system-info>One returned image is embedded with ID '4fzzzxjy' and will be attached within '<system-info></system-info>' tags later.</system-info>",
+            },
+            {
+                role: 'user',
+                name: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: "<system-info>The multimodal contents returned from the tool call are as follows:\n<image_data id='4fzzzxjy'>",
+                    },
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: 'data:image/png;base64,img64',
+                        },
+                    },
+                    { type: 'text', text: '</image_data>\n</system-info>' },
+                ],
+            },
+        ]);
+    });
+});

package/src/formatter/openai-chat-formatter.ts

@@ -0,0 +1,301 @@
+import { existsSync } from 'fs';
+import { readFile } from 'fs/promises';
+import { extname } from 'path';
+import { fileURLToPath } from 'url';
+
+import { FormatterBase } from './base';
+import { DataBlock, Msg, TextBlock, getContentBlocks } from '../message';
+
+interface OpenAIFormatterOptions {
+    /**
+     * Most LLM APIs don't support multimodal tool outputs, this option controls whether to
+     * promote multimodal tool results to follow-up user messages.
+     */
+    promoteMultimodalToolResult?:
+        | {
+              image?: boolean;
+              audio?: boolean;
+              video?: boolean;
+          }
+        | boolean;
+}
+
+/**
+ * Format AgentScope message objects into OpenAI Chat Completions message format.
+ */
+export class OpenAIChatFormatter extends FormatterBase {
+    private promoteMultimodalToolResult:
+        | { image?: boolean; audio?: boolean; video?: boolean }
+        | boolean;
+
+    /**
+     * Initializes a new instance of the OpenAIChatFormatter class.
+     * @param root0
+     * @param root0.promoteMultimodalToolResult
+     */
+    constructor({ promoteMultimodalToolResult = false }: OpenAIFormatterOptions = {}) {
+        super();
+        this.promoteMultimodalToolResult = promoteMultimodalToolResult;
+    }
+
+    /**
+     * Format the input messages into OpenAI Chat Completions message format.
+     * @param root0
+     * @param root0.msgs
+     * @returns An array of formatted messages compatible with OpenAI Chat Completions API.
+     */
+    async format({ msgs }: { msgs: Array<Msg> }): Promise<Record<string, unknown>[]> {
+        const formattedMsgs: Array<Record<string, unknown>> = [];
+        let index = 0;
+
+        while (index < msgs.length) {
+            const msg = msgs[index];
+            const formattedMsg: {
+                role: string;
+                name: string;
+                content: Record<string, unknown>[] | null;
+                tool_calls?: {
+                    id: string;
+                    type: 'function';
+                    function: {
+                        name: string;
+                        arguments: string;
+                    };
+                }[];
+            } = {
+                role: msg.role,
+                name: msg.name,
+                content: null,
+            };
+            const content: Record<string, unknown>[] = [];
+
+            // Cache tool-result messages to keep the sequence right after current message.
+            const cachedMsgs: Record<string, unknown>[] = [];
+            for (const block of getContentBlocks(msg)) {
+                switch (block.type) {
+                    case 'text':
+                        content.push(this._formatTextBlock(block));
+                        break;
+                    case 'thinking':
+                        break;
+                    case 'tool_call':
+                        if (!formattedMsg.tool_calls) {
+                            formattedMsg.tool_calls = [];
+                        }
+                        formattedMsg.tool_calls.push({
+                            id: block.id,
+                            type: 'function',
+                            function: {
+                                name: block.name,
+                                arguments: block.input,
+                            },
+                        });
+                        break;
+                    case 'tool_result':
+                        const formattedToolResult = this.convertToolOutputToString(
+                            block.output,
+                            this.promoteMultimodalToolResult
+                        );
+                        cachedMsgs.push({
+                            role: 'tool',
+                            tool_call_id: block.id,
+                            name: block.name,
+                            content: formattedToolResult.text,
+                        });
+                        if (formattedToolResult.promotedMsg?.content.length) {
+                            msgs.splice(index + 1, 0, formattedToolResult.promotedMsg);
+                        }
+                        break;
+                    case 'data':
+                        content.push(
+                            ...(await this._formatMultimodalBlock({ block, role: msg.role }))
+                        );
+                        break;
+                }
+            }
+
+            if (content.length > 0) {
+                formattedMsg.content = content;
+            }
+            if (formattedMsg.content || formattedMsg.tool_calls) {
+                formattedMsgs.push(formattedMsg);
+            }
+            if (cachedMsgs.length > 0) {
+                formattedMsgs.push(...cachedMsgs);
+            }
+
+            index++;
+        }
+
+        return formattedMsgs;
+    }
+
+    /**
+     * Format a text block into OpenAI Chat Completions message content format.
+     * @param block
+     * @returns An object representing the formatted text block.
+     */
+    _formatTextBlock(block: TextBlock) {
+        return {
+            type: 'text',
+            text: block.text,
+        };
+    }
+
+    /**
+     * Format a multimodal data block into OpenAI Chat Completions message content format.
+     * @param root0
+     * @param root0.block
+     * @param root0.role
+     * @returns The formatted content blocks
+     */
+    async _formatMultimodalBlock({
+        block,
+        role,
+    }: {
+        block: DataBlock;
+        role: Msg['role'];
+    }): Promise<Record<string, unknown>[]> {
+        const type = block.source.mediaType.split('/')[0];
+        if (type === 'image') {
+            return [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: await this._toOpenAIImageURL(block),
+                    },
+                },
+            ];
+        }
+
+        if (type === 'audio') {
+            // Skip assistant output audio to avoid carrying generated audio back into next request.
+            if (role === 'assistant') {
+                return [];
+            }
+            return [
+                {
+                    type: 'input_audio',
+                    input_audio: await this._toOpenAIAudioData(block),
+                },
+            ];
+        }
+
+        console.log(
+            `Skip unsupported media type ${block.source.mediaType} in OpenAIChatFormatter. Only image and audio are supported.`
+        );
+        return [];
+    }
+
+    /**
+     * Convert the data block to an OpenAI compatible image URL.
+     * @param block
+     * @returns A promise that resolves to a string representing the image URL in a format compatible with OpenAI Chat Completions API.
+     */
+    protected async _toOpenAIImageURL(block: DataBlock): Promise<string> {
+        if (block.source.type === 'base64') {
+            return `data:${block.source.mediaType};base64,${block.source.data}`;
+        }
+
+        const sourceUrl = block.source.url;
+        if (sourceUrl.startsWith('http://') || sourceUrl.startsWith('https://')) {
+            return sourceUrl;
+        }
+        if (sourceUrl.startsWith('data:')) {
+            return sourceUrl;
+        }
+
+        const localPath = this._toLocalPath(sourceUrl);
+        if (!localPath || !existsSync(localPath)) {
+            throw new Error(`Image path not found: ${sourceUrl}`);
+        }
+
+        const ext = extname(localPath).toLowerCase();
+        const supportedImageExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.webp'];
+        if (!supportedImageExtensions.includes(ext)) {
+            throw new TypeError(
+                `Unsupported image extension: ${ext}. Supported: ${supportedImageExtensions.join(', ')}`
+            );
+        }
+
+        const file = await readFile(localPath);
+        const mime = block.source.mediaType || `image/${ext.slice(1)}`;
+        return `data:${mime};base64,${file.toString('base64')}`;
+    }
+
+    /**
+     * Converts a data block to OpenAI compatible audio data format.
+     *
+     * @param block - The data block containing audio information.
+     * @returns A promise that resolves to an object with audio data and format.
+     */
+    protected async _toOpenAIAudioData(
+        block: DataBlock
+    ): Promise<{ data: string; format: 'wav' | 'mp3' }> {
+        const supportedMediaTypes = new Map<string, 'wav' | 'mp3'>([
+            ['audio/wav', 'wav'],
+            ['audio/mp3', 'mp3'],
+            ['audio/mpeg', 'mp3'],
+        ]);
+
+        if (block.source.type === 'base64') {
+            const format = supportedMediaTypes.get(block.source.mediaType);
+            if (!format) {
+                throw new TypeError(
+                    `Unsupported audio media type: ${block.source.mediaType}, only audio/wav and audio/mp3 are supported.`
+                );
+            }
+            return { data: block.source.data, format };
+        }
+
+        const sourceUrl = block.source.url;
+        const localPath = this._toLocalPath(sourceUrl);
+        let data: string;
+
+        if (localPath && existsSync(localPath)) {
+            const file = await readFile(localPath);
+            data = file.toString('base64');
+        } else if (sourceUrl.startsWith('http://') || sourceUrl.startsWith('https://')) {
+            const response = await fetch(sourceUrl);
+            if (!response.ok) {
+                throw new Error(
+                    `Failed to fetch audio from URL: ${sourceUrl} (${response.status})`
+                );
+            }
+            const arr = await response.arrayBuffer();
+            data = Buffer.from(arr).toString('base64');
+        } else {
+            throw new Error(
+                `Unsupported audio source: ${sourceUrl}, it should be a local file path, file URL, or an HTTP URL.`
+            );
+        }
+
+        const ext = extname(localPath || sourceUrl).toLowerCase();
+        const extToFormat = new Map<string, 'wav' | 'mp3'>([
+            ['.wav', 'wav'],
+            ['.mp3', 'mp3'],
+        ]);
+        const format = extToFormat.get(ext);
+        if (!format) {
+            throw new TypeError(`Unsupported audio extension: ${ext}, wav and mp3 are supported.`);
+        }
+
+        return { data, format };
+    }
+
+    /**
+     * Converts a URL or path to a local file path.
+     *
+     * @param urlOrPath - The URL or path to convert.
+     * @returns The local file path, or null if not a local path.
+     */
+    protected _toLocalPath(urlOrPath: string) {
+        if (urlOrPath.startsWith('file://')) {
+            return fileURLToPath(urlOrPath);
+        }
+        if (!urlOrPath.includes('://')) {
+            return urlOrPath;
+        }
+        return null;
+    }
+}