@agentscope-ai/agentscope 0.0.2

package/src/formatter/dashscope-chat-formatter.test.ts

@@ -0,0 +1,372 @@
+import { createMsg } from '../message';
+import { DashScopeChatFormatter } from './dashscope-chat-formatter';
+
+describe('DashScopeChatFormatter', () => {
+    test('format textual messages', async () => {
+        const msgs = [
+            createMsg({
+                name: 'system',
+                content: [
+                    { id: crypto.randomUUID(), type: 'text', text: 'You are a helpful assistant.' },
+                ],
+                role: 'system',
+            }),
+            createMsg({
+                name: 'user',
+                content: [{ id: crypto.randomUUID(), type: 'text', text: 'Hello, how are you?' }],
+                role: 'user',
+            }),
+            createMsg({
+                name: 'assistant',
+                content: [{ id: crypto.randomUUID(), type: 'text', text: 'I am fine, thank you!' }],
+                role: 'assistant',
+            }),
+        ];
+
+        const formatter = new DashScopeChatFormatter();
+        const res = await formatter.format({ msgs });
+        expect(res).toEqual([
+            {
+                role: 'system',
+                content: [{ text: 'You are a helpful assistant.' }],
+            },
+            {
+                role: 'user',
+                content: [{ text: 'Hello, how are you?' }],
+            },
+            {
+                role: 'assistant',
+                content: [{ text: 'I am fine, thank you!' }],
+            },
+        ]);
+    });
+
+    test('format tool messages', async () => {
+        const msgs = [
+            createMsg({
+                name: 'system',
+                content: [
+                    { type: 'text', text: 'You are a helpful assistant.', id: crypto.randomUUID() },
+                ],
+                role: 'system',
+            }),
+            createMsg({
+                name: 'user',
+                content: [{ type: 'text', text: 'Please use the tool.', id: crypto.randomUUID() }],
+                role: 'user',
+            }),
+            createMsg({
+                name: 'assistant',
+                content: [
+                    {
+                        type: 'tool_call',
+                        id: '1',
+                        name: 'google_search',
+                        input: '{"query": "example1"}',
+                    },
+                    {
+                        type: 'tool_call',
+                        id: '2',
+                        name: 'bing_search',
+                        input: '{"query": "example2"}',
+                    },
+                    {
+                        type: 'tool_result',
+                        id: '1',
+                        name: 'google_search',
+                        output: 'Google search result for example1',
+                        state: 'success',
+                    },
+                    {
+                        type: 'tool_result',
+                        id: '2',
+                        name: 'bing_search',
+                        output: 'Bing search result for example2',
+                        state: 'success',
+                    },
+                ],
+                role: 'assistant',
+            }),
+        ];
+
+        const formatter = new DashScopeChatFormatter();
+
+        const res = await formatter.format({ msgs });
+        expect(res).toEqual([
+            {
+                role: 'system',
+                content: [{ text: 'You are a helpful assistant.' }],
+            },
+            {
+                role: 'user',
+                content: [{ text: 'Please use the tool.' }],
+            },
+            {
+                role: 'assistant',
+                content: [],
+                tool_calls: [
+                    {
+                        id: '1',
+                        type: 'function',
+                        function: {
+                            name: 'google_search',
+                            arguments: '{"query": "example1"}',
+                        },
+                    },
+                    {
+                        id: '2',
+                        type: 'function',
+                        function: {
+                            name: 'bing_search',
+                            arguments: '{"query": "example2"}',
+                        },
+                    },
+                ],
+            },
+            {
+                role: 'tool',
+                tool_call_id: '1',
+                name: 'google_search',
+                content: 'Google search result for example1',
+            },
+            {
+                role: 'tool',
+                tool_call_id: '2',
+                name: 'bing_search',
+                content: 'Bing search result for example2',
+            },
+        ]);
+    });
+
+    test('format multimodal messages', async () => {
+        const msgs = [
+            createMsg({
+                name: 'system',
+                content: [
+                    { id: crypto.randomUUID(), type: 'text', text: 'You are a helpful assistant.' },
+                ],
+                role: 'system',
+            }),
+            createMsg({
+                name: 'user',
+                content: [
+                    { id: crypto.randomUUID(), type: 'text', text: 'Please see the image below.' },
+                    {
+                        id: crypto.randomUUID(),
+                        type: 'data',
+                        source: {
+                            type: 'url',
+                            url: 'https://example.com/image.png',
+                            mediaType: 'image/png',
+                        },
+                    },
+                ],
+                role: 'user',
+            }),
+            createMsg({
+                name: 'assistant',
+                content: [
+                    {
+                        id: crypto.randomUUID(),
+                        type: 'text',
+                        text: 'Here is the image you requested.',
+                    },
+                ],
+                role: 'assistant',
+            }),
+            createMsg({
+                name: 'user',
+                content: [
+                    {
+                        id: crypto.randomUUID(),
+                        type: 'data',
+                        source: { type: 'base64', data: 'xxx', mediaType: 'audio/mp3' },
+                    },
+                    {
+                        id: crypto.randomUUID(),
+                        type: 'data',
+                        source: {
+                            type: 'url',
+                            url: 'file:///local/path/to/video.mp4',
+                            mediaType: 'video/mp4',
+                        },
+                    },
+                    {
+                        id: crypto.randomUUID(),
+                        type: 'data',
+                        source: {
+                            type: 'url',
+                            url: 'file:///C:/local/path/to/image.jpg',
+                            mediaType: 'image/jpg',
+                        },
+                    },
+                ],
+                role: 'user',
+            }),
+        ];
+
+        const formatter = new DashScopeChatFormatter();
+
+        const res = await formatter.format({ msgs });
+        expect(res).toEqual([
+            {
+                role: 'system',
+                content: [{ text: 'You are a helpful assistant.' }],
+            },
+            {
+                role: 'user',
+                content: [
+                    { text: 'Please see the image below.' },
+                    {
+                        image: 'https://example.com/image.png',
+                    },
+                ],
+            },
+            {
+                role: 'assistant',
+                content: [{ text: 'Here is the image you requested.' }],
+            },
+            {
+                role: 'user',
+                content: [
+                    {
+                        audio: 'data:audio/mp3;base64,xxx',
+                    },
+                    {
+                        video: 'file:///local/path/to/video.mp4',
+                    },
+                    {
+                        image: 'file:///C:/local/path/to/image.jpg',
+                    },
+                ],
+            },
+        ]);
+    });
+
+    test('format multimodal tool results', async () => {
+        // Mock Math.random to generate predictable IDs
+        const mockRandom = jest.spyOn(Math, 'random');
+        mockRandom.mockReturnValueOnce(0.123456789); // Will generate '4fzzzxjy'
+        mockRandom.mockReturnValueOnce(0.456789012); // Will generate 'gfzy4slm'
+
+        const msgs = [
+            createMsg({
+                name: 'user',
+                content: [{ id: crypto.randomUUID(), type: 'text', text: 'Please use the tool.' }],
+                role: 'user',
+            }),
+            createMsg({
+                name: 'assistant',
+                content: [
+                    {
+                        type: 'tool_call',
+                        id: '1',
+                        name: 'google_search',
+                        input: '{\"query\": \"example1\"}',
+                    },
+                    {
+                        type: 'tool_result',
+                        id: '1',
+                        name: 'google_search',
+                        output: [
+                            { type: 'text', text: 'content 1', id: crypto.randomUUID() },
+                            {
+                                id: crypto.randomUUID(),
+                                type: 'data',
+                                source: {
+                                    type: 'url',
+                                    url: 'https://example.com/image1.png',
+                                    mediaType: 'image/png',
+                                },
+                            },
+                            { id: crypto.randomUUID(), type: 'text', text: 'content 2' },
+                            {
+                                id: crypto.randomUUID(),
+                                type: 'data',
+                                source: { type: 'base64', data: 'xxx', mediaType: 'image/png' },
+                            },
+                            {
+                                id: crypto.randomUUID(),
+                                type: 'data',
+                                source: { type: 'base64', data: 'yyy', mediaType: 'audio/mp3' },
+                            },
+                            {
+                                id: crypto.randomUUID(),
+                                type: 'data',
+                                source: {
+                                    type: 'url',
+                                    url: '/local/path/to/video1.mp4',
+                                    mediaType: 'video/mp4',
+                                },
+                            },
+                        ],
+                        state: 'success',
+                    },
+                ],
+                role: 'assistant',
+            }),
+        ];
+
+        const formatter = new DashScopeChatFormatter({ promoteMultimodalToolResult: true });
+
+        const res = await formatter.format({ msgs });
+
+        // Restore the mock
+        mockRandom.mockRestore();
+
+        expect(res).toEqual([
+            {
+                content: [
+                    {
+                        text: 'Please use the tool.',
+                    },
+                ],
+                role: 'user',
+            },
+            {
+                content: [],
+                role: 'assistant',
+                tool_calls: [
+                    {
+                        function: {
+                            arguments: '{"query": "example1"}',
+                            name: 'google_search',
+                        },
+                        id: '1',
+                        type: 'function',
+                    },
+                ],
+            },
+            {
+                content:
+                    "content 1\n<system-info>One returned image can be found at: https://example.com/image1.png</system-info>\ncontent 2\n<system-info>One returned image is embedded with ID '4fzzzxjy' and will be attached within '<system-info></system-info>' tags later.</system-info>\n<system-info>One returned audio is embedded with ID 'gfzy4slm' and will be attached within '<system-info></system-info>' tags later.</system-info>\n<system-info>One returned video can be found at: /local/path/to/video1.mp4</system-info>",
+                name: 'google_search',
+                role: 'tool',
+                tool_call_id: '1',
+            },
+            {
+                content: [
+                    {
+                        text: "<system-info>The multimodal contents returned from the tool call are as follows:\n<image_data id='4fzzzxjy'>",
+                    },
+                    {
+                        image: 'data:image/png;base64,xxx',
+                    },
+                    {
+                        text: '</image_data>\n',
+                    },
+                    {
+                        text: "<audio_data id='gfzy4slm'>",
+                    },
+                    {
+                        audio: 'data:audio/mp3;base64,yyy',
+                    },
+                    {
+                        text: '</audio_data>\n</system-info>',
+                    },
+                ],
+                role: 'user',
+            },
+        ]);
+    });
+});

package/src/formatter/dashscope-chat-formatter.ts

@@ -0,0 +1,163 @@
+import { FormatterBase } from './base';
+import { Msg, TextBlock, getContentBlocks } from '../message';
+import { DataBlock } from '../message';
+
+interface DashScopeFormatterOptions {
+    /**
+     * Since DashScope API doesn't support multimodal tool outputs, this option indicates whether to
+     * promote the multimodal tool results to the prompt messages, so that LLMs can see them.
+     * Note you should ensure your model supports the corresponding modalities.
+     */
+    promoteMultimodalToolResult?:
+        | {
+              image?: boolean;
+              audio?: boolean;
+              video?: boolean;
+          }
+        | boolean;
+}
+
+/**
+ *
+ */
+export class DashScopeChatFormatter extends FormatterBase {
+    private promoteMultimodalToolResult:
+        | { image?: boolean; audio?: boolean; video?: boolean }
+        | boolean;
+
+    /**
+     * Initialize a DashScopeChatFormatter instance.
+     *
+     * @param promoteMultimodalToolResult - Since DashScope API doesn't support multimodal tool outputs, this option
+     *  indicates whether to promote the multimodal tool results to the prompt messages, so that LLMs can see them.
+     *  Note you should ensure your model supports the corresponding modalities.
+     * @param promoteMultimodalToolResult.promoteMultimodalToolResult
+     */
+    constructor({ promoteMultimodalToolResult = false }: DashScopeFormatterOptions = {}) {
+        super();
+        this.promoteMultimodalToolResult = promoteMultimodalToolResult;
+    }
+
+    /**
+     * Format the input message objects into the required format by DashScope API.
+     *
+     * @param msgs - An array of Msg instances to be formatted.
+     * @param msgs.msgs
+     * @returns A promise that resolves to an array of formatted message objects.
+     */
+    async format({ msgs }: { msgs: Array<Msg> }): Promise<Record<string, unknown>[]> {
+        const formattedMsgs: Array<Record<string, unknown>> = [];
+        let index = 0;
+        while (index < msgs.length) {
+            const msg = msgs[index];
+            const formattedMsg: {
+                role: string;
+                content: Record<string, unknown>[];
+                tool_calls?: {
+                    id: string;
+                    type: 'function';
+                    function: {
+                        name: string;
+                        arguments: string;
+                    };
+                }[];
+            } = {
+                role: msg.role,
+                content: [],
+            };
+
+            // The cached messages that should be pushed after the current message, to keep the order of messages correct.
+            const cachedMsgs = [];
+            for (const block of getContentBlocks(msg)) {
+                switch (block.type) {
+                    case 'text':
+                        formattedMsg.content.push(this._formatTextBlock(block));
+                        break;
+                    case 'thinking':
+                        break;
+                    case 'tool_call':
+                        if (!formattedMsg.tool_calls) {
+                            formattedMsg.tool_calls = [];
+                        }
+                        formattedMsg.tool_calls.push({
+                            id: block.id,
+                            type: 'function',
+                            function: {
+                                name: block.name,
+                                arguments: block.input,
+                            },
+                        });
+                        break;
+                    case 'tool_result':
+                        const formattedToolResult = this.convertToolOutputToString(
+                            block.output,
+                            this.promoteMultimodalToolResult
+                        );
+
+                        cachedMsgs.push({
+                            role: 'tool',
+                            tool_call_id: block.id,
+                            name: block.name,
+                            content: formattedToolResult.text,
+                        });
+                        if (formattedToolResult.promotedMsg) {
+                            // Insert the promoted message into the array as the next message to be processed
+                            msgs.splice(index + 1, 0, formattedToolResult.promotedMsg);
+                        }
+                        break;
+                    case 'data':
+                        formattedMsg.content.push(...this._formatMultimodalBlock(block));
+                        break;
+                }
+            }
+            if (formattedMsg.content.length > 0 || formattedMsg.tool_calls) {
+                formattedMsgs.push(formattedMsg);
+            }
+            if (cachedMsgs.length > 0) {
+                formattedMsgs.push(...cachedMsgs);
+            }
+
+            // Process next message
+            index++;
+        }
+        return formattedMsgs;
+    }
+
+    /**
+     * Format a text content block into the required format.
+     *
+     * @param block - The text content block to format.
+     * @returns An object representing the formatted text content.
+     */
+    _formatTextBlock(block: TextBlock) {
+        return { text: block.text };
+    }
+
+    /**
+     * Format a multimodal data block into the required format.
+     * In DashScope API, the local file paths should be prefixed with "file://". URLs are kept unchanged.
+     *
+     * @param block - The multimodal content block to format.
+     * @returns An object representing the formatted multimodal content.
+     */
+    _formatMultimodalBlock(block: DataBlock) {
+        const type = block.source.mediaType.split('/')[0];
+
+        if (!['image', 'audio', 'video'].includes(type)) {
+            console.log(
+                `Skip unsupported media type ${block.source.mediaType} in DashScopeChatFormatter. Only image, audio and video are supported.`
+            );
+            return [];
+        }
+
+        if (block.source.type === 'url') {
+            return [{ [type]: block.source.url }];
+        }
+
+        return [
+            {
+                [type]: `data:${block.source.mediaType};base64,${block.source.data}`,
+            },
+        ];
+    }
+}

package/src/formatter/deepseek-chat-formatter.ts

@@ -0,0 +1,130 @@
+import { FormatterBase } from './base';
+import { Msg, getContentBlocks } from '../message';
+
+interface DeepSeekFormatterOptions {
+    /**
+     * Most LLM APIs don't support multimodal tool outputs, this option controls whether to
+     * promote multimodal tool results to follow-up user messages.
+     */
+    promoteMultimodalToolResult?:
+        | {
+              image?: boolean;
+              audio?: boolean;
+              video?: boolean;
+          }
+        | boolean;
+}
+
+/**
+ * Format AgentScope message objects into DeepSeek Chat Completions message format.
+ */
+export class DeepSeekChatFormatter extends FormatterBase {
+    private promoteMultimodalToolResult:
+        | { image?: boolean; audio?: boolean; video?: boolean }
+        | boolean;
+
+    /**
+     * Initializes a new instance of the DeepSeekChatFormatter class.
+     * @param root0
+     * @param root0.promoteMultimodalToolResult
+     */
+    constructor({ promoteMultimodalToolResult = false }: DeepSeekFormatterOptions = {}) {
+        super();
+        this.promoteMultimodalToolResult = promoteMultimodalToolResult;
+    }
+
+    /**
+     * Format the input messages into the structure expected by DeepSeek Chat Completions API.
+     * @param root0
+     * @param root0.msgs
+     * @returns An array of formatted message objects ready to be sent to the DeepSeek API.
+     */
+    async format({ msgs }: { msgs: Array<Msg> }): Promise<Record<string, unknown>[]> {
+        const formattedMsgs: Array<Record<string, unknown>> = [];
+        let index = 0;
+
+        while (index < msgs.length) {
+            const msg = msgs[index];
+            const formattedMsg: {
+                role: string;
+                name: string;
+                content: Record<string, unknown>[] | null;
+                tool_calls?: {
+                    id: string;
+                    type: 'function';
+                    function: {
+                        name: string;
+                        arguments: string;
+                    };
+                }[];
+            } = {
+                role: msg.role,
+                name: msg.name,
+                content: null,
+            };
+            const content: Record<string, unknown>[] = [];
+
+            // Cache tool-result messages to keep the sequence right after current message.
+            const cachedMsgs: Record<string, unknown>[] = [];
+            for (const block of getContentBlocks(msg)) {
+                switch (block.type) {
+                    case 'text':
+                        content.push({
+                            type: 'text',
+                            text: block.text,
+                        });
+                        break;
+                    case 'thinking':
+                        break;
+                    case 'tool_call':
+                        if (!formattedMsg.tool_calls) {
+                            formattedMsg.tool_calls = [];
+                        }
+                        formattedMsg.tool_calls.push({
+                            id: block.id,
+                            type: 'function',
+                            function: {
+                                name: block.name,
+                                arguments: block.input,
+                            },
+                        });
+                        break;
+                    case 'tool_result':
+                        const formattedToolResult = this.convertToolOutputToString(
+                            block.output,
+                            this.promoteMultimodalToolResult
+                        );
+                        cachedMsgs.push({
+                            role: 'tool',
+                            tool_call_id: block.id,
+                            name: block.name,
+                            content: formattedToolResult.text,
+                        });
+                        if (formattedToolResult.promotedMsg?.content.length) {
+                            msgs.splice(index + 1, 0, formattedToolResult.promotedMsg);
+                        }
+                        break;
+                    case 'data':
+                        console.warn(
+                            `DeepSeek models don't support multimodal data for now (2026-03), skip the data block in message content.`
+                        );
+                        break;
+                }
+            }
+
+            if (content.length > 0) {
+                formattedMsg.content = content;
+            }
+            if (formattedMsg.content || formattedMsg.tool_calls) {
+                formattedMsgs.push(formattedMsg);
+            }
+            if (cachedMsgs.length > 0) {
+                formattedMsgs.push(...cachedMsgs);
+            }
+
+            index++;
+        }
+
+        return formattedMsgs;
+    }
+}

package/src/formatter/index.ts

@@ -0,0 +1,5 @@
+export { FormatterBase } from './base';
+export { DashScopeChatFormatter } from './dashscope-chat-formatter';
+export { DeepSeekChatFormatter } from './deepseek-chat-formatter';
+export { OllamaChatFormatter } from './ollama-chat-formatter';
+export { OpenAIChatFormatter } from './openai-chat-formatter';