cerevox 3.0.0-alpha.2 → 3.0.0-alpha.21

package/dist/mcp/servers/zerocut.js

@@ -55,7 +55,7 @@ const node_fs_1 = require("node:fs");
 const coze_1 = require("../../utils/coze");
 const mp3_duration_1 = __importDefault(require("mp3-duration"));
 const image_size_1 = __importDefault(require("image-size"));
-function createErrorResponse(error, operation) {
+function createErrorResponse(error, operation, details) {
     const errorMessage = error instanceof Error ? error.message : String(error);
     console.error(`[${operation}] Error:`, error);
     return {
@@ -67,6 +67,7 @@ function createErrorResponse(error, operation) {
                     error: errorMessage,
                     operation,
                     timestamp: new Date().toISOString(),
+                    details,
                 }),
             },
         ],
@@ -359,12 +360,11 @@ server.registerTool('retrieve-rules-context', {
         purpose: zod_1.z
             .enum([
             'general-video',
-            'music-video',
-            'stage-play',
             'story-telling',
             'creative-ad',
             'expert',
             'material-creation',
+            'anim',
             'freeform',
             'custom',
         ])
@@ -372,12 +372,11 @@ server.registerTool('retrieve-rules-context', {
             .describe(`The purpose of the rules context to retrieve.
 
 - general-video 创建通用视频
-- music-video 创建音乐视频
-- stage-play 创建舞台播放视频
 - story-telling 创建故事讲述视频
 - creative-ad 创建创意广告视频
 - expert 以专家模式创建视频，必须用户主动要求才触发
 - material-creation 素材创作模式，必须用户主动要求才触发
+- anim 漫剧创作模式，必须用户主动要求才触发
 - freeform 自由创作模式，必须用户主动要求才触发
 - custom 自定义模式`),
     },
@@ -400,12 +399,11 @@ server.registerTool('retrieve-rules-context', {
         // 当 projectRulesFile 不存在时，设置 checkStoryboardSubtitlesFlag 为 false
         checkStoryboardSubtitlesFlag = false;
         if (purpose !== 'general-video' &&
-            purpose !== 'music-video' &&
-            purpose !== 'stage-play' &&
             purpose !== 'story-telling' &&
             purpose !== 'creative-ad' &&
             purpose !== 'expert' &&
             purpose !== 'material-creation' &&
+            purpose !== 'anim' &&
             purpose !== 'freeform') {
             return createErrorResponse(`Project rules file not found: ${projectRulesFile}`, 'retrieve-rules-context');
         }
@@ -739,7 +737,10 @@ server.registerTool('generate-character-image', {
     title: 'Generate Character Image',
     description: 'Generate a turnaround image or portrait for any character.',
     inputSchema: {
-        type: zod_1.z.enum(['banana', 'seedream']).optional().default('banana'),
+        type: zod_1.z
+            .enum(['banana', 'banana-pro', 'seedream'])
+            .optional()
+            .default('banana'),
         name: zod_1.z.string().describe('The name of the character.'),
         gender: zod_1.z
             .enum(['male', 'female'])
@@ -1069,7 +1070,10 @@ server.registerTool('generate-image', {
     title: 'Generate Image',
     description: `生成图片`,
     inputSchema: {
-        type: zod_1.z.enum(['banana', 'seedream']).optional().default('seedream'),
+        type: zod_1.z
+            .enum(['banana', 'banana-pro', 'seedream'])
+            .optional()
+            .default('seedream'),
         prompt: zod_1.z
             .string()
             .describe('The prompt to generate. 一般要严格对应 storyboard 中当前场景的 start_frame 或 end_frame 中的字段描述'),
@@ -1434,6 +1438,11 @@ server.registerTool('edit-image', {
     description: 'Edit the image.',
     inputSchema: {
         prompt: zod_1.z.string().describe('要编辑图片的中文提示词'),
+        type: zod_1.z
+            .enum(['banana-pro', 'banana', 'seedream'])
+            .optional()
+            .default('seedream')
+            .describe('The type of image model to use.'),
         sourceImageFileName: zod_1.z.string().describe('The source image file name.'),
         saveToFileName: zod_1.z.string().describe('The filename to save.'),
         size: zod_1.z
@@ -1477,7 +1486,7 @@ server.registerTool('edit-image', {
             .default(false)
             .describe('Whether to add watermark to the image.'),
     },
-}, async ({ prompt, sourceImageFileName, saveToFileName, size, watermark }) => {
+}, async ({ prompt, type, sourceImageFileName, saveToFileName, size, watermark, }) => {
     try {
         // 验证session状态
         const currentSession = await validateSession('edit-image');
@@ -1503,8 +1512,9 @@ server.registerTool('edit-image', {
         const base64String = `data:${mimeType};base64,${imageBuffer.toString('base64')}`;
         imageBase64Array.push(base64String);
         const ai = currentSession.ai;
-        const processedPrompt = `转绘参考图片，${prompt}`;
+        const processedPrompt = `转绘我提供的参考图片，${prompt}`;
         const res = await ai.generateImage({
+            type,
             prompt: processedPrompt,
             size,
             watermark,
@@ -2761,7 +2771,9 @@ server.registerTool('get-schema', {
     title: 'Get Storyboard Schema or Draft Content Schema',
     description: 'Get the complete Storyboard or Draft Content JSON Schema definition. Use this schema to validate storyboard.json or draft_content.json files.',
     inputSchema: {
-        type: zod_1.z.enum(['storyboard', 'draft_content']),
+        type: zod_1.z
+            .enum(['storyboard', 'draft_content'])
+            .describe('The type of schema to retrieve. Must be either "storyboard" or "draft_content". 用 type: storyboard 的 schema 生成 storyboard.json；用 type: draft_content 的 schema 生成 draft_content.json'),
     },
 }, async ({ type }) => {
     try {
@@ -2795,34 +2807,6 @@ server.registerTool('get-schema', {
         return createErrorResponse(error, 'get-schema');
     }
 });
-server.registerTool('do-storyboard-optimization', {
-    title: 'Do Storyboard Optimization',
-    description: 'Get storyboard optimization guidelines and action instructions.',
-    inputSchema: {},
-}, async () => {
-    try {
-        // 调用 do-storyboard-optimization 工具时，设置 checkStoryboardFlag 为 true
-        checkStoryboardFlag = true;
-        const guidelinePath = (0, node_path_1.resolve)(__dirname, './prompts/actions/storyboard_optimization.md');
-        const storyboardOptimizationGuidelines = await (0, promises_1.readFile)(guidelinePath, 'utf-8');
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify({
-                        content: {
-                            guideline: storyboardOptimizationGuidelines,
-                            action: '你应当根据guideline优化storyboard.json',
-                        },
-                    }),
-                },
-            ],
-        };
-    }
-    catch (error) {
-        return createErrorResponse(error, 'do-storyboard-optimization');
-    }
-});
 server.registerTool('search-voices', {
     title: 'Search Voices',
     description: 'Search voices from doubao_voices_full based on scenes, emotions, languages, and gender.',
@@ -3401,7 +3385,11 @@ server.registerTool('audio-video-sync', {
     description: 'Generate audio-video-synced video by matching video with audio. 还可以对口型。',
     inputSchema: {
         lipSync: zod_1.z.boolean().default(false),
-        lipSyncType: zod_1.z.enum(['pixv', 'vidu', 'basic', 'lite']).default('pixv'),
+        lipSyncType: zod_1.z
+            .enum(['pixv', 'vidu', 'basic', 'lite'])
+            .optional()
+            .default('pixv'),
+        lipSyncPadAudio: zod_1.z.boolean().optional().default(true),
         videoFileName: zod_1.z
             .string()
             .describe('The video file name in materials directory.'),
@@ -3420,7 +3408,7 @@ server.registerTool('audio-video-sync', {
             .string()
             .describe('The filename to save the audio-video-synced video.'),
     },
-}, async ({ lipSync, lipSyncType, videoFileName, audioFileName, audioInMs, refPhotoFileName, saveToFileName, }, context) => {
+}, async ({ lipSync, lipSyncType, lipSyncPadAudio, videoFileName, audioFileName, audioInMs, refPhotoFileName, saveToFileName, }, context) => {
     try {
         // 验证session状态
         const currentSession = await validateSession('audio-video-sync');
@@ -3492,6 +3480,7 @@ server.registerTool('audio-video-sync', {
             audioUrl,
             audioInMs,
             ref_photo_url: refPhotoUrl,
+            pad_audio: lipSyncPadAudio,
             onProgress: async (metaData) => {
                 console.log('Lip sync progress:', metaData);
                 try {
@@ -3551,6 +3540,11 @@ server.registerTool('generate-video-by-ref', {
         prompt: zod_1.z
             .string()
             .describe('The prompt to generate video with or without reference images.'),
+        rewritePrompt: zod_1.z
+            .boolean()
+            .optional()
+            .default(true)
+            .describe('Whether to rewrite the prompt.'),
         referenceImages: zod_1.z
             .array(zod_1.z.object({
             name: zod_1.z
@@ -3571,7 +3565,12 @@ server.registerTool('generate-video-by-ref', {
             .optional()
             .default(5)
             .describe('The duration of the video in seconds.'),
-        size: zod_1.z.enum(['720x1280', '1280x720']).describe('The size of the video.'),
+        aspectRatio: zod_1.z
+            .enum(['16:9', '9:16'])
+            .describe('The aspect ratio of the video.'),
+        resolution: zod_1.z
+            .enum(['720p', '1080p'])
+            .describe('The resolution of the video.'),
         watermark: zod_1.z
             .boolean()
             .optional()
@@ -3614,7 +3613,7 @@ server.registerTool('generate-video-by-ref', {
             .default(false)
             .describe('Whether to optimize the prompt.'),
     },
-}, async ({ prompt, referenceImages, duration, size, watermark, type, mute, saveToFileName, sceneIndex, storyBoardFile, skipConsistencyCheck, optimizePrompt, }, context) => {
+}, async ({ prompt, rewritePrompt, referenceImages, duration, aspectRatio, resolution, watermark, type, mute, saveToFileName, sceneIndex, storyBoardFile, skipConsistencyCheck, optimizePrompt, }, context) => {
     try {
         // 验证session状态
         const currentSession = await validateSession('generate-video-by-ref');
@@ -3656,6 +3655,9 @@ server.registerTool('generate-video-by-ref', {
                             if (scene.is_continuous === true) {
                                 return createErrorResponse('连续镜头应使用首尾帧，请修改连续镜头设置，或将本场景改为首尾帧方式实现', 'generate-video-by-ref');
                             }
+                            if (scene.video_type !== 'references') {
+                                return createErrorResponse(`场景 ${sceneIndex} 中的 video_type (${scene.video_type}) 未设置为 'references'，不应当使用参考生视频，请使用图生视频 generate-video 方式生成`, 'generate-video-by-ref');
+                            }
                             // 检查 use_video_model 与 type 参数的一致性
                             if (scene.use_video_model &&
                                 type &&
@@ -3704,15 +3706,15 @@ server.registerTool('generate-video-by-ref', {
                             }
                             // 检查 storyBoard.orientation 与 size 参数的一致性
                             if (storyBoard.orientation) {
-                                const isLandscapeSize = size === '1280x720';
-                                const isPortraitSize = size === '720x1280';
+                                const isLandscapeSize = aspectRatio === '16:9';
+                                const isPortraitSize = aspectRatio === '9:16';
                                 if (storyBoard.orientation === 'landscape' &&
                                     !isLandscapeSize) {
-                                    return createErrorResponse(`故事板设定为横屏模式(orientation: landscape)，但视频尺寸 ${size} 为竖屏格式，请使用横屏尺寸 1280x720`, 'generate-video-by-ref');
+                                    return createErrorResponse(`故事板设定为横屏模式(orientation: landscape)，但视频为竖屏格式，请使用横屏尺寸 1280x720`, 'generate-video-by-ref');
                                 }
                                 if (storyBoard.orientation === 'portrait' &&
                                     !isPortraitSize) {
-                                    return createErrorResponse(`故事板设定为竖屏模式(orientation: portrait)，但视频尺寸 ${size} 为横屏格式，请使用竖屏尺寸 720x1280`, 'generate-video-by-ref');
+                                    return createErrorResponse(`故事板设定为竖屏模式(orientation: portrait)，但视频为横屏格式，请使用竖屏尺寸 720x1280`, 'generate-video-by-ref');
                                 }
                             }
                         }
@@ -3781,17 +3783,21 @@ server.registerTool('generate-video-by-ref', {
                 url: imageUrl,
             });
             console.log(`Added reference image URL: ${imageUrl} (name: ${imageRef.name}, type: ${imageRef.type})`);
-            promptPrefix += `参考“${imageRef.name}”（图${referenceImageUrls.length}）${imageRef.type === 'subject' ? '主体形象' : '背景'}\n`;
+            if (rewritePrompt) {
+                promptPrefix += `参考“${imageRef.name}”（图${referenceImageUrls.length}）${imageRef.type === 'subject' ? '主体形象' : '背景'}\n`;
+            }
         }
         if (promptPrefix) {
             promptPrefix += '\n';
         }
+        const finalPrompt = `${promptPrefix}${prompt}`;
         // 调用 referencesToVideo 函数
         const result = await currentSession.ai.referencesToVideo({
-            prompt: `${promptPrefix}${prompt}`,
+            prompt: finalPrompt,
             reference_images: referenceImageUrls, // 使用URL数组而不是base64数组
             duration,
-            size,
+            aspect_ratio: aspectRatio,
+            resolution,
             watermark,
             type,
             mute,
@@ -3803,9 +3809,23 @@ server.registerTool('generate-video-by-ref', {
         if (result.error) {
             return createErrorResponse(result.error, 'generate-video-by-ref');
         }
-        if (!result.url) {
+        if (!result.url && !result.taskUrl) {
             return createErrorResponse('Video generation failed: no video URL returned', 'generate-video-by-ref');
         }
+        else if (result.taskUrl) {
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: JSON.stringify({
+                            success: true,
+                            message: '该视频生成任务正在运行中，它是异步任务，且执行时间较长，你应立即调用工具 wait-for-task-finish 来等待任务结束，如该工具调用超时，你应立即再次重新调用直到任务结束。',
+                            taskUrl: result.taskUrl,
+                        }),
+                    },
+                ],
+            };
+        }
         // 下载生成的视频
         await saveMaterial(currentSession, result.url, validatedFileName);
         // 更新媒体日志
@@ -3831,6 +3851,8 @@ server.registerTool('generate-video-by-ref', {
                         ratio: result.ratio,
                         url: result.url,
                         last_frame_url: result.last_frame_url,
+                        referenceImageUrls,
+                        prompt: finalPrompt,
                     }, null, 2),
                 },
             ],
@@ -3854,6 +3876,10 @@ server.registerTool('extend-video-duration', {
             .max(7)
             .default(3)
             .describe('Duration to extend the video in seconds (1-7).'),
+        resolution: zod_1.z
+            .enum(['720p', '1080p'])
+            .default('720p')
+            .describe('The resolution of the video.'),
         prompt: zod_1.z
             .string()
             .optional()
@@ -3870,7 +3896,7 @@ server.registerTool('extend-video-duration', {
             .string()
             .describe('The filename to save the extended video.'),
     },
-}, async ({ videoFileName, duration, prompt, type = 'turbo', endFrame, saveToFileName, }, context) => {
+}, async ({ videoFileName, duration, resolution, prompt, type = 'turbo', endFrame, saveToFileName, }, context) => {
     try {
         await validateSession('extend-video');
         validateFileName(videoFileName);
@@ -3899,6 +3925,7 @@ server.registerTool('extend-video-duration', {
             video_url: videoUri,
             prompt: finalPrompt,
             duration,
+            resolution,
             end_frame: endFrameUri,
             onProgress: async (metaData) => {
                 sendProgress(context, ++progress, undefined, `Extension progress: ${Math.round(progress * 100)}%`);
@@ -3952,49 +3979,46 @@ server.registerTool('extend-video-duration', {
         return createErrorResponse(error, 'extend-video');
     }
 });
-server.registerTool('generate-video-by-template', {
-    title: 'Generate Video by Template',
-    description: 'Generate a video based on a template. The template must be a valid JSON string.',
+server.registerTool('use-template', {
+    title: 'Use Template',
+    description: 'Find a template that matches the user request, and use it to generate a new material.',
     inputSchema: {
-        purpose: zod_1.z
-            .string()
-            .describe('The prompt to generate the video. 自动根据意图匹配模板'),
-        text_prompts: zod_1.z
-            .array(zod_1.z.string().describe('Text prompt for the template to build video.'))
-            .optional()
-            .describe('Optional text prompts to use in the template.'),
+        user_request: zod_1.z.string().describe('用户请求，根据意图自动匹配模板'),
         materials: zod_1.z
             .array(zod_1.z.string().describe('Material file name in materials directory.'))
             .optional()
             .describe('Optional materials to use in the template.'),
         saveToFileName: zod_1.z
             .string()
-            .describe('The filename to save the generated video.'),
+            .describe('The filename to save the generated material.'),
     },
-}, async ({ purpose, text_prompts, saveToFileName, materials }) => {
+}, async ({ user_request, saveToFileName, materials }) => {
     try {
-        const templates = {
-            '7569583728302817331': '宠物唱歌',
-            '7569605825011367976': '万圣节宠物弹吉他',
-        };
         const currentSession = await validateSession('generate-video-by-template');
-        const validatedFileName = validateFileName(saveToFileName);
         const ai = currentSession.ai;
+        const data = await ai.listTemplates('all');
+        const templates = data.map(item => ({
+            id: item.workflow_id,
+            name: item.details.name,
+            description: item.details.description,
+            trigger: item.details.trigger,
+        }));
+        const validatedFileName = validateFileName(saveToFileName);
         let completion = await ai.getCompletions({
-            model: 'Doubao-Seed-1.6-flash',
+            model: 'Doubao-Seed-1.6',
             messages: [
                 {
                     role: 'system',
-                    content: `你根据用户需求，从以下模板中选择一个匹配的模板，返回模板ID：\n\n${JSON.stringify(templates)}\n\n**约束**：只输出模板ID，不需要其他解释。`,
+                    content: `你根据用户需求，从以下模板中选择一个匹配的模板，返回模板ID：\n\n${JSON.stringify(templates)}\n\n**约束**：只输出模板ID，不需要其他解释，如果没有匹配的模版，输出"无匹配模版"`,
                 },
                 {
                     role: 'user',
-                    content: `用户意图：${purpose}`,
+                    content: user_request,
                 },
             ],
         });
         const templateId = completion.choices[0]?.message?.content.trim();
-        if (!templateId) {
+        if (!templateId || templateId === '无匹配模版') {
             throw new Error('Failed to get template ID');
         }
         const workflowInfo = await ai.getCozeWorkflowInfo(templateId);
@@ -4012,16 +4036,17 @@ server.registerTool('generate-video-by-template', {
                 required: ['parameters'],
             },
         };
-        const prompt = `你根据模板工作流输入 schema 和 prompt、materials 生成一个 JSON 字符串，作为模板工作流的参数。
+        const prompt = `你根据模板工作流描述中的 input_schema、用户需求 prompt 和 materials 生成一个 JSON 对象，作为模板工作流的调用参数。
             
-## **工作流输入 schema**：
-${JSON.stringify(workflowInfo.data.workflow_detail.description)}
+## **工作流描述**：
+${workflowInfo.data.workflow_detail.description}
 
-## **prompt**:
-${text_prompts}
+## **用户需求 prompt**:
+${user_request}
 
 ## **materials**：
 ${JSON.stringify(materialUrls)}`;
+        // console.log(prompt);
         completion = await ai.getCompletions({
             model: 'Doubao-Seed-1.6-flash',
             messages: [
@@ -4043,7 +4068,6 @@ ${JSON.stringify(materialUrls)}`;
         if (!parameters) {
             throw new Error('Failed to get parameters');
         }
-        console.log(parameters);
         const result = await ai.runCozeWorkflow(templateId, JSON.parse(parameters).parameters);
         if (result.url) {
             // 保存到项目材料目录
@@ -4075,7 +4099,7 @@ ${JSON.stringify(materialUrls)}`;
         };
     }
     catch (error) {
-        return createErrorResponse(error, 'generate-video-by-template');
+        return createErrorResponse(error, 'use-template');
     }
 });
 server.registerTool('run-ffmpeg-command', {
@@ -4172,6 +4196,34 @@ server.registerTool('run-ffmpeg-command', {
         return createErrorResponse(error, 'run-ffmpeg-command');
     }
 });
+server.registerTool('do-storyboard-optimization', {
+    title: 'Do Storyboard Optimization',
+    description: 'Get storyboard optimization guidelines and action instructions.',
+    inputSchema: {},
+}, async () => {
+    try {
+        // 调用 do-storyboard-optimization 工具时，设置 checkStoryboardFlag 为 true
+        checkStoryboardFlag = true;
+        const guidelinePath = (0, node_path_1.resolve)(__dirname, './prompts/actions/storyboard_optimization.md');
+        const storyboardOptimizationGuidelines = await (0, promises_1.readFile)(guidelinePath, 'utf-8');
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: JSON.stringify({
+                        content: {
+                            guideline: storyboardOptimizationGuidelines,
+                            action: '你应当根据guideline优化storyboard.json',
+                        },
+                    }),
+                },
+            ],
+        };
+    }
+    catch (error) {
+        return createErrorResponse(error, 'do-storyboard-optimization');
+    }
+});
 server.registerTool('search-context', {
     title: 'Search Context',
     description: 'Search the context.',