cerevox 2.27.1 → 2.28.0

package/dist/mcp/servers/zerocut.js

@@ -58,6 +58,7 @@ const node_child_process_1 = require("node:child_process");
 const node_util_1 = require("node:util");
 const os = __importStar(require("node:os"));
 const mp3_duration_1 = __importDefault(require("mp3-duration"));
+const image_size_1 = __importDefault(require("image-size"));
 const execAsync = (0, node_util_1.promisify)(node_child_process_1.exec);
 // 错误处理工具函数
 const handleError = (error) => {
@@ -1569,6 +1570,7 @@ server.registerTool('edit-image', {
         return createErrorResponse(error, 'edit-image');
     }
 });
+let lastEffect = '';
 server.registerTool('generate-video', {
     title: 'Generate Video',
     description: `生成视频`,
@@ -1607,6 +1609,7 @@ server.registerTool('generate-video', {
             'kling',
             'kling-pro',
             'pixv',
+            'kenburns',
         ])
             .default('lite')
             .describe('除非用户明确提出使用其他模型，否则一律用lite模型'),
@@ -1773,6 +1776,208 @@ server.registerTool('generate-video', {
                 console.error('Failed to optimize prompt:', error);
             }
         }
+        if (type === 'kenburns') {
+            // 实现 getImageSize 函数
+            async function getImageSize(imageUri) {
+                // imageUri 是一个可以 http 访问的图片
+                // eslint-disable-next-line custom/no-fetch-in-src
+                const response = await fetch(imageUri);
+                const arrayBuffer = await response.arrayBuffer();
+                const buffer = Buffer.from(arrayBuffer);
+                const image = (0, image_size_1.default)(buffer);
+                return [image.width, image.height];
+            }
+            const [imageWidth, imageHeight] = await getImageSize(startFrameUri);
+            const kenburnsPrompt = `根据用户描述和配置，生成适合的 kenburns 动画参数。
+
+## 配置信息
+- 视频时长：${duration}秒
+- 分辨率：${resolution}
+- 保存最后一帧：${saveLastFrameAs ? '是' : '否'}
+- 图片宽高：${imageWidth}x${imageHeight}
+- 上一次选择的特效：${lastEffect || '无'}
+
+## camera_motion 可选特效
+- zoom_in
+- zoom_out
+- zoom_in_left_top
+- zoom_out_left_top
+- zoom_in_right_top
+- zoom_out_right_top
+- zoom_in_left_bottom
+- zoom_out_left_bottom
+- zoom_in_right_bottom
+- zoom_out_right_bottom
+- pan_up
+- pan_down
+- pan_left
+- pan_right
+- static
+
+## 特效选用规则
+- 视频时长6秒内：优先选择zoom类型(zoom_in, zoom_out)
+- 视频时长6秒以上：优先选择pan类型(pan_left, pan_right, pan_up, pan_down)
+- 除非用户指定，否则不要主动使用 static 效果
+- 除非用户指定，否则不要和上一次选择的效果重复
+`;
+            const schema = {
+                name: 'kenburns_effect',
+                schema: {
+                    type: 'object',
+                    properties: {
+                        camera_motion: {
+                            type: 'string',
+                            description: 'kenburns 特效类型',
+                            enum: [
+                                'zoom_in',
+                                'zoom_out',
+                                'zoom_in_left_top',
+                                'zoom_out_left_top',
+                                'zoom_in_right_top',
+                                'zoom_out_right_top',
+                                'zoom_in_left_bottom',
+                                'zoom_out_left_bottom',
+                                'zoom_in_right_bottom',
+                                'zoom_out_right_bottom',
+                                'pan_up',
+                                'pan_down',
+                                'pan_left',
+                                'pan_right',
+                                'static',
+                            ],
+                        },
+                        size: {
+                            type: 'string',
+                            description: 'kenburns 视频 宽x高',
+                        },
+                    },
+                    duration: {
+                        type: 'number',
+                        description: 'kenburns 视频 时长',
+                    },
+                    required: ['camera_motion', 'size', 'duration'],
+                },
+            };
+            const analysisPayload = {
+                model: 'Doubao-Seed-1.6-flash',
+                messages: [
+                    {
+                        role: 'system',
+                        content: kenburnsPrompt,
+                    },
+                    {
+                        role: 'user',
+                        content: prompt.trim(),
+                    },
+                ],
+                response_format: {
+                    type: 'json_schema',
+                    json_schema: schema,
+                },
+            };
+            console.log(JSON.stringify(analysisPayload, null, 2));
+            const completion = await ai.getCompletions(analysisPayload);
+            const analysisResult = completion.choices[0]?.message?.content;
+            if (!analysisResult) {
+                throw new Error('Failed to generate kenburns effect');
+            }
+            try {
+                const kenburnsEffect = JSON.parse(analysisResult);
+                const { camera_motion, size, duration } = kenburnsEffect;
+                if (!camera_motion || !size || !duration) {
+                    throw new Error('Invalid kenburns effect parameters');
+                }
+                lastEffect = camera_motion;
+                const files = currentSession.files;
+                const terminal = currentSession.terminal;
+                start_frame = (0, node_path_1.resolve)('/home/user/cerevox-zerocut/projects', terminal.id, 'materials', (0, node_path_1.basename)(start_frame));
+                const saveToPath = `/home/user/cerevox-zerocut/projects/${terminal.id}/materials/${validatedFileName}`;
+                const saveLocalPath = (0, node_path_1.resolve)(projectLocalDir, 'materials', validatedFileName);
+                // 解析尺寸参数
+                const sizeArray = size.split('x');
+                if (sizeArray.length !== 2) {
+                    throw new Error(`Invalid size format: ${size}. Expected format: WIDTHxHEIGHT`);
+                }
+                const [widthStr, heightStr] = sizeArray;
+                const width = Number(widthStr);
+                const height = Number(heightStr);
+                if (isNaN(width) || isNaN(height) || width <= 0 || height <= 0) {
+                    throw new Error(`Invalid dimensions: ${width}x${height}. Both width and height must be positive numbers`);
+                }
+                console.log(`Compiling Ken Burns motion command...`);
+                let command;
+                try {
+                    command = await (0, videokit_1.compileKenBurnsMotion)(start_frame, duration, camera_motion, {
+                        output: saveToPath,
+                        width,
+                        height,
+                    });
+                }
+                catch (compileError) {
+                    console.error('Failed to compile Ken Burns motion command:', compileError);
+                    throw new Error(`Failed to compile Ken Burns motion: ${compileError}`);
+                }
+                console.log(`Executing FFmpeg command: ${command.substring(0, 100)}...`);
+                const res = await terminal.run(command);
+                const result = await res.json();
+                if (result.exitCode !== 0) {
+                    console.error('FFmpeg command failed:', result);
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: JSON.stringify({
+                                    success: false,
+                                    error: 'Ken Burns motion generation failed',
+                                    exitCode: result.exitCode,
+                                    stderr: result.stderr,
+                                    command,
+                                    timestamp: new Date().toISOString(),
+                                }),
+                            },
+                        ],
+                    };
+                }
+                console.log('Ken Burns motion generated successfully, downloading file...');
+                try {
+                    await files.download(saveToPath, saveLocalPath);
+                }
+                catch (downloadError) {
+                    console.warn('Failed to download file to local:', downloadError);
+                    // 继续执行，因为远程文件已生成成功
+                }
+                const resultData = {
+                    success: true,
+                    uri: saveToPath,
+                    durationMs: Math.floor(duration * 1000),
+                    cameraMotion: camera_motion,
+                    imagePath: start_frame,
+                    size,
+                    dimensions: { width, height },
+                    timestamp: new Date().toISOString(),
+                    systemPrompt: kenburnsPrompt,
+                };
+                console.log(`Ken Burns motion completed: ${saveToPath}`);
+                currentSession.track('KenBurns Video Generated', {
+                    durationMs: Math.floor(duration * 1000).toString(),
+                    cameraMotion: camera_motion,
+                    imagePath: start_frame,
+                    size,
+                    dimensions: size,
+                });
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: JSON.stringify(resultData),
+                        },
+                    ],
+                };
+            }
+            catch (error) {
+                throw new Error('Failed to parse kenburns effect parameters');
+            }
+        }
         const res = await ai.framesToVideo({
             prompt: prompt.trim(),
             start_frame: startFrameUri,
@@ -1852,152 +2057,185 @@ server.registerTool('generate-video', {
         return createErrorResponse(error, 'generate-video');
     }
 });
-server.registerTool('generate-video-kenburns', {
-    title: 'Generate Ken Burns Motion Video',
-    description: 'Generate Ken Burns motion video with customizable camera movements.',
-    inputSchema: {
-        image_path: zod_1.z.string().describe('The start frame image path（not URL).'),
-        duration: zod_1.z
-            .number()
-            .min(2)
-            .max(16)
-            .describe('The duration of the video.'),
-        camera_motion: zod_1.z
-            .enum([
-            'zoom_in',
-            'zoom_out',
-            'zoom_in_left_top',
-            'zoom_out_left_top',
-            'zoom_in_right_top',
-            'zoom_out_right_top',
-            'zoom_in_left_bottom',
-            'zoom_out_left_bottom',
-            'zoom_in_right_bottom',
-            'zoom_out_right_bottom',
-            'pan_up',
-            'pan_down',
-            'pan_left',
-            'pan_right',
-            'static',
-        ])
-            .optional()
-            .default('zoom_in')
-            .describe('The camera motion type for Ken Burns effect.'),
-        size: zod_1.z
-            .enum([
-            '1024x1024',
-            '864x1152',
-            '1152x864',
-            '1280x720',
-            '720x1280',
-            '832x1248',
-            '1248x832',
-            '1512x648',
-        ])
-            .describe('The size of the image.'),
-        saveToFileName: zod_1.z.string().describe('The filename to save.'),
-    },
-}, async ({ image_path, duration, camera_motion = 'zoom_in', size, saveToFileName, }) => {
-    try {
-        // 验证session状态
-        const currentSession = await validateSession('generate-video-kenburns');
-        const validatedFileName = validateFileName(saveToFileName);
-        const files = currentSession.files;
-        const terminal = currentSession.terminal;
-        // 如果 image_path 是文件名，需要拼接完整路径
-        if ((0, node_path_1.dirname)(image_path) === '.') {
-            image_path = (0, node_path_1.resolve)('/home/user/cerevox-zerocut/projects', terminal.id, 'materials', image_path);
-        }
-        console.log(`Generating Ken Burns motion: ${image_path} (${duration}s, ${size}, ${camera_motion})`);
-        if (!terminal) {
-            throw new Error('Terminal not available in current session');
-        }
-        const saveToPath = `/home/user/cerevox-zerocut/projects/${terminal.id}/materials/${validatedFileName}`;
-        const saveLocalPath = (0, node_path_1.resolve)(projectLocalDir, 'materials', validatedFileName);
-        // 解析尺寸参数
-        const sizeArray = size.split('x');
-        if (sizeArray.length !== 2) {
-            throw new Error(`Invalid size format: ${size}. Expected format: WIDTHxHEIGHT`);
-        }
-        const [widthStr, heightStr] = sizeArray;
-        const width = Number(widthStr);
-        const height = Number(heightStr);
-        if (isNaN(width) || isNaN(height) || width <= 0 || height <= 0) {
-            throw new Error(`Invalid dimensions: ${width}x${height}. Both width and height must be positive numbers`);
-        }
-        console.log(`Compiling Ken Burns motion command...`);
-        let command;
-        try {
-            command = await (0, videokit_1.compileKenBurnsMotion)(image_path, duration, camera_motion, {
-                output: saveToPath,
-                width,
-                height,
-            });
-        }
-        catch (compileError) {
-            console.error('Failed to compile Ken Burns motion command:', compileError);
-            throw new Error(`Failed to compile Ken Burns motion: ${compileError}`);
-        }
-        console.log(`Executing FFmpeg command: ${command.substring(0, 100)}...`);
-        const res = await terminal.run(command);
-        const result = await res.json();
-        if (result.exitCode !== 0) {
-            console.error('FFmpeg command failed:', result);
-            return {
-                content: [
-                    {
-                        type: 'text',
-                        text: JSON.stringify({
-                            success: false,
-                            error: 'Ken Burns motion generation failed',
-                            exitCode: result.exitCode,
-                            stderr: result.stderr,
-                            command,
-                            timestamp: new Date().toISOString(),
-                        }),
-                    },
-                ],
-            };
-        }
-        console.log('Ken Burns motion generated successfully, downloading file...');
-        try {
-            await files.download(saveToPath, saveLocalPath);
-        }
-        catch (downloadError) {
-            console.warn('Failed to download file to local:', downloadError);
-            // 继续执行，因为远程文件已生成成功
-        }
-        const resultData = {
-            success: true,
-            uri: saveToPath,
-            durationMs: Math.floor(duration * 1000),
-            cameraMotion: camera_motion,
-            imagePath: image_path,
-            size,
-            dimensions: { width, height },
-            timestamp: new Date().toISOString(),
-        };
-        console.log(`Ken Burns motion completed: ${saveToPath}`);
-        currentSession.track('KenBurns Video Generated', {
-            durationMs: Math.floor(duration * 1000).toString(),
-            cameraMotion: camera_motion,
-            imagePath: image_path,
-            size,
-            dimensions: size,
-        });
-        return {
-            content: [
-                {
-                    type: 'text',
-                    text: JSON.stringify(resultData),
-                },
-            ],
-        };
-    }
-    catch (error) {
-        return createErrorResponse(error, 'generate-video-kenburns');
-    }
-});
+// server.registerTool(
+//   'generate-video-kenburns',
+//   {
+//     title: 'Generate Ken Burns Motion Video',
+//     description:
+//       'Generate Ken Burns motion video with customizable camera movements.',
+//     inputSchema: {
+//       image_path: z.string().describe('The start frame image path（not URL).'),
+//       duration: z
+//         .number()
+//         .min(2)
+//         .max(16)
+//         .describe('The duration of the video.'),
+//       camera_motion: z
+//         .enum([
+//           'zoom_in',
+//           'zoom_out',
+//           'zoom_in_left_top',
+//           'zoom_out_left_top',
+//           'zoom_in_right_top',
+//           'zoom_out_right_top',
+//           'zoom_in_left_bottom',
+//           'zoom_out_left_bottom',
+//           'zoom_in_right_bottom',
+//           'zoom_out_right_bottom',
+//           'pan_up',
+//           'pan_down',
+//           'pan_left',
+//           'pan_right',
+//           'static',
+//         ])
+//         .optional()
+//         .default('zoom_in')
+//         .describe('The camera motion type for Ken Burns effect.'),
+//       size: z
+//         .enum([
+//           '1024x1024',
+//           '864x1152',
+//           '1152x864',
+//           '1280x720',
+//           '720x1280',
+//           '832x1248',
+//           '1248x832',
+//           '1512x648',
+//         ])
+//         .describe('The size of the image.'),
+//       saveToFileName: z.string().describe('The filename to save.'),
+//     },
+//   },
+//   async ({
+//     image_path,
+//     duration,
+//     camera_motion = 'zoom_in',
+//     size,
+//     saveToFileName,
+//   }) => {
+//     try {
+//       // 验证session状态
+//       const currentSession = await validateSession('generate-video-kenburns');
+//       const validatedFileName = validateFileName(saveToFileName);
+//       const files = currentSession.files;
+//       const terminal = currentSession.terminal;
+//       // 如果 image_path 是文件名，需要拼接完整路径
+//       if (dirname(image_path) === '.') {
+//         image_path = resolve(
+//           '/home/user/cerevox-zerocut/projects',
+//           terminal.id,
+//           'materials',
+//           image_path
+//         );
+//       }
+//       console.log(
+//         `Generating Ken Burns motion: ${image_path} (${duration}s, ${size}, ${camera_motion})`
+//       );
+//       if (!terminal) {
+//         throw new Error('Terminal not available in current session');
+//       }
+//       const saveToPath = `/home/user/cerevox-zerocut/projects/${terminal.id}/materials/${validatedFileName}`;
+//       const saveLocalPath = resolve(
+//         projectLocalDir,
+//         'materials',
+//         validatedFileName
+//       );
+//       // 解析尺寸参数
+//       const sizeArray = size.split('x');
+//       if (sizeArray.length !== 2) {
+//         throw new Error(
+//           `Invalid size format: ${size}. Expected format: WIDTHxHEIGHT`
+//         );
+//       }
+//       const [widthStr, heightStr] = sizeArray;
+//       const width = Number(widthStr);
+//       const height = Number(heightStr);
+//       if (isNaN(width) || isNaN(height) || width <= 0 || height <= 0) {
+//         throw new Error(
+//           `Invalid dimensions: ${width}x${height}. Both width and height must be positive numbers`
+//         );
+//       }
+//       console.log(`Compiling Ken Burns motion command...`);
+//       let command: string;
+//       try {
+//         command = await compileKenBurnsMotion(
+//           image_path,
+//           duration,
+//           camera_motion as CameraMotion,
+//           {
+//             output: saveToPath,
+//             width,
+//             height,
+//           }
+//         );
+//       } catch (compileError) {
+//         console.error(
+//           'Failed to compile Ken Burns motion command:',
+//           compileError
+//         );
+//         throw new Error(`Failed to compile Ken Burns motion: ${compileError}`);
+//       }
+//       console.log(`Executing FFmpeg command: ${command.substring(0, 100)}...`);
+//       const res = await terminal.run(command);
+//       const result = await res.json();
+//       if (result.exitCode !== 0) {
+//         console.error('FFmpeg command failed:', result);
+//         return {
+//           content: [
+//             {
+//               type: 'text' as const,
+//               text: JSON.stringify({
+//                 success: false,
+//                 error: 'Ken Burns motion generation failed',
+//                 exitCode: result.exitCode,
+//                 stderr: result.stderr,
+//                 command,
+//                 timestamp: new Date().toISOString(),
+//               }),
+//             },
+//           ],
+//         };
+//       }
+//       console.log(
+//         'Ken Burns motion generated successfully, downloading file...'
+//       );
+//       try {
+//         await files.download(saveToPath, saveLocalPath);
+//       } catch (downloadError) {
+//         console.warn('Failed to download file to local:', downloadError);
+//         // 继续执行，因为远程文件已生成成功
+//       }
+//       const resultData = {
+//         success: true,
+//         uri: saveToPath,
+//         durationMs: Math.floor(duration * 1000),
+//         cameraMotion: camera_motion,
+//         imagePath: image_path,
+//         size,
+//         dimensions: { width, height },
+//         timestamp: new Date().toISOString(),
+//       };
+//       console.log(`Ken Burns motion completed: ${saveToPath}`);
+//       currentSession.track('KenBurns Video Generated', {
+//         durationMs: Math.floor(duration * 1000).toString(),
+//         cameraMotion: camera_motion,
+//         imagePath: image_path,
+//         size,
+//         dimensions: size,
+//       });
+//       return {
+//         content: [
+//           {
+//             type: 'text' as const,
+//             text: JSON.stringify(resultData),
+//           },
+//         ],
+//       };
+//     } catch (error) {
+//       return createErrorResponse(error, 'generate-video-kenburns');
+//     }
+//   }
+// );
 server.registerTool('generate-sound-effect', {
     title: 'Generate Sound Effect',
     description: 'Generate sound effect with text prompt.',
@@ -3361,17 +3599,22 @@ server.registerTool('lip-sync', {
     title: 'Lip Sync',
     description: 'Generate lip-synced video by matching video with audio.',
     inputSchema: {
+        type: zod_1.z.enum(['vidu', 'basic', 'lite']).default('vidu'),
         videoFileName: zod_1.z
             .string()
             .describe('The video file name in materials directory.'),
         audioFileName: zod_1.z
             .string()
             .describe('The audio file name in materials directory.'),
+        refPhotoFileName: zod_1.z
+            .string()
+            .optional()
+            .describe('The reference photo face for lip sync.'),
         saveToFileName: zod_1.z
             .string()
             .describe('The filename to save the lip-synced video.'),
     },
-}, async ({ videoFileName, audioFileName, saveToFileName }, context) => {
+}, async ({ type, videoFileName, audioFileName, refPhotoFileName, saveToFileName }, context) => {
     try {
         // 验证session状态
         const currentSession = await validateSession('lip-sync');
@@ -3386,13 +3629,19 @@ server.registerTool('lip-sync', {
         const videoUrl = getMaterialUri(currentSession, videoFileName);
         const audioUrl = getMaterialUri(currentSession, audioFileName // processedAudioFileName
         );
+        const refPhotoUrl = refPhotoFileName
+            ? getMaterialUri(currentSession, refPhotoFileName)
+            : undefined;
         console.log(`Video URL: ${videoUrl}`);
         console.log(`Audio URL: ${audioUrl}`);
+        console.log(`Ref Photo URL: ${refPhotoUrl}`);
         // 调用AI的lipSync方法，使用处理后的音频
         let progress = 0;
         const result = await currentSession.ai.lipSync({
+            type,
             videoUrl,
             audioUrl,
+            ref_photo_url: refPhotoUrl,
             onProgress: async (metaData) => {
                 console.log('Lip sync progress:', metaData);
                 try {