cerevox 3.13.2 → 4.0.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/ai.d.ts +2 -0
- package/dist/core/ai.d.ts.map +1 -1
- package/dist/core/ai.js +11 -1
- package/dist/core/ai.js.map +1 -1
- package/dist/mcp/servers/prompts/outlines-backup.md +41 -14
- package/dist/mcp/servers/prompts/rules/core-rules.md +111 -0
- package/dist/mcp/servers/prompts/skills//344/270/200/351/224/256/346/210/220/347/211/207.md +16 -25
- package/dist/mcp/servers/prompts/skills//345/210/206/351/225/234/345/270/210.md +12 -7
- package/dist/mcp/servers/prompts/skills//347/264/240/346/235/220/345/210/233/344/275/234/357/274/210/351/200/232/347/224/250/357/274/211.md +6 -0
- package/dist/mcp/servers/prompts/zerocut-core-web-old.md +95 -0
- package/dist/mcp/servers/prompts/zerocut-core.md +39 -87
- package/dist/mcp/servers/zerocut.d.ts.map +1 -1
- package/dist/mcp/servers/zerocut.js +80 -1000
- package/dist/mcp/servers/zerocut.js.map +1 -1
- package/package.json +1 -1
- package/dist/mcp/servers/prompts/rules/anim.md +0 -72
- package/dist/mcp/servers/prompts/rules/creative-ad.md +0 -125
- package/dist/mcp/servers/prompts/rules/expert.md +0 -254
- package/dist/mcp/servers/prompts/rules/freeform.md +0 -188
- package/dist/mcp/servers/prompts/rules/story-telling.md +0 -91
|
@@ -54,6 +54,7 @@ const node_fs_1 = require("node:fs");
|
|
|
54
54
|
const mp3_duration_1 = __importDefault(require("mp3-duration"));
|
|
55
55
|
const image_size_1 = __importDefault(require("image-size"));
|
|
56
56
|
const seed_1 = require("../../utils/seed");
|
|
57
|
+
const uuid_1 = require("uuid");
|
|
57
58
|
function createErrorResponse(error, operation, details) {
|
|
58
59
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
59
60
|
console.error(`[${operation}] Error:`, error);
|
|
@@ -83,9 +84,13 @@ async function validateSession(operation, checkRules = true) {
|
|
|
83
84
|
throw new Error(`Session not initialized. Please call 'project-open' first before using ${operation}.`);
|
|
84
85
|
}
|
|
85
86
|
const projectRulesFile = (0, node_path_1.resolve)(projectLocalDir, '.trae', 'rules', `project_rules.md`);
|
|
87
|
+
const skillContextFile = (0, node_path_1.resolve)(projectLocalDir, '.trae', 'rules', `skill_context.md`);
|
|
86
88
|
if (checkRules && !(0, node_fs_1.existsSync)(projectRulesFile)) {
|
|
87
89
|
throw new Error(`Project rules file not found: ${projectRulesFile}. Please call 'retrieve-rules-context' first.`);
|
|
88
90
|
}
|
|
91
|
+
if (checkRules && !(0, node_fs_1.existsSync)(skillContextFile)) {
|
|
92
|
+
throw new Error(`Skill context file not found: ${skillContextFile}. 请先审视当前技能列表 '.trae/skills',确保已选择了正确的技能,将技能信息记录到 '.trae/rules/skill_context.md' 中`);
|
|
93
|
+
}
|
|
89
94
|
return session;
|
|
90
95
|
}
|
|
91
96
|
// 文件名验证
|
|
@@ -136,6 +141,8 @@ async function saveMaterial(session, url, saveToFileName) {
|
|
|
136
141
|
const terminal = session.terminal;
|
|
137
142
|
const saveToPath = `/home/user/cerevox-zerocut/projects/${terminal.id}/materials/${saveToFileName}`;
|
|
138
143
|
const saveLocalPath = (0, node_path_1.resolve)(projectLocalDir, 'materials', saveToFileName);
|
|
144
|
+
// 确保目录存在
|
|
145
|
+
await (0, promises_1.mkdir)((0, node_path_1.dirname)(saveLocalPath), { recursive: true });
|
|
139
146
|
// 先下载到本地,再上传 sandbox,比直接 sandbox 更好,也以免下载超时
|
|
140
147
|
// 通过 fetch 下载到本地
|
|
141
148
|
const res = await fetch(url);
|
|
@@ -257,7 +264,7 @@ async function filterMaterialsForUpload(materials, projectLocalDir) {
|
|
|
257
264
|
regularFiles.push(material);
|
|
258
265
|
}
|
|
259
266
|
}
|
|
260
|
-
//
|
|
267
|
+
// 处理分镜文件(scXX_ 前缀)
|
|
261
268
|
if (sceneFiles.length > 0 && mediaLogs.length > 0) {
|
|
262
269
|
// 按前缀和扩展名分组
|
|
263
270
|
const groupedFiles = new Map();
|
|
@@ -305,7 +312,7 @@ async function filterMaterialsForUpload(materials, projectLocalDir) {
|
|
|
305
312
|
}
|
|
306
313
|
}
|
|
307
314
|
else {
|
|
308
|
-
// 如果没有 media_logs.json
|
|
315
|
+
// 如果没有 media_logs.json 或没有分镜文件,全部上传
|
|
309
316
|
filesToUpload.push(...sceneFiles);
|
|
310
317
|
}
|
|
311
318
|
// 处理普通文件(全量上传)
|
|
@@ -352,19 +359,14 @@ server.registerPrompt('zerocut-guideline', {
|
|
|
352
359
|
};
|
|
353
360
|
}
|
|
354
361
|
catch (error) {
|
|
355
|
-
console.error('Failed to load zerocut-guideline prompt:', error);
|
|
356
|
-
throw new Error(`Failed to load zerocut-guideline prompt: ${error}`);
|
|
362
|
+
console.error('Failed to load zerocut-guideline-web prompt:', error);
|
|
363
|
+
throw new Error(`Failed to load zerocut-guideline-web prompt: ${error}`);
|
|
357
364
|
}
|
|
358
365
|
});
|
|
359
366
|
server.registerTool('project-open', {
|
|
360
367
|
title: 'Open Project',
|
|
361
368
|
description: 'Launch a new Cerevox session with a Chromium browser instance and open a new project context. Supports smart file filtering to optimize upload performance.',
|
|
362
369
|
inputSchema: {
|
|
363
|
-
localDir: zod_1.z
|
|
364
|
-
.string()
|
|
365
|
-
.optional()
|
|
366
|
-
.default('.')
|
|
367
|
-
.describe('The path of the file to upload.'),
|
|
368
370
|
tosFiles: zod_1.z
|
|
369
371
|
.array(zod_1.z.string())
|
|
370
372
|
.optional()
|
|
@@ -376,7 +378,7 @@ server.registerTool('project-open', {
|
|
|
376
378
|
.default(false)
|
|
377
379
|
.describe('Whether to upload all files without filtering. If true, skips the smart filtering logic.'),
|
|
378
380
|
},
|
|
379
|
-
}, async ({
|
|
381
|
+
}, async ({ uploadAllFiles, tosFiles }, context) => {
|
|
380
382
|
try {
|
|
381
383
|
if (closeSessionTimerId) {
|
|
382
384
|
clearTimeout(closeSessionTimerId);
|
|
@@ -422,7 +424,13 @@ server.registerTool('project-open', {
|
|
|
422
424
|
}
|
|
423
425
|
console.log('Initializing project...');
|
|
424
426
|
const workDir = await initProject(session);
|
|
425
|
-
|
|
427
|
+
if (!process.env.ZEROCUT_PROJECT_CWD &&
|
|
428
|
+
!process.env.ZEROCUT_WORKSPACE_DIR) {
|
|
429
|
+
throw new Error('ZEROCUT_WORKSPACE_DIR environment variable is required');
|
|
430
|
+
}
|
|
431
|
+
projectLocalDir = process.env.ZEROCUT_PROJECT_CWD
|
|
432
|
+
? (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD, '.')
|
|
433
|
+
: (0, node_path_1.resolve)(process.env.ZEROCUT_WORKSPACE_DIR, (0, uuid_1.v4)());
|
|
426
434
|
const syncDir = (0, node_path_1.resolve)(projectLocalDir, 'materials');
|
|
427
435
|
try {
|
|
428
436
|
await (0, promises_1.mkdir)(syncDir, { recursive: true });
|
|
@@ -435,25 +443,23 @@ server.registerTool('project-open', {
|
|
|
435
443
|
// 文件过滤逻辑
|
|
436
444
|
let filesToUpload = [];
|
|
437
445
|
let skippedFiles = [];
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
skippedFiles = filterResult.skippedFiles;
|
|
456
|
-
}
|
|
446
|
+
try {
|
|
447
|
+
materials = await listFiles(syncDir);
|
|
448
|
+
}
|
|
449
|
+
catch (listError) {
|
|
450
|
+
console.warn('Failed to list materials:', listError);
|
|
451
|
+
materials = [];
|
|
452
|
+
}
|
|
453
|
+
if (uploadAllFiles) {
|
|
454
|
+
// 如果 uploadAllFiles 为 true,跳过智能过滤,上传所有文件
|
|
455
|
+
filesToUpload = materials;
|
|
456
|
+
skippedFiles = [];
|
|
457
|
+
}
|
|
458
|
+
else {
|
|
459
|
+
// 智能文件过滤逻辑
|
|
460
|
+
const filterResult = await filterMaterialsForUpload(materials, projectLocalDir);
|
|
461
|
+
filesToUpload = filterResult.filesToUpload;
|
|
462
|
+
skippedFiles = filterResult.skippedFiles;
|
|
457
463
|
}
|
|
458
464
|
const files = session.files;
|
|
459
465
|
let progress = 0;
|
|
@@ -567,83 +573,22 @@ server.registerTool('retrieve-rules-context', {
|
|
|
567
573
|
}, async ({ request }) => {
|
|
568
574
|
const currentSession = await validateSession('retrieve-rules-context', false);
|
|
569
575
|
const projectRulesFile = (0, node_path_1.resolve)(projectLocalDir, '.trae', 'rules', `project_rules.md`);
|
|
570
|
-
// const customRulesFile = resolve(
|
|
571
|
-
// projectLocalDir,
|
|
572
|
-
// '.trae',
|
|
573
|
-
// 'rules',
|
|
574
|
-
// `custom_rules.md`
|
|
575
|
-
// );
|
|
576
576
|
const skillsIndexFile = (0, node_path_1.resolve)(projectLocalDir, '.trae', 'skills', `skills_index.md`);
|
|
577
577
|
let promptContent = '';
|
|
578
578
|
if ((0, node_fs_1.existsSync)(projectRulesFile)) {
|
|
579
579
|
promptContent = await (0, promises_1.readFile)(projectRulesFile, 'utf-8');
|
|
580
|
-
// if (existsSync(customRulesFile)) {
|
|
581
|
-
// promptContent +=
|
|
582
|
-
// '\n\n---\n\n' + (await readFile(customRulesFile, 'utf-8'));
|
|
583
|
-
// }
|
|
584
580
|
}
|
|
585
581
|
else {
|
|
586
582
|
// 当 projectRulesFile 不存在时,设置 checkStoryboardFlag 为 false
|
|
587
583
|
checkStoryboardFlag = false;
|
|
588
584
|
// 当 projectRulesFile 不存在时,设置 checkAudioVideoDurationFlag 为 false
|
|
589
585
|
checkAudioVideoDurationFlag = false;
|
|
590
|
-
// 当 projectRulesFile 不存在时,设置 checkStoryboardSubtitlesFlag 为 false
|
|
591
|
-
// checkStoryboardSubtitlesFlag = false;
|
|
592
586
|
}
|
|
593
587
|
try {
|
|
594
588
|
const ai = currentSession.ai;
|
|
595
589
|
const { data: rules } = await ai.listContextRules();
|
|
596
590
|
if (!promptContent) {
|
|
597
|
-
const
|
|
598
|
-
.filter((rule) => !rule.name.startsWith('skill-'))
|
|
599
|
-
.map((rule) => ({
|
|
600
|
-
name: rule.name,
|
|
601
|
-
trigger: rule.trigger,
|
|
602
|
-
}));
|
|
603
|
-
const chooseRulePrompt = `当前ZeroCut版本:${constants_1.VERSION}
|
|
604
|
-
|
|
605
|
-
请分析用户具体指令,匹配可用规则中的trigger,选择最合适的规则。
|
|
606
|
-
|
|
607
|
-
## 可用规则:
|
|
608
|
-
${JSON.stringify(rulesList, null, 2)}
|
|
609
|
-
|
|
610
|
-
请返回一个 JSON 字符串,包含用户意图对应的规则名称。格式为:{"rule_name": "规则名称"}
|
|
611
|
-
`;
|
|
612
|
-
const schema = {
|
|
613
|
-
name: 'choose_rule',
|
|
614
|
-
schema: {
|
|
615
|
-
type: 'object',
|
|
616
|
-
properties: {
|
|
617
|
-
rule_name: {
|
|
618
|
-
type: 'string',
|
|
619
|
-
description: '用户意图对应的规则名称',
|
|
620
|
-
},
|
|
621
|
-
},
|
|
622
|
-
required: ['rule_name'],
|
|
623
|
-
},
|
|
624
|
-
};
|
|
625
|
-
const completion = await ai.getCompletions({
|
|
626
|
-
model: 'Doubao-Seed-1.6-flash',
|
|
627
|
-
messages: [
|
|
628
|
-
{
|
|
629
|
-
role: 'system',
|
|
630
|
-
content: chooseRulePrompt,
|
|
631
|
-
},
|
|
632
|
-
{
|
|
633
|
-
role: 'user',
|
|
634
|
-
content: request,
|
|
635
|
-
},
|
|
636
|
-
],
|
|
637
|
-
response_format: {
|
|
638
|
-
type: 'json_schema',
|
|
639
|
-
json_schema: schema,
|
|
640
|
-
},
|
|
641
|
-
});
|
|
642
|
-
const ruleName = JSON.parse(completion.choices[0].message.content).rule_name;
|
|
643
|
-
let rule = rules.find((rule) => rule.name === ruleName);
|
|
644
|
-
if (!rule) {
|
|
645
|
-
rule = rules.find((rule) => rule.name === '通用视频助手');
|
|
646
|
-
}
|
|
591
|
+
const rule = rules.find((rule) => rule.name === 'core-rules');
|
|
647
592
|
promptContent = rule.prompt;
|
|
648
593
|
// 确保目录存在
|
|
649
594
|
await (0, promises_1.mkdir)((0, node_path_1.dirname)(projectRulesFile), { recursive: true });
|
|
@@ -1077,26 +1022,17 @@ server.registerTool('generate-image', {
|
|
|
1077
1022
|
.default('seedream-pro'),
|
|
1078
1023
|
prompt: zod_1.z
|
|
1079
1024
|
.string()
|
|
1080
|
-
.describe('The prompt to generate. 一般要严格对应 storyboard
|
|
1025
|
+
.describe('The prompt to generate. 一般要严格对应 storyboard 中当前分镜的 start_frame 或 end_frame 中的字段描述,如果是生成线稿,则 type 使用 line-sketch,如果是生成故事板印样或人物镜头宫格,则 type 使用 shot-grid'),
|
|
1081
1026
|
sceneIndex: zod_1.z
|
|
1082
1027
|
.number()
|
|
1083
1028
|
.min(1)
|
|
1084
1029
|
.optional()
|
|
1085
|
-
.describe('
|
|
1030
|
+
.describe('分镜索引,从1开始的下标,如果非分镜对应素材,则可不传,分镜素材必传'),
|
|
1086
1031
|
storyBoardFile: zod_1.z
|
|
1087
1032
|
.string()
|
|
1088
1033
|
.optional()
|
|
1089
1034
|
.default('storyboard.json')
|
|
1090
1035
|
.describe('故事板文件路径'),
|
|
1091
|
-
skipConsistencyCheck: zod_1.z
|
|
1092
|
-
.boolean()
|
|
1093
|
-
.optional()
|
|
1094
|
-
.default(false)
|
|
1095
|
-
.describe('是否跳过一致性检查,默认为false(即默认进行一致性检查)'),
|
|
1096
|
-
skipCheckWithSceneReason: zod_1.z
|
|
1097
|
-
.string()
|
|
1098
|
-
.optional()
|
|
1099
|
-
.describe('跳过校验的理由,如果skipConsistencyCheck设为true,必须要传这个参数'),
|
|
1100
1036
|
size: zod_1.z
|
|
1101
1037
|
.enum([
|
|
1102
1038
|
'1024x1024',
|
|
@@ -1181,22 +1117,15 @@ server.registerTool('generate-image', {
|
|
|
1181
1117
|
\`\`\`
|
|
1182
1118
|
`),
|
|
1183
1119
|
},
|
|
1184
|
-
}, async ({ type = 'seedream', prompt, sceneIndex, storyBoardFile = 'storyboard.json',
|
|
1120
|
+
}, async ({ type = 'seedream', prompt, sceneIndex, storyBoardFile = 'storyboard.json', size = '720x1280', imageCount = 1, saveToFileNames, watermark, referenceImages, }, context) => {
|
|
1185
1121
|
try {
|
|
1186
1122
|
// 验证session状态
|
|
1187
1123
|
const currentSession = await validateSession('generate-image');
|
|
1188
1124
|
const storyBoardPath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, storyBoardFile);
|
|
1189
1125
|
const outlineSheetImagePath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, 'materials', 'outline_sheet.png');
|
|
1190
1126
|
const hasOutlineSheet = (0, node_fs_1.existsSync)(outlineSheetImagePath);
|
|
1191
|
-
//
|
|
1192
|
-
if (
|
|
1193
|
-
!checkStoryboardFlag &&
|
|
1194
|
-
(0, node_fs_1.existsSync)(storyBoardPath)) {
|
|
1195
|
-
checkStoryboardFlag = true;
|
|
1196
|
-
return createErrorResponse('必须先审查生成的 storyboard.json 内容,确保每个场景中的stage_atmosphere内容按照规则被正确融合到start_frame和video_prompt中,不得遗漏,检查完成后先汇报,如果有问题,应当先修改 storyboard.json 内容,然后再调用 generate-image 生成图片。注意修改 storyboard 内容时,仅修改相应字段的字符串值,不要破坏JSON格式!', 'generate-image');
|
|
1197
|
-
}
|
|
1198
|
-
// 校验 prompt 与 storyboard.json 中场景设定的一致性
|
|
1199
|
-
if (sceneIndex && !skipConsistencyCheck) {
|
|
1127
|
+
// 校验 prompt 与 storyboard.json 中分镜设定的一致性
|
|
1128
|
+
if (sceneIndex) {
|
|
1200
1129
|
try {
|
|
1201
1130
|
if ((0, node_fs_1.existsSync)(storyBoardPath)) {
|
|
1202
1131
|
const storyBoardContent = await (0, promises_1.readFile)(storyBoardPath, 'utf8');
|
|
@@ -1215,7 +1144,7 @@ server.registerTool('generate-image', {
|
|
|
1215
1144
|
const endFrame = scene.end_frame;
|
|
1216
1145
|
// 检查 prompt 是否严格等于 start_frame 或 end_frame
|
|
1217
1146
|
if (prompt !== startFrame && prompt !== endFrame) {
|
|
1218
|
-
return createErrorResponse('图片提示词必须严格遵照storyboard
|
|
1147
|
+
return createErrorResponse('图片提示词必须严格遵照storyboard的设定', 'generate-image');
|
|
1219
1148
|
}
|
|
1220
1149
|
if (hasOutlineSheet &&
|
|
1221
1150
|
(!referenceImages ||
|
|
@@ -1283,73 +1212,6 @@ server.registerTool('generate-image', {
|
|
|
1283
1212
|
// 检查并替换英文单引号包裹的中文内容为中文双引号
|
|
1284
1213
|
// 这样才能让 seedream 生成更好的中文文字
|
|
1285
1214
|
let processedPrompt = prompt.replace(/'([^']*[\u4e00-\u9fff][^']*)'/g, '“$1”');
|
|
1286
|
-
try {
|
|
1287
|
-
const ai = currentSession.ai;
|
|
1288
|
-
const promptOptimizer = await (0, promises_1.readFile)((0, node_path_1.resolve)(__dirname, './prompts/image-prompt-optimizer.md'), 'utf8');
|
|
1289
|
-
if (!hasOutlineSheet) {
|
|
1290
|
-
const schema = {
|
|
1291
|
-
name: 'optimize_image_prompt',
|
|
1292
|
-
schema: {
|
|
1293
|
-
type: 'object',
|
|
1294
|
-
properties: {
|
|
1295
|
-
prompt_optimized: {
|
|
1296
|
-
type: 'string',
|
|
1297
|
-
description: '优化后的提示词',
|
|
1298
|
-
},
|
|
1299
|
-
metaphor_modifiers: {
|
|
1300
|
-
type: 'array',
|
|
1301
|
-
description: '从 prompt_optimized 中抽取的所有比喻修饰词(字符串数组)',
|
|
1302
|
-
items: {
|
|
1303
|
-
type: 'string',
|
|
1304
|
-
description: '比喻性修饰词,例如 “如羽毛般轻盈”、“像晨雾一样柔和”',
|
|
1305
|
-
},
|
|
1306
|
-
},
|
|
1307
|
-
},
|
|
1308
|
-
required: ['prompt_optimized', 'metaphor_modifiers'],
|
|
1309
|
-
},
|
|
1310
|
-
};
|
|
1311
|
-
const completion = await ai.getCompletions({
|
|
1312
|
-
model: 'Doubao-Seed-1.6',
|
|
1313
|
-
messages: [
|
|
1314
|
-
{
|
|
1315
|
-
role: 'system',
|
|
1316
|
-
content: promptOptimizer,
|
|
1317
|
-
},
|
|
1318
|
-
{
|
|
1319
|
-
role: 'user',
|
|
1320
|
-
content: `## 用户指令
|
|
1321
|
-
|
|
1322
|
-
${processedPrompt.trim()}
|
|
1323
|
-
|
|
1324
|
-
## 参考图
|
|
1325
|
-
|
|
1326
|
-
${referenceImages?.map((ref, index) => `图${index + 1}:${ref.image}`).join('\n') || '无'}`,
|
|
1327
|
-
},
|
|
1328
|
-
],
|
|
1329
|
-
response_format: {
|
|
1330
|
-
type: 'json_schema',
|
|
1331
|
-
json_schema: schema,
|
|
1332
|
-
},
|
|
1333
|
-
});
|
|
1334
|
-
const optimizedPrompt = completion.choices[0]?.message?.content.trim();
|
|
1335
|
-
if (optimizedPrompt) {
|
|
1336
|
-
try {
|
|
1337
|
-
const { prompt_optimized, metaphor_modifiers } = JSON.parse(optimizedPrompt);
|
|
1338
|
-
processedPrompt = `${prompt_optimized}`;
|
|
1339
|
-
if (metaphor_modifiers?.length) {
|
|
1340
|
-
processedPrompt += `\n\n注意:下面这些是形象比喻,并不是输出内容。\n${metaphor_modifiers}`;
|
|
1341
|
-
}
|
|
1342
|
-
}
|
|
1343
|
-
catch (ex) {
|
|
1344
|
-
console.error('Failed to parse optimized prompt:', ex);
|
|
1345
|
-
processedPrompt = optimizedPrompt;
|
|
1346
|
-
}
|
|
1347
|
-
}
|
|
1348
|
-
}
|
|
1349
|
-
}
|
|
1350
|
-
catch (error) {
|
|
1351
|
-
console.error('Failed to optimize prompt:', error);
|
|
1352
|
-
}
|
|
1353
1215
|
if (imageCount > 1) {
|
|
1354
1216
|
processedPrompt = `请生成${imageCount}张${size}大小的关联图片,每张图分别生成,不要拼接为大图 ${processedPrompt}`;
|
|
1355
1217
|
}
|
|
@@ -1422,7 +1284,7 @@ server.registerTool('generate-image', {
|
|
|
1422
1284
|
if (objectPrefix.length > 0) {
|
|
1423
1285
|
processedPrompt = `${objectPrefix.join('\n')}
|
|
1424
1286
|
|
|
1425
|
-
${turnaroundMessage}
|
|
1287
|
+
${turnaroundMessage}参考以上图片,执行以下绘图指令:
|
|
1426
1288
|
${processedPrompt}`.trim();
|
|
1427
1289
|
}
|
|
1428
1290
|
}
|
|
@@ -1459,7 +1321,7 @@ ${processedPrompt}`.trim();
|
|
|
1459
1321
|
];
|
|
1460
1322
|
}
|
|
1461
1323
|
else {
|
|
1462
|
-
//
|
|
1324
|
+
// 多图
|
|
1463
1325
|
uris = await Promise.all(res.urls.map((url, i) => {
|
|
1464
1326
|
if (!url)
|
|
1465
1327
|
return '';
|
|
@@ -1724,14 +1586,8 @@ server.registerTool('generate-short-video-outlines', {
|
|
|
1724
1586
|
voice_type,
|
|
1725
1587
|
scenes: scenes.map((scene) => {
|
|
1726
1588
|
let video_prompt = scene.video_prompt;
|
|
1727
|
-
if (video_prompt.includes('画外音')
|
|
1728
|
-
video_prompt.
|
|
1729
|
-
if (voiceover_tone) {
|
|
1730
|
-
video_prompt = video_prompt.replace(/画外音[::]\s*“([^”]*)”/g, `画外音(${voiceover_tone})镜头内所有角色都不言语,从远处传来广播声<广播开始>$1——</广播结束>`);
|
|
1731
|
-
}
|
|
1732
|
-
else {
|
|
1733
|
-
video_prompt = video_prompt.replace(/画外音[::]\s*“([^”]*)”/g, `镜头内所有角色都不言语,从远处传来广播声<广播开始>$1——</广播结束>`);
|
|
1734
|
-
}
|
|
1589
|
+
if (voiceover_tone && video_prompt.includes('画外音')) {
|
|
1590
|
+
video_prompt = video_prompt.replace(/画外音[::]/g, `画外音(${voiceover_tone}):`);
|
|
1735
1591
|
}
|
|
1736
1592
|
return {
|
|
1737
1593
|
...scene,
|
|
@@ -2083,722 +1939,22 @@ server.registerTool('text-to-speech', {
|
|
|
2083
1939
|
return createErrorResponse(error, 'text-to-speech');
|
|
2084
1940
|
}
|
|
2085
1941
|
});
|
|
2086
|
-
// server.registerTool(
|
|
2087
|
-
// 'generate-scene-tts',
|
|
2088
|
-
// {
|
|
2089
|
-
// title: 'Generate Scene TTS',
|
|
2090
|
-
// description: `生成场景配音`,
|
|
2091
|
-
// inputSchema: {
|
|
2092
|
-
// text: z.string().describe('The text to generate.'),
|
|
2093
|
-
// sceneIndex: z
|
|
2094
|
-
// .number()
|
|
2095
|
-
// .min(1)
|
|
2096
|
-
// .optional()
|
|
2097
|
-
// .describe(
|
|
2098
|
-
// '场景索引,从1开始的下标,如果非场景对应素材,则可不传,场景素材必传'
|
|
2099
|
-
// ),
|
|
2100
|
-
// storyBoardFile: z
|
|
2101
|
-
// .string()
|
|
2102
|
-
// .optional()
|
|
2103
|
-
// .default('storyboard.json')
|
|
2104
|
-
// .describe('故事板文件路径'),
|
|
2105
|
-
// skipConsistencyCheck: z
|
|
2106
|
-
// .boolean()
|
|
2107
|
-
// .optional()
|
|
2108
|
-
// .default(false)
|
|
2109
|
-
// .describe('是否跳过一致性检查,默认为false(即默认进行一致性检查)'),
|
|
2110
|
-
// skipCheckWithSceneReason: z
|
|
2111
|
-
// .string()
|
|
2112
|
-
// .optional()
|
|
2113
|
-
// .describe(
|
|
2114
|
-
// '跳过校验的理由,如果skipConsistencyCheck设为true,必须要传这个参数'
|
|
2115
|
-
// ),
|
|
2116
|
-
// saveToFileName: z
|
|
2117
|
-
// .string()
|
|
2118
|
-
// .describe('The filename to save. 应该是mp3文件'),
|
|
2119
|
-
// speed: z
|
|
2120
|
-
// .number()
|
|
2121
|
-
// .min(0.5)
|
|
2122
|
-
// .max(2)
|
|
2123
|
-
// .optional()
|
|
2124
|
-
// .default(1)
|
|
2125
|
-
// .describe('The speed of the tts.'),
|
|
2126
|
-
// pitch: z
|
|
2127
|
-
// .number()
|
|
2128
|
-
// .min(-12)
|
|
2129
|
-
// .max(12)
|
|
2130
|
-
// .optional()
|
|
2131
|
-
// .default(0)
|
|
2132
|
-
// .describe('The pitch of the tts.'),
|
|
2133
|
-
// volume: z
|
|
2134
|
-
// .number()
|
|
2135
|
-
// .min(0)
|
|
2136
|
-
// .max(10)
|
|
2137
|
-
// .optional()
|
|
2138
|
-
// .default(1.0)
|
|
2139
|
-
// .describe('The volume of the tts.'),
|
|
2140
|
-
// voiceID: z
|
|
2141
|
-
// .string()
|
|
2142
|
-
// .describe(
|
|
2143
|
-
// `适合作为视频配音的音色ID,除非用户指定,否则你必须确保已通过 pick-voice 工具挑选出真实存在的音色。`
|
|
2144
|
-
// ),
|
|
2145
|
-
// context_texts: z
|
|
2146
|
-
// .array(z.string())
|
|
2147
|
-
// .default([])
|
|
2148
|
-
// .describe(
|
|
2149
|
-
// `语音合成的辅助信息,用于模型对话式合成,能更好的体现语音情感
|
|
2150
|
-
// 可以探索,比如常见示例有以下几种:
|
|
2151
|
-
// 1. 语速调整
|
|
2152
|
-
// - context_texts: ["你可以说慢一点吗?"]
|
|
2153
|
-
// 2. 情绪/语气调整
|
|
2154
|
-
// - context_texts=["你可以用特别特别痛心的语气说话吗?"]
|
|
2155
|
-
// - context_texts=["嗯,你的语气再欢乐一点"]
|
|
2156
|
-
// 3. 音量调整
|
|
2157
|
-
// - context_texts=["你嗓门再小点。"]
|
|
2158
|
-
// 4. 音感调整
|
|
2159
|
-
// - context_texts=["你能用骄傲的语气来说话吗?"]
|
|
2160
|
-
// `
|
|
2161
|
-
// ),
|
|
2162
|
-
// explicit_language: z.enum(['zh', 'en', 'ja']).optional().default('zh'),
|
|
2163
|
-
// },
|
|
2164
|
-
// },
|
|
2165
|
-
// async ({
|
|
2166
|
-
// text,
|
|
2167
|
-
// sceneIndex,
|
|
2168
|
-
// storyBoardFile,
|
|
2169
|
-
// skipConsistencyCheck,
|
|
2170
|
-
// voiceID,
|
|
2171
|
-
// saveToFileName,
|
|
2172
|
-
// speed,
|
|
2173
|
-
// pitch,
|
|
2174
|
-
// volume,
|
|
2175
|
-
// context_texts,
|
|
2176
|
-
// explicit_language,
|
|
2177
|
-
// }) => {
|
|
2178
|
-
// try {
|
|
2179
|
-
// // 验证session状态
|
|
2180
|
-
// const currentSession = await validateSession('generate-scene-tts');
|
|
2181
|
-
// const validatedFileName = validateFileName(saveToFileName);
|
|
2182
|
-
// const finalSpeed = speed ?? 1;
|
|
2183
|
-
// volume = volume ?? 1;
|
|
2184
|
-
// const ai = currentSession.ai;
|
|
2185
|
-
// let scene = null;
|
|
2186
|
-
// // 校验 text 与 storyboard.json 中场景设定的一致性
|
|
2187
|
-
// if (sceneIndex && !skipConsistencyCheck) {
|
|
2188
|
-
// try {
|
|
2189
|
-
// const voice = (await ai.listVoices()).find(v => v.id === voiceID);
|
|
2190
|
-
// if (!voice) {
|
|
2191
|
-
// return createErrorResponse(
|
|
2192
|
-
// `Voice ${voiceID} not found in voice-list. Use pick-voice tool to pick an available voice. 若用户坚持要使用该音色,需跳过一致性检查。`,
|
|
2193
|
-
// 'generate-scene-tts'
|
|
2194
|
-
// );
|
|
2195
|
-
// }
|
|
2196
|
-
// const storyBoardPath = resolve(
|
|
2197
|
-
// process.env.ZEROCUT_PROJECT_CWD || process.cwd(),
|
|
2198
|
-
// projectLocalDir,
|
|
2199
|
-
// storyBoardFile
|
|
2200
|
-
// );
|
|
2201
|
-
// if (existsSync(storyBoardPath)) {
|
|
2202
|
-
// const storyBoardContent = await readFile(storyBoardPath, 'utf8');
|
|
2203
|
-
// // 检查 storyBoard JSON 语法合法性
|
|
2204
|
-
// let storyBoard;
|
|
2205
|
-
// try {
|
|
2206
|
-
// storyBoard = JSON.parse(storyBoardContent);
|
|
2207
|
-
// } catch (jsonError) {
|
|
2208
|
-
// return createErrorResponse(
|
|
2209
|
-
// `storyBoard 文件 ${storyBoardFile} 存在 JSON 语法错误,请修复后重试。错误详情: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}`,
|
|
2210
|
-
// 'generate-scene-tts'
|
|
2211
|
-
// );
|
|
2212
|
-
// }
|
|
2213
|
-
// if (storyBoard.scenes && Array.isArray(storyBoard.scenes)) {
|
|
2214
|
-
// scene = storyBoard.scenes[sceneIndex - 1]; // sceneIndex 从1开始,数组从0开始
|
|
2215
|
-
// if (scene) {
|
|
2216
|
-
// const script = scene.script;
|
|
2217
|
-
// let isValidText = false;
|
|
2218
|
-
// // 检查 text 是否严格等于 script
|
|
2219
|
-
// if (script && text === script) {
|
|
2220
|
-
// isValidText = true;
|
|
2221
|
-
// }
|
|
2222
|
-
// // 检查 text 是否严格等于 dialog 数组中某个元素的 script
|
|
2223
|
-
// if (
|
|
2224
|
-
// !isValidText &&
|
|
2225
|
-
// scene.dialog &&
|
|
2226
|
-
// Array.isArray(scene.dialog)
|
|
2227
|
-
// ) {
|
|
2228
|
-
// for (const dialogItem of scene.dialog) {
|
|
2229
|
-
// if (dialogItem.script && text === dialogItem.script) {
|
|
2230
|
-
// isValidText = true;
|
|
2231
|
-
// break;
|
|
2232
|
-
// }
|
|
2233
|
-
// }
|
|
2234
|
-
// }
|
|
2235
|
-
// if (!isValidText) {
|
|
2236
|
-
// return createErrorResponse(
|
|
2237
|
-
// '配音文本必须严格遵照storyboard的设定,如果用户明确指出不需要遵守,请将skipConsistencyCheck设置为true后再次调用',
|
|
2238
|
-
// 'generate-scene-tts'
|
|
2239
|
-
// );
|
|
2240
|
-
// }
|
|
2241
|
-
// } else {
|
|
2242
|
-
// console.warn(
|
|
2243
|
-
// `Scene index ${sceneIndex} not found in storyboard.json`
|
|
2244
|
-
// );
|
|
2245
|
-
// }
|
|
2246
|
-
// }
|
|
2247
|
-
// } else {
|
|
2248
|
-
// console.warn(`Story board file not found: ${storyBoardPath}`);
|
|
2249
|
-
// }
|
|
2250
|
-
// } catch (error) {
|
|
2251
|
-
// console.error('Failed to validate text with story board:', error);
|
|
2252
|
-
// // 如果读取或解析 storyboard.json 失败,继续执行但记录警告
|
|
2253
|
-
// }
|
|
2254
|
-
// }
|
|
2255
|
-
// console.log(
|
|
2256
|
-
// `Generating TTS with voice: ${voiceID}, speed: ${finalSpeed}, text: ${text.substring(0, 100)}...`
|
|
2257
|
-
// );
|
|
2258
|
-
// if (voiceID.startsWith('BV0')) {
|
|
2259
|
-
// throw new Error(
|
|
2260
|
-
// `BV0* 系列音色已弃用,你必须通过 pick-voice 工具挑选一个真实存在的音色。`
|
|
2261
|
-
// );
|
|
2262
|
-
// }
|
|
2263
|
-
// const type =
|
|
2264
|
-
// voiceID.startsWith('zh_') ||
|
|
2265
|
-
// voiceID.startsWith('en_') ||
|
|
2266
|
-
// voiceID.startsWith('multi_') ||
|
|
2267
|
-
// voiceID.startsWith('saturn_') ||
|
|
2268
|
-
// voiceID.startsWith('ICL_')
|
|
2269
|
-
// ? 'volcano'
|
|
2270
|
-
// : 'minimax';
|
|
2271
|
-
// let res;
|
|
2272
|
-
// let emotion = 'auto';
|
|
2273
|
-
// if (type === 'volcano') {
|
|
2274
|
-
// volume = Math.max(Math.min(volume, 2.0), 0.5);
|
|
2275
|
-
// res = await ai.textToSpeechVolc({
|
|
2276
|
-
// text: text.trim(),
|
|
2277
|
-
// speaker: voiceID,
|
|
2278
|
-
// speed: Math.floor(100 * (finalSpeed - 1)),
|
|
2279
|
-
// volume: Math.floor(100 * (volume - 1)),
|
|
2280
|
-
// context_texts,
|
|
2281
|
-
// explicit_language,
|
|
2282
|
-
// voice_to_caption:
|
|
2283
|
-
// explicit_language === 'zh' || explicit_language === 'en',
|
|
2284
|
-
// });
|
|
2285
|
-
// } else {
|
|
2286
|
-
// emotion = 'neutral';
|
|
2287
|
-
// if (context_texts.length > 0) {
|
|
2288
|
-
// const prompt = `根据用户输入语音内容和上下文内容,从文字判断语音合理的情感,然后选择以下情感**之一**返回结果:
|
|
2289
|
-
// "happy", "sad", "angry", "fearful", "disgusted", "surprised", "calm", "fluent", "whisper", "neutral"
|
|
2290
|
-
// ## 要求
|
|
2291
|
-
// 输出 JSON 格式,包含一个 emotion 字段,值为以上情感之一。
|
|
2292
|
-
// `;
|
|
2293
|
-
// const schema = {
|
|
2294
|
-
// name: 'emotion_schema',
|
|
2295
|
-
// schema: {
|
|
2296
|
-
// type: 'object',
|
|
2297
|
-
// properties: {
|
|
2298
|
-
// emotion: {
|
|
2299
|
-
// type: 'string',
|
|
2300
|
-
// enum: [
|
|
2301
|
-
// 'neutral',
|
|
2302
|
-
// 'happy',
|
|
2303
|
-
// 'sad',
|
|
2304
|
-
// 'angry',
|
|
2305
|
-
// 'fearful',
|
|
2306
|
-
// 'disgusted',
|
|
2307
|
-
// 'surprised',
|
|
2308
|
-
// 'calm',
|
|
2309
|
-
// 'fluent',
|
|
2310
|
-
// 'whisper',
|
|
2311
|
-
// ],
|
|
2312
|
-
// description: '用户输入语音的情感',
|
|
2313
|
-
// },
|
|
2314
|
-
// },
|
|
2315
|
-
// required: ['emotion'],
|
|
2316
|
-
// },
|
|
2317
|
-
// };
|
|
2318
|
-
// const payload: any = {
|
|
2319
|
-
// model: 'Doubao-Seed-1.6',
|
|
2320
|
-
// messages: [
|
|
2321
|
-
// {
|
|
2322
|
-
// role: 'system',
|
|
2323
|
-
// content: prompt,
|
|
2324
|
-
// },
|
|
2325
|
-
// {
|
|
2326
|
-
// role: 'user',
|
|
2327
|
-
// content: `## 语音内容:
|
|
2328
|
-
// ${text.trim()}
|
|
2329
|
-
// ## 语音上下文
|
|
2330
|
-
// ${context_texts.join('\n')}
|
|
2331
|
-
// `,
|
|
2332
|
-
// },
|
|
2333
|
-
// ],
|
|
2334
|
-
// response_format: {
|
|
2335
|
-
// type: 'json_schema',
|
|
2336
|
-
// json_schema: schema,
|
|
2337
|
-
// },
|
|
2338
|
-
// };
|
|
2339
|
-
// const completion = await ai.getCompletions(payload);
|
|
2340
|
-
// const emotionObj = JSON.parse(
|
|
2341
|
-
// completion.choices[0]?.message?.content ?? '{}'
|
|
2342
|
-
// );
|
|
2343
|
-
// emotion = emotionObj.emotion ?? 'neutral';
|
|
2344
|
-
// }
|
|
2345
|
-
// res = await ai.textToSpeech({
|
|
2346
|
-
// text: text.trim(),
|
|
2347
|
-
// voiceName: voiceID,
|
|
2348
|
-
// speed: finalSpeed,
|
|
2349
|
-
// pitch,
|
|
2350
|
-
// volume,
|
|
2351
|
-
// emotion,
|
|
2352
|
-
// voice_to_caption:
|
|
2353
|
-
// explicit_language === 'zh' || explicit_language === 'en',
|
|
2354
|
-
// });
|
|
2355
|
-
// }
|
|
2356
|
-
// if (!res) {
|
|
2357
|
-
// throw new Error('Failed to generate TTS: no response from AI service');
|
|
2358
|
-
// }
|
|
2359
|
-
// if (res.url) {
|
|
2360
|
-
// console.log('TTS generated successfully, saving to materials...');
|
|
2361
|
-
// const { url, duration, ...opts } = res;
|
|
2362
|
-
// if (!skipConsistencyCheck && duration > 16) {
|
|
2363
|
-
// return createErrorResponse(
|
|
2364
|
-
// 'TTS duration exceeds 16 seconds, 建议调整文本长度、提升语速或拆分场景...,⚠️如简化文本内容或拆分文本,需要立即更新 storyboard 以保持内容同步!如仍要生成,可设置 skipConsistencyCheck 为 true,跳过一致性检查。',
|
|
2365
|
-
// 'generate-scene-tts'
|
|
2366
|
-
// );
|
|
2367
|
-
// }
|
|
2368
|
-
// if (!duration) {
|
|
2369
|
-
// return createErrorResponse(
|
|
2370
|
-
// 'TTS duration not returned from AI service',
|
|
2371
|
-
// 'generate-scene-tts'
|
|
2372
|
-
// );
|
|
2373
|
-
// }
|
|
2374
|
-
// const uri = await saveMaterial(currentSession, url, validatedFileName);
|
|
2375
|
-
// let warn = '';
|
|
2376
|
-
// if (scene) {
|
|
2377
|
-
// const minDur = Math.ceil(duration);
|
|
2378
|
-
// if (scene.audio_mode === 'vo_sync' && scene.duration !== minDur) {
|
|
2379
|
-
// warn = `场景${sceneIndex}设定的时长${scene.duration}秒与实际生成的语音时长${minDur}秒不一致,音画同步将有问题,建议修改场景时长为${minDur}秒`;
|
|
2380
|
-
// } else if (scene.duration < minDur) {
|
|
2381
|
-
// warn = `场景${sceneIndex}设定的时长${scene.duration}秒小于实际生成的语音时长${duration}秒,可能会导致场景结束时音频未播放完成,建议修改场景时长为${minDur}秒`;
|
|
2382
|
-
// }
|
|
2383
|
-
// }
|
|
2384
|
-
// const result = {
|
|
2385
|
-
// success: true,
|
|
2386
|
-
// warn,
|
|
2387
|
-
// source: url, // 方便调试
|
|
2388
|
-
// uri,
|
|
2389
|
-
// durationMs: Math.floor((duration || 0) * 1000),
|
|
2390
|
-
// text,
|
|
2391
|
-
// emotion,
|
|
2392
|
-
// context_texts,
|
|
2393
|
-
// voiceName: voiceID,
|
|
2394
|
-
// speed: finalSpeed,
|
|
2395
|
-
// timestamp: new Date().toISOString(),
|
|
2396
|
-
// ...opts,
|
|
2397
|
-
// };
|
|
2398
|
-
// // Update media_logs.json
|
|
2399
|
-
// try {
|
|
2400
|
-
// await updateMediaLogs(
|
|
2401
|
-
// currentSession,
|
|
2402
|
-
// validatedFileName,
|
|
2403
|
-
// result,
|
|
2404
|
-
// 'audio'
|
|
2405
|
-
// );
|
|
2406
|
-
// } catch (error) {
|
|
2407
|
-
// console.warn(
|
|
2408
|
-
// `Failed to update media_logs.json for ${validatedFileName}:`,
|
|
2409
|
-
// error
|
|
2410
|
-
// );
|
|
2411
|
-
// }
|
|
2412
|
-
// return {
|
|
2413
|
-
// content: [
|
|
2414
|
-
// {
|
|
2415
|
-
// type: 'text' as const,
|
|
2416
|
-
// text: JSON.stringify(result),
|
|
2417
|
-
// },
|
|
2418
|
-
// ],
|
|
2419
|
-
// };
|
|
2420
|
-
// } else {
|
|
2421
|
-
// console.warn('TTS generation completed but no URL returned');
|
|
2422
|
-
// return {
|
|
2423
|
-
// content: [
|
|
2424
|
-
// {
|
|
2425
|
-
// type: 'text' as const,
|
|
2426
|
-
// text: JSON.stringify({
|
|
2427
|
-
// success: false,
|
|
2428
|
-
// error:
|
|
2429
|
-
// 'No TTS URL returned from AI service. You should use pick-voice tool to pick an available voice.',
|
|
2430
|
-
// response: res,
|
|
2431
|
-
// timestamp: new Date().toISOString(),
|
|
2432
|
-
// }),
|
|
2433
|
-
// },
|
|
2434
|
-
// ],
|
|
2435
|
-
// };
|
|
2436
|
-
// }
|
|
2437
|
-
// } catch (error) {
|
|
2438
|
-
// return createErrorResponse(error, 'generate-scene-tts');
|
|
2439
|
-
// }
|
|
2440
|
-
// }
|
|
2441
|
-
// );
|
|
2442
|
-
// server.registerTool(
|
|
2443
|
-
// 'compile-and-run',
|
|
2444
|
-
// {
|
|
2445
|
-
// title: 'Compile And Run',
|
|
2446
|
-
// description: 'Compile project to ffmpeg command and run it.',
|
|
2447
|
-
// inputSchema: {
|
|
2448
|
-
// projectFileName: z
|
|
2449
|
-
// .string()
|
|
2450
|
-
// .describe('The VideoProject configuration object.'),
|
|
2451
|
-
// outputFileName: z
|
|
2452
|
-
// .string()
|
|
2453
|
-
// .optional()
|
|
2454
|
-
// .describe('Output video filename (optional, defaults to output.mp4).'),
|
|
2455
|
-
// },
|
|
2456
|
-
// },
|
|
2457
|
-
// async ({ projectFileName, outputFileName }) => {
|
|
2458
|
-
// try {
|
|
2459
|
-
// // 验证session状态
|
|
2460
|
-
// const currentSession = await validateSession('compile-and-run');
|
|
2461
|
-
// // 检查字幕内容匹配标记
|
|
2462
|
-
// if (!checkStoryboardSubtitlesFlag) {
|
|
2463
|
-
// checkStoryboardSubtitlesFlag = true;
|
|
2464
|
-
// return createErrorResponse(
|
|
2465
|
-
// `请先对 draft_content 进行以下一致性检查:
|
|
2466
|
-
// 1. 检查字幕文字内容是否与 storyboard 中各个场景的 script 或 dialog 内容完全一致(⚠️ 允许字幕分段展示,只要最终文本保持一致就行)
|
|
2467
|
-
// 2. 检查视频 resolution 设定是否与 storyboard 的 orientation 设置一致,默认 720p 情况下视频尺寸应为横屏 1280x720,竖屏 720x1280,若视频为 1080p 则尺寸应分别为横屏 1920x1080 和竖屏 1080x1920,切勿设反
|
|
2468
|
-
// 3. 除非用户明确表示不要背景音乐,否则应检查是否有生成并配置了 BGM,若无,则生成 BGM 并将其加入素材和轨道配置
|
|
2469
|
-
// 以上检查任何一项有问题,先修复 draft_content 使其符合要求后再进行合成`,
|
|
2470
|
-
// 'compile-and-run'
|
|
2471
|
-
// );
|
|
2472
|
-
// }
|
|
2473
|
-
// console.log('Starting video compilation and rendering...');
|
|
2474
|
-
// // 验证terminal可用性
|
|
2475
|
-
// const terminal = currentSession.terminal;
|
|
2476
|
-
// if (!terminal) {
|
|
2477
|
-
// throw new Error('Terminal not available in current session');
|
|
2478
|
-
// }
|
|
2479
|
-
// const localProjectFile = resolve(
|
|
2480
|
-
// projectLocalDir,
|
|
2481
|
-
// basename(projectFileName)
|
|
2482
|
-
// );
|
|
2483
|
-
// const project = JSON.parse(await readFile(localProjectFile, 'utf-8'));
|
|
2484
|
-
// // 验证输出文件名安全性
|
|
2485
|
-
// const outFile = outputFileName || project.export.outFile || 'output.mp4';
|
|
2486
|
-
// const validatedFileName = validateFileName(outFile);
|
|
2487
|
-
// console.log(`Output file: ${validatedFileName}`);
|
|
2488
|
-
// // 构建工作目录路径
|
|
2489
|
-
// const workDir = `/home/user/cerevox-zerocut/projects/${terminal.id}`;
|
|
2490
|
-
// const outputDir = `${workDir}/output`;
|
|
2491
|
-
// const outputPath = `${outputDir}/${validatedFileName}`;
|
|
2492
|
-
// // Project已经通过zVideoProject schema验证
|
|
2493
|
-
// const validated = { ...project };
|
|
2494
|
-
// // 更新导出配置
|
|
2495
|
-
// validated.export = {
|
|
2496
|
-
// ...validated.export,
|
|
2497
|
-
// outFile: outputPath,
|
|
2498
|
-
// };
|
|
2499
|
-
// console.log('Compiling VideoProject to FFmpeg command...');
|
|
2500
|
-
// // 编译为FFmpeg命令
|
|
2501
|
-
// let compiled: CompileResult;
|
|
2502
|
-
// try {
|
|
2503
|
-
// compiled = await compileToFfmpeg(validated, {
|
|
2504
|
-
// workingDir: outputDir,
|
|
2505
|
-
// subtitleStrategy: 'ass',
|
|
2506
|
-
// subtitlesFileName: `${outFile.replace(/\.mp4$/, '')}.subtitles.ass`,
|
|
2507
|
-
// });
|
|
2508
|
-
// } catch (compileError) {
|
|
2509
|
-
// console.error('Failed to compile VideoProject:', compileError);
|
|
2510
|
-
// throw new Error(`Failed to compile VideoProject: ${compileError}`);
|
|
2511
|
-
// }
|
|
2512
|
-
// console.log(`FFmpeg command generated (${compiled.cmd.length} chars)`);
|
|
2513
|
-
// console.log('FFmpeg Command:', compiled.cmd.substring(0, 200) + '...');
|
|
2514
|
-
// // 执行FFmpeg命令
|
|
2515
|
-
// console.log('Executing FFmpeg command...');
|
|
2516
|
-
// const result = await runFfmpeg(currentSession, compiled);
|
|
2517
|
-
// if (result.exitCode === 0) {
|
|
2518
|
-
// console.log('Video compilation completed successfully');
|
|
2519
|
-
// // 自动下载输出文件
|
|
2520
|
-
// console.log('Starting automatic download of output files...');
|
|
2521
|
-
// let downloadResult = null;
|
|
2522
|
-
// try {
|
|
2523
|
-
// const workDir = `/home/user/cerevox-zerocut/projects/${terminal.id}/output`;
|
|
2524
|
-
// let outputs: string[] = [];
|
|
2525
|
-
// try {
|
|
2526
|
-
// outputs = (await currentSession.files.listFiles(workDir)) || [];
|
|
2527
|
-
// } catch (listError) {
|
|
2528
|
-
// console.warn('Failed to list output files:', listError);
|
|
2529
|
-
// outputs = [];
|
|
2530
|
-
// }
|
|
2531
|
-
// if (outputs.length > 0) {
|
|
2532
|
-
// const outputDir = resolve(
|
|
2533
|
-
// process.env.ZEROCUT_PROJECT_CWD || process.cwd(),
|
|
2534
|
-
// projectLocalDir,
|
|
2535
|
-
// 'output'
|
|
2536
|
-
// );
|
|
2537
|
-
// await mkdir(outputDir, { recursive: true });
|
|
2538
|
-
// const downloadErrors: string[] = [];
|
|
2539
|
-
// const successfulDownloads: string[] = [];
|
|
2540
|
-
// const promises = outputs.map(async output => {
|
|
2541
|
-
// try {
|
|
2542
|
-
// await currentSession.files.download(
|
|
2543
|
-
// `${workDir}/${output}`,
|
|
2544
|
-
// `${outputDir}/${output}`
|
|
2545
|
-
// );
|
|
2546
|
-
// successfulDownloads.push(output);
|
|
2547
|
-
// return output;
|
|
2548
|
-
// } catch (downloadError) {
|
|
2549
|
-
// const errorMsg = `Failed to download ${output}: ${downloadError}`;
|
|
2550
|
-
// console.error(errorMsg);
|
|
2551
|
-
// downloadErrors.push(errorMsg);
|
|
2552
|
-
// return null;
|
|
2553
|
-
// }
|
|
2554
|
-
// });
|
|
2555
|
-
// const results = await Promise.all(promises);
|
|
2556
|
-
// const sources = results.filter(
|
|
2557
|
-
// output => output !== null
|
|
2558
|
-
// ) as string[];
|
|
2559
|
-
// downloadResult = {
|
|
2560
|
-
// totalFiles: outputs.length,
|
|
2561
|
-
// successfulDownloads: sources.length,
|
|
2562
|
-
// failedDownloads: downloadErrors.length,
|
|
2563
|
-
// downloadErrors:
|
|
2564
|
-
// downloadErrors.length > 0 ? downloadErrors : undefined,
|
|
2565
|
-
// downloadPath: outputDir,
|
|
2566
|
-
// sources,
|
|
2567
|
-
// };
|
|
2568
|
-
// console.log(
|
|
2569
|
-
// `Download completed: ${sources.length}/${outputs.length} files successful`
|
|
2570
|
-
// );
|
|
2571
|
-
// } else {
|
|
2572
|
-
// console.log('No output files found to download');
|
|
2573
|
-
// downloadResult = {
|
|
2574
|
-
// totalFiles: 0,
|
|
2575
|
-
// successfulDownloads: 0,
|
|
2576
|
-
// failedDownloads: 0,
|
|
2577
|
-
// message: 'No output files found to download',
|
|
2578
|
-
// };
|
|
2579
|
-
// }
|
|
2580
|
-
// } catch (downloadError) {
|
|
2581
|
-
// console.error('Download process failed:', downloadError);
|
|
2582
|
-
// downloadResult = {
|
|
2583
|
-
// error: `Download failed: ${downloadError}`,
|
|
2584
|
-
// totalFiles: 0,
|
|
2585
|
-
// successfulDownloads: 0,
|
|
2586
|
-
// failedDownloads: 1,
|
|
2587
|
-
// };
|
|
2588
|
-
// }
|
|
2589
|
-
// const successResult = {
|
|
2590
|
-
// success: true,
|
|
2591
|
-
// outputPath,
|
|
2592
|
-
// outputFileName: validatedFileName,
|
|
2593
|
-
// command: compiled.cmd,
|
|
2594
|
-
// message: 'Video compilation and download completed successfully',
|
|
2595
|
-
// download: downloadResult,
|
|
2596
|
-
// timestamp: new Date().toISOString(),
|
|
2597
|
-
// };
|
|
2598
|
-
// return {
|
|
2599
|
-
// content: [
|
|
2600
|
-
// {
|
|
2601
|
-
// type: 'text' as const,
|
|
2602
|
-
// text: JSON.stringify(successResult),
|
|
2603
|
-
// },
|
|
2604
|
-
// ],
|
|
2605
|
-
// };
|
|
2606
|
-
// } else {
|
|
2607
|
-
// console.error(`FFmpeg failed with exit code: ${result.exitCode}`);
|
|
2608
|
-
// const failureResult = {
|
|
2609
|
-
// success: false,
|
|
2610
|
-
// exitCode: result.exitCode,
|
|
2611
|
-
// outputPath,
|
|
2612
|
-
// // command: compiled.cmd,
|
|
2613
|
-
// stderr: result.stderr,
|
|
2614
|
-
// message: `FFmpeg exited with code ${result.exitCode}`,
|
|
2615
|
-
// timestamp: new Date().toISOString(),
|
|
2616
|
-
// };
|
|
2617
|
-
// if (result.exitCode === 254) {
|
|
2618
|
-
// failureResult.message =
|
|
2619
|
-
// 'FFmpeg failed with code 254. Close current session immediately (inMinutes = 0) and re-open a new session to try again.';
|
|
2620
|
-
// }
|
|
2621
|
-
// return {
|
|
2622
|
-
// content: [
|
|
2623
|
-
// {
|
|
2624
|
-
// type: 'text' as const,
|
|
2625
|
-
// text: JSON.stringify(failureResult),
|
|
2626
|
-
// },
|
|
2627
|
-
// ],
|
|
2628
|
-
// };
|
|
2629
|
-
// }
|
|
2630
|
-
// } catch (error) {
|
|
2631
|
-
// return createErrorResponse(error, 'compile-and-run');
|
|
2632
|
-
// }
|
|
2633
|
-
// }
|
|
2634
|
-
// );
|
|
2635
|
-
// server.registerTool(
|
|
2636
|
-
// 'get-schema',
|
|
2637
|
-
// {
|
|
2638
|
-
// title: 'Get Storyboard Schema or Draft Content Schema',
|
|
2639
|
-
// description:
|
|
2640
|
-
// 'Get the complete Storyboard or Draft Content JSON Schema definition. Use this schema to validate storyboard.json or draft_content.json files.',
|
|
2641
|
-
// inputSchema: {
|
|
2642
|
-
// type: z
|
|
2643
|
-
// .enum(['storyboard', 'draft_content'])
|
|
2644
|
-
// .describe(
|
|
2645
|
-
// 'The type of schema to retrieve. Must be either "storyboard" or "draft_content". 用 type: storyboard 的 schema 生成 storyboard.json;用 type: draft_content 的 schema 生成 draft_content.json'
|
|
2646
|
-
// ),
|
|
2647
|
-
// },
|
|
2648
|
-
// },
|
|
2649
|
-
// async ({ type }) => {
|
|
2650
|
-
// try {
|
|
2651
|
-
// const schemaPath = resolve(__dirname, `./prompts/${type}-schema.json`);
|
|
2652
|
-
// const schemaContent = await readFile(schemaPath, 'utf-8');
|
|
2653
|
-
// const schema = JSON.parse(schemaContent);
|
|
2654
|
-
// let important_guidelines = '';
|
|
2655
|
-
// if (type === 'draft_content') {
|
|
2656
|
-
// important_guidelines = `⚠️ 生成文件时请严格遵守输出规范,字幕文本内容必须与 storyboard.json 中的 script(或dialog) 字段的文本内容完全一致。
|
|
2657
|
-
// ** 字幕优化 **
|
|
2658
|
-
// * 在保证字幕文本内容与 storyboard.json 中的 script(或dialog) 字段的文本内容完全一致的前提下,可根据 tts 返回的 \`captions.utterances\` 字段对字幕的显示进行优化,将过长的字幕分段显示,在 draft_content.json 中使用分段字幕,captions 的内容在 media_logs.json 中可查询到。
|
|
2659
|
-
// * 如用户未特殊指定,字幕样式(字体及大小)务必遵守输出规范
|
|
2660
|
-
// `;
|
|
2661
|
-
// }
|
|
2662
|
-
// return {
|
|
2663
|
-
// content: [
|
|
2664
|
-
// {
|
|
2665
|
-
// type: 'text' as const,
|
|
2666
|
-
// text: JSON.stringify({
|
|
2667
|
-
// success: true,
|
|
2668
|
-
// schema,
|
|
2669
|
-
// important_guidelines,
|
|
2670
|
-
// timestamp: new Date().toISOString(),
|
|
2671
|
-
// }),
|
|
2672
|
-
// },
|
|
2673
|
-
// ],
|
|
2674
|
-
// };
|
|
2675
|
-
// } catch (error) {
|
|
2676
|
-
// return createErrorResponse(error, 'get-schema');
|
|
2677
|
-
// }
|
|
2678
|
-
// }
|
|
2679
|
-
// );
|
|
2680
|
-
// server.registerTool(
|
|
2681
|
-
// 'pick-voice',
|
|
2682
|
-
// {
|
|
2683
|
-
// title: 'Pick Voice',
|
|
2684
|
-
// description:
|
|
2685
|
-
// '根据用户需求,选择尽可能符合要求的语音,在合适的情况下,优先采用 volcano_tts_2 类型的语音',
|
|
2686
|
-
// inputSchema: {
|
|
2687
|
-
// prompt: z
|
|
2688
|
-
// .string()
|
|
2689
|
-
// .describe('用户需求描述,例如:一个有亲和力的,适合给孩子讲故事的语音'),
|
|
2690
|
-
// custom_design: z
|
|
2691
|
-
// .boolean()
|
|
2692
|
-
// .optional()
|
|
2693
|
-
// .describe(
|
|
2694
|
-
// '是否自定义语音,由于要消耗较多积分,因此**只有用户明确要求自己设计语音**,才将该参数设为true'
|
|
2695
|
-
// ),
|
|
2696
|
-
// custom_design_preview: z
|
|
2697
|
-
// .string()
|
|
2698
|
-
// .optional()
|
|
2699
|
-
// .describe(
|
|
2700
|
-
// '用户自定义语音的预览文本,用于展示自定义语音的效果,只有 custom_design 为 true 时才需要'
|
|
2701
|
-
// ),
|
|
2702
|
-
// custom_design_save_to: z
|
|
2703
|
-
// .string()
|
|
2704
|
-
// .optional()
|
|
2705
|
-
// .describe(
|
|
2706
|
-
// '自定义语音的保存路径,例如:custom_voice.mp3 custom_voice_{id}.mp3'
|
|
2707
|
-
// ),
|
|
2708
|
-
// },
|
|
2709
|
-
// },
|
|
2710
|
-
// async ({
|
|
2711
|
-
// prompt,
|
|
2712
|
-
// custom_design,
|
|
2713
|
-
// custom_design_preview,
|
|
2714
|
-
// custom_design_save_to,
|
|
2715
|
-
// }) => {
|
|
2716
|
-
// try {
|
|
2717
|
-
// // 验证session状态
|
|
2718
|
-
// const currentSession = await validateSession('pick-voice');
|
|
2719
|
-
// const ai = currentSession.ai;
|
|
2720
|
-
// if (custom_design) {
|
|
2721
|
-
// if (!custom_design_preview) {
|
|
2722
|
-
// throw new Error(
|
|
2723
|
-
// 'custom_design_preview is required when custom_design is true'
|
|
2724
|
-
// );
|
|
2725
|
-
// }
|
|
2726
|
-
// const data = await currentSession.ai.voiceDesign({
|
|
2727
|
-
// prompt,
|
|
2728
|
-
// previewText: custom_design_preview,
|
|
2729
|
-
// });
|
|
2730
|
-
// if (data.voice_id) {
|
|
2731
|
-
// const trial_audio = data.trial_audio;
|
|
2732
|
-
// let uri = '';
|
|
2733
|
-
// if (trial_audio) {
|
|
2734
|
-
// uri = await saveMaterial(
|
|
2735
|
-
// currentSession,
|
|
2736
|
-
// trial_audio,
|
|
2737
|
-
// custom_design_save_to || `custom_voice_${data.voice_id}.mp3`
|
|
2738
|
-
// );
|
|
2739
|
-
// }
|
|
2740
|
-
// return {
|
|
2741
|
-
// content: [
|
|
2742
|
-
// {
|
|
2743
|
-
// type: 'text' as const,
|
|
2744
|
-
// text: JSON.stringify({
|
|
2745
|
-
// success: true,
|
|
2746
|
-
// ...data,
|
|
2747
|
-
// uri,
|
|
2748
|
-
// timestamp: new Date().toISOString(),
|
|
2749
|
-
// }),
|
|
2750
|
-
// },
|
|
2751
|
-
// ],
|
|
2752
|
-
// };
|
|
2753
|
-
// } else {
|
|
2754
|
-
// throw new Error(`Voice design failed, ${JSON.stringify(data)}`);
|
|
2755
|
-
// }
|
|
2756
|
-
// }
|
|
2757
|
-
// const data = await ai.pickVoice({ prompt });
|
|
2758
|
-
// return {
|
|
2759
|
-
// content: [
|
|
2760
|
-
// {
|
|
2761
|
-
// type: 'text' as const,
|
|
2762
|
-
// text: JSON.stringify({
|
|
2763
|
-
// success: true,
|
|
2764
|
-
// ...data,
|
|
2765
|
-
// timestamp: new Date().toISOString(),
|
|
2766
|
-
// }),
|
|
2767
|
-
// },
|
|
2768
|
-
// ],
|
|
2769
|
-
// };
|
|
2770
|
-
// } catch (error) {
|
|
2771
|
-
// return createErrorResponse(error, 'pick-voice');
|
|
2772
|
-
// }
|
|
2773
|
-
// }
|
|
2774
|
-
// );
|
|
2775
1942
|
let lastEffect = '';
|
|
2776
1943
|
server.registerTool('generate-video', {
|
|
2777
1944
|
title: 'Generate Video',
|
|
2778
1945
|
description: `图生视频和首尾帧生视频工具`,
|
|
2779
1946
|
inputSchema: {
|
|
2780
|
-
prompt: zod_1.z
|
|
2781
|
-
.string()
|
|
2782
|
-
.describe('The prompt to generate. 一般要严格对应 storyboard 中当前场景的 video_prompt 字段描述;传这个参数时,若跳过了一致性检查,记得保留镜头切换语言,如“切镜至第二镜头”这样的指令不应当省略。'),
|
|
1947
|
+
prompt: zod_1.z.string().describe('The prompt to generate. '),
|
|
2783
1948
|
sceneIndex: zod_1.z
|
|
2784
1949
|
.number()
|
|
2785
1950
|
.min(1)
|
|
2786
1951
|
.optional()
|
|
2787
|
-
.describe('
|
|
1952
|
+
.describe('分镜索引,从1开始的下标,如果非分镜对应素材,则可不传,分镜素材必传'),
|
|
2788
1953
|
storyBoardFile: zod_1.z
|
|
2789
1954
|
.string()
|
|
2790
1955
|
.optional()
|
|
2791
1956
|
.default('storyboard.json')
|
|
2792
1957
|
.describe('故事板文件路径'),
|
|
2793
|
-
skipConsistencyCheck: zod_1.z
|
|
2794
|
-
.boolean()
|
|
2795
|
-
.optional()
|
|
2796
|
-
.default(false)
|
|
2797
|
-
.describe('是否跳过一致性检查,默认为false(即默认进行一致性检查)'),
|
|
2798
|
-
skipCheckWithSceneReason: zod_1.z
|
|
2799
|
-
.string()
|
|
2800
|
-
.optional()
|
|
2801
|
-
.describe('跳过校验的理由,如果skipConsistencyCheck设为true,必须要传这个参数'),
|
|
2802
1958
|
type: zod_1.z
|
|
2803
1959
|
.enum([
|
|
2804
1960
|
'pro',
|
|
@@ -2859,7 +2015,7 @@ server.registerTool('generate-video', {
|
|
|
2859
2015
|
.default(false)
|
|
2860
2016
|
.describe('Whether to optimize the prompt.'),
|
|
2861
2017
|
},
|
|
2862
|
-
}, async ({ prompt, sceneIndex, storyBoardFile = 'storyboard.json',
|
|
2018
|
+
}, async ({ prompt, sceneIndex, storyBoardFile = 'storyboard.json', saveToFileName, start_frame, end_frame, duration, resolution, type = 'vidu', optimizePrompt, saveLastFrameAs, mute = true, seed, }, context) => {
|
|
2863
2019
|
try {
|
|
2864
2020
|
// 验证session状态
|
|
2865
2021
|
const currentSession = await validateSession('generate-video');
|
|
@@ -2880,8 +2036,8 @@ server.registerTool('generate-video', {
|
|
|
2880
2036
|
console.warn(`zero 模型的视频仅支持 1080p 分辨率,用户指定的分辨率为 %s,已自动将 ${resolution} 转换为 1080p`, resolution);
|
|
2881
2037
|
resolution = '1080p';
|
|
2882
2038
|
}
|
|
2883
|
-
// 校验 prompt 与 storyboard.json
|
|
2884
|
-
if (sceneIndex
|
|
2039
|
+
// 校验 prompt 与 storyboard.json 中分镜设定的一致性
|
|
2040
|
+
if (sceneIndex) {
|
|
2885
2041
|
try {
|
|
2886
2042
|
const storyBoardPath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, storyBoardFile);
|
|
2887
2043
|
if ((0, node_fs_1.existsSync)(storyBoardPath)) {
|
|
@@ -2899,26 +2055,26 @@ server.registerTool('generate-video', {
|
|
|
2899
2055
|
if (scene) {
|
|
2900
2056
|
const videoPrompt = scene.video_prompt;
|
|
2901
2057
|
if (videoPrompt && prompt !== videoPrompt) {
|
|
2902
|
-
return createErrorResponse('视频提示词必须严格遵照storyboard
|
|
2058
|
+
return createErrorResponse('视频提示词必须严格遵照storyboard的设定', 'generate-video');
|
|
2903
2059
|
}
|
|
2904
2060
|
if (scene.is_continuous && !end_frame) {
|
|
2905
|
-
return createErrorResponse('
|
|
2061
|
+
return createErrorResponse('连续分镜必须指定end_frame参数', 'generate-video');
|
|
2906
2062
|
}
|
|
2907
2063
|
if (scene.video_duration != null &&
|
|
2908
2064
|
duration !== scene.video_duration) {
|
|
2909
|
-
return createErrorResponse(`视频时长必须严格遵照storyboard的设定,用户指定的时长为 ${duration} 秒,而 storyboard 中建议的时长为 ${scene.video_duration}
|
|
2065
|
+
return createErrorResponse(`视频时长必须严格遵照storyboard的设定,用户指定的时长为 ${duration} 秒,而 storyboard 中建议的时长为 ${scene.video_duration} 秒。`, 'generate-video');
|
|
2910
2066
|
}
|
|
2911
2067
|
if (storyBoard.voice_type &&
|
|
2912
2068
|
storyBoard.voice_type !== 'slient') {
|
|
2913
2069
|
if (mute) {
|
|
2914
|
-
return createErrorResponse('
|
|
2070
|
+
return createErrorResponse('有对话和旁白的分镜不能静音,请将mute设为false再重新使用工具。', 'generate-video');
|
|
2915
2071
|
}
|
|
2916
2072
|
}
|
|
2917
2073
|
// 检查 use_video_model 与 type 参数的一致性
|
|
2918
2074
|
if (scene.use_video_model &&
|
|
2919
2075
|
type &&
|
|
2920
2076
|
scene.use_video_model !== type) {
|
|
2921
|
-
return createErrorResponse(
|
|
2077
|
+
return createErrorResponse(`分镜建议的视频模型(${scene.use_video_model})与传入的type参数(${type})不一致。请确保use_video_model与type参数值相同。`, 'generate-video');
|
|
2922
2078
|
}
|
|
2923
2079
|
}
|
|
2924
2080
|
else {
|
|
@@ -2934,60 +2090,6 @@ server.registerTool('generate-video', {
|
|
|
2934
2090
|
console.error('Failed to validate prompt with story board:', error);
|
|
2935
2091
|
// 如果读取或解析 storyboard.json 失败,继续执行但记录警告
|
|
2936
2092
|
}
|
|
2937
|
-
// 校验视频时长与 timeline_analysis.json 中 proposed_video_scenes 的匹配
|
|
2938
|
-
try {
|
|
2939
|
-
const timelineAnalysisPath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, 'timeline_analysis.json');
|
|
2940
|
-
if ((0, node_fs_1.existsSync)(timelineAnalysisPath)) {
|
|
2941
|
-
const timelineAnalysisContent = await (0, promises_1.readFile)(timelineAnalysisPath, 'utf8');
|
|
2942
|
-
const timelineAnalysis = JSON.parse(timelineAnalysisContent);
|
|
2943
|
-
if (timelineAnalysis.proposed_video_scenes &&
|
|
2944
|
-
Array.isArray(timelineAnalysis.proposed_video_scenes)) {
|
|
2945
|
-
const videoScene = timelineAnalysis.proposed_video_scenes[sceneIndex - 1]; // sceneIndex 从1开始,数组从0开始
|
|
2946
|
-
if (videoScene && videoScene.video_duration_s) {
|
|
2947
|
-
const expectedDuration = videoScene.video_duration_s;
|
|
2948
|
-
if (duration !== expectedDuration) {
|
|
2949
|
-
return createErrorResponse(`视频时长必须与timeline_analysis中的设定匹配。当前场景${videoScene.scene_id}要求时长为${expectedDuration}秒,但传入的duration为${duration}秒。请调整duration参数为${expectedDuration}秒后重试。如果用户明确指出不需要遵守,请将skipConsistencyCheck设置为true后再次调用。`, 'generate-video');
|
|
2950
|
-
}
|
|
2951
|
-
}
|
|
2952
|
-
else {
|
|
2953
|
-
console.warn(`Scene ${sceneIndex} (scene_${sceneIndex.toString().padStart(2, '0')}) not found in timeline_analysis.json proposed_video_scenes`);
|
|
2954
|
-
}
|
|
2955
|
-
}
|
|
2956
|
-
}
|
|
2957
|
-
else {
|
|
2958
|
-
// 检查音频时长标志
|
|
2959
|
-
try {
|
|
2960
|
-
const storyBoardPath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, storyBoardFile);
|
|
2961
|
-
if ((0, node_fs_1.existsSync)(storyBoardPath)) {
|
|
2962
|
-
const storyBoardContent = await (0, promises_1.readFile)(storyBoardPath, 'utf8');
|
|
2963
|
-
const storyBoard = JSON.parse(storyBoardContent);
|
|
2964
|
-
if (storyBoard.scenes && Array.isArray(storyBoard.scenes)) {
|
|
2965
|
-
const scene = storyBoard.scenes[sceneIndex - 1]; // sceneIndex 从1开始,数组从0开始
|
|
2966
|
-
if (scene &&
|
|
2967
|
-
((scene.script && scene.script.trim() !== '') ||
|
|
2968
|
-
scene.dialog)) {
|
|
2969
|
-
if (!checkAudioVideoDurationFlag) {
|
|
2970
|
-
checkAudioVideoDurationFlag = true;
|
|
2971
|
-
if (scene.audio_mode === 'vo_sync') {
|
|
2972
|
-
return createErrorResponse('请先自我检查 media_logs 中的音频时长,确保 storyboard 中视频时长为音频时长向上取整 即 ceil(音频时长),然后再按照正确的视频时长创建视频', 'generate-video');
|
|
2973
|
-
}
|
|
2974
|
-
else if (scene.audio_mode === 'dialogue') {
|
|
2975
|
-
return createErrorResponse('请先自我检查 media_logs 中的音频时长,确保 storyboard 中视频时长**不小于**音频时长向上取整 即 ceil(音频时长),然后再按照正确的视频时长创建视频', 'generate-video');
|
|
2976
|
-
}
|
|
2977
|
-
}
|
|
2978
|
-
}
|
|
2979
|
-
}
|
|
2980
|
-
}
|
|
2981
|
-
}
|
|
2982
|
-
catch (error) {
|
|
2983
|
-
console.error('Failed to check audio duration flag:', error);
|
|
2984
|
-
}
|
|
2985
|
-
}
|
|
2986
|
-
}
|
|
2987
|
-
catch (error) {
|
|
2988
|
-
console.error('Failed to validate duration with timeline analysis:', error);
|
|
2989
|
-
// 如果读取或解析 timeline_analysis.json 失败,继续执行但记录警告
|
|
2990
|
-
}
|
|
2991
2093
|
}
|
|
2992
2094
|
const validatedFileName = validateFileName(saveToFileName);
|
|
2993
2095
|
console.log(`Generating video with prompt: ${prompt.substring(0, 100)}...`);
|
|
@@ -3624,11 +2726,15 @@ server.registerTool('audio-video-sync', {
|
|
|
3624
2726
|
.describe('The volume of video audio. 0.0 to 2.0.'),
|
|
3625
2727
|
loopAudio: zod_1.z.boolean().optional().default(true),
|
|
3626
2728
|
addSubtitles: zod_1.z.boolean().optional().default(false),
|
|
2729
|
+
subtitlesContext: zod_1.z
|
|
2730
|
+
.string()
|
|
2731
|
+
.optional()
|
|
2732
|
+
.describe('字幕的参考上下文(非必需),用于提升字幕准确性'),
|
|
3627
2733
|
saveToFileName: zod_1.z
|
|
3628
2734
|
.string()
|
|
3629
2735
|
.describe('The filename to save the audio-video-synced video. 应该是mp4文件'),
|
|
3630
2736
|
},
|
|
3631
|
-
}, async ({ videos, audio, audioInMs, audioFadeOutMs, audioVolume, videoAudioVolume, saveToFileName, loopAudio, addSubtitles, }, context) => {
|
|
2737
|
+
}, async ({ videos, audio, audioInMs, audioFadeOutMs, audioVolume, videoAudioVolume, saveToFileName, loopAudio, addSubtitles, subtitlesContext, }, context) => {
|
|
3632
2738
|
try {
|
|
3633
2739
|
// 验证session状态
|
|
3634
2740
|
const currentSession = await validateSession('audio-video-sync');
|
|
@@ -3662,6 +2768,7 @@ server.registerTool('audio-video-sync', {
|
|
|
3662
2768
|
videoAudioVolume,
|
|
3663
2769
|
loopAudio,
|
|
3664
2770
|
subtitles: addSubtitles,
|
|
2771
|
+
subtitlesContext,
|
|
3665
2772
|
});
|
|
3666
2773
|
if (result.url) {
|
|
3667
2774
|
console.log('Audio sync completed successfully');
|
|
@@ -3709,7 +2816,7 @@ server.registerTool('generate-video-by-ref', {
|
|
|
3709
2816
|
inputSchema: {
|
|
3710
2817
|
prompt: zod_1.z
|
|
3711
2818
|
.string()
|
|
3712
|
-
.describe('The prompt to generate video with or without reference images.
|
|
2819
|
+
.describe('The prompt to generate video with or without reference images. '),
|
|
3713
2820
|
rewritePrompt: zod_1.z
|
|
3714
2821
|
.boolean()
|
|
3715
2822
|
.optional()
|
|
@@ -3734,7 +2841,7 @@ server.registerTool('generate-video-by-ref', {
|
|
|
3734
2841
|
.max(16)
|
|
3735
2842
|
.optional()
|
|
3736
2843
|
.default(5)
|
|
3737
|
-
.describe('The duration of the video in seconds
|
|
2844
|
+
.describe('The duration of the video in seconds.可以传0,此时会根据视频提示词内容自动确定时长'),
|
|
3738
2845
|
aspectRatio: zod_1.z
|
|
3739
2846
|
.enum(['16:9', '9:16'])
|
|
3740
2847
|
.describe('The aspect ratio of the video.'),
|
|
@@ -3770,58 +2877,31 @@ server.registerTool('generate-video-by-ref', {
|
|
|
3770
2877
|
.number()
|
|
3771
2878
|
.min(1)
|
|
3772
2879
|
.optional()
|
|
3773
|
-
.describe('
|
|
2880
|
+
.describe('分镜索引,从1开始的下标,如果非分镜对应素材,则可不传,分镜素材必传'),
|
|
3774
2881
|
storyBoardFile: zod_1.z
|
|
3775
2882
|
.string()
|
|
3776
2883
|
.optional()
|
|
3777
2884
|
.default('storyboard.json')
|
|
3778
2885
|
.describe('故事板文件路径'),
|
|
3779
|
-
skipConsistencyCheck: zod_1.z
|
|
3780
|
-
.boolean()
|
|
3781
|
-
.optional()
|
|
3782
|
-
.default(false)
|
|
3783
|
-
.describe('是否跳过一致性检查,默认为false(即默认进行一致性检查)'),
|
|
3784
|
-
skipCheckWithSceneReason: zod_1.z
|
|
3785
|
-
.string()
|
|
3786
|
-
.optional()
|
|
3787
|
-
.describe('跳过校验的理由,如果skipConsistencyCheck设为true,必须要传这个参数'),
|
|
3788
2886
|
optimizePrompt: zod_1.z
|
|
3789
2887
|
.boolean()
|
|
3790
2888
|
.optional()
|
|
3791
2889
|
.default(false)
|
|
3792
2890
|
.describe('Whether to optimize the prompt.'),
|
|
3793
2891
|
},
|
|
3794
|
-
}, async ({ prompt, rewritePrompt, referenceImages, duration, aspectRatio, resolution, type = 'vidu', mute, saveToFileName, sceneIndex, storyBoardFile,
|
|
2892
|
+
}, async ({ prompt, rewritePrompt, referenceImages, duration, aspectRatio, resolution, type = 'vidu', mute, saveToFileName, sceneIndex, storyBoardFile, optimizePrompt, seed, }, context) => {
|
|
3795
2893
|
try {
|
|
3796
2894
|
// 验证session状态
|
|
3797
2895
|
const currentSession = await validateSession('generate-video-by-ref');
|
|
3798
2896
|
const storyBoardPath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, storyBoardFile);
|
|
3799
|
-
if (type !== 'pro' && duration === 0) {
|
|
3800
|
-
return createErrorResponse('非 pro 模型的视频时长不能为 0', 'generate-video');
|
|
3801
|
-
}
|
|
3802
2897
|
const outlineSheetImagePath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, 'materials', 'outline_sheet.png');
|
|
3803
2898
|
const hasOutlineSheet = (0, node_fs_1.existsSync)(outlineSheetImagePath);
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
}
|
|
3807
|
-
// 检查 storyboard 标志
|
|
3808
|
-
if (!hasOutlineSheet &&
|
|
3809
|
-
!checkStoryboardFlag &&
|
|
3810
|
-
(0, node_fs_1.existsSync)(storyBoardPath)) {
|
|
3811
|
-
checkStoryboardFlag = true;
|
|
3812
|
-
return createErrorResponse(`必须先审查生成的 storyboard.json 内容,按照如下步骤:
|
|
3813
|
-
|
|
3814
|
-
1. 确保每个场景中的stage_atmosphere内容按照规则被正确融合到video_prompt中,不得遗漏
|
|
3815
|
-
2. 如有main_characters设定且包含了reference_image,或有reference_objects,需确保video_prompt描述已包含该场景相关main_characters和所有reference_objects中的物品或背景,并确保参考图具体内容已经在video_prompt中有明确描述,如果没有,可忽略。
|
|
3816
|
-
3. 如有配音,先自我检查 media_logs 中的查音频时长,确保以匹配音频时长来生成视频
|
|
3817
|
-
|
|
3818
|
-
检查完上述问题后先汇报,如果有需要,应当先修改 storyboard.json 内容,然后再调用 generate-video-by-ref 生成视频。注意修改 storyboard 内容时,仅修改相应字段的字符串值,不要破坏JSON格式!
|
|
3819
|
-
|
|
3820
|
-
再次调用 generate-video-by-ref 时,如需要参考图,要确保referenceImages使用正确(main_characters中的reference_image作为参考人物,reference_objects中的image作为参考物品或参考背景)`, 'generate-image');
|
|
3821
|
-
}
|
|
3822
|
-
// 校验 prompt 与 storyboard.json 中场景设定的一致性(如果提供了 sceneIndex)
|
|
3823
|
-
if (!skipConsistencyCheck && sceneIndex) {
|
|
2899
|
+
// 校验 prompt 与 storyboard.json 中分镜设定的一致性(如果提供了 sceneIndex)
|
|
2900
|
+
if (sceneIndex) {
|
|
3824
2901
|
try {
|
|
2902
|
+
if (hasOutlineSheet) {
|
|
2903
|
+
return createErrorResponse('监测到素材中存在outline_sheet.png这张图(由outline工具生成的),应采用 generate-video 图生视频。', 'generate-video-by-ref');
|
|
2904
|
+
}
|
|
3825
2905
|
if ((0, node_fs_1.existsSync)(storyBoardPath)) {
|
|
3826
2906
|
const storyBoardContent = await (0, promises_1.readFile)(storyBoardPath, 'utf8');
|
|
3827
2907
|
// 检查 storyBoard JSON 语法合法性
|
|
@@ -3837,20 +2917,20 @@ server.registerTool('generate-video-by-ref', {
|
|
|
3837
2917
|
if (scene) {
|
|
3838
2918
|
const videoPrompt = scene.video_prompt;
|
|
3839
2919
|
if (videoPrompt && prompt !== videoPrompt) {
|
|
3840
|
-
return createErrorResponse('视频提示词必须严格遵照storyboard
|
|
2920
|
+
return createErrorResponse('视频提示词必须严格遵照storyboard的设定', 'generate-video-by-ref');
|
|
3841
2921
|
}
|
|
3842
2922
|
// 检查 scene.is_continuous 是否为 true
|
|
3843
2923
|
if (scene.is_continuous === true) {
|
|
3844
|
-
return createErrorResponse('
|
|
2924
|
+
return createErrorResponse('连续镜头应使用首尾帧,请修改连续镜头设置,或将本分镜改为首尾帧方式实现', 'generate-video-by-ref');
|
|
3845
2925
|
}
|
|
3846
2926
|
if (scene.video_type !== 'references') {
|
|
3847
|
-
return createErrorResponse(
|
|
2927
|
+
return createErrorResponse(`分镜 ${sceneIndex} 中的 video_type (${scene.video_type}) 未设置为 'references',不应当使用参考生视频,请使用图生视频 generate-video 方式生成`, 'generate-video-by-ref');
|
|
3848
2928
|
}
|
|
3849
2929
|
// 检查 use_video_model 与 type 参数的一致性
|
|
3850
2930
|
if (scene.use_video_model &&
|
|
3851
2931
|
type &&
|
|
3852
2932
|
scene.use_video_model !== type) {
|
|
3853
|
-
return createErrorResponse(
|
|
2933
|
+
return createErrorResponse(`分镜 ${sceneIndex} 中的 use_video_model (${scene.use_video_model}) 必须与 type 参数 (${type}) 保持一致`, 'generate-video-by-ref');
|
|
3854
2934
|
}
|
|
3855
2935
|
// 检查 scene.references 的一致性
|
|
3856
2936
|
if (scene.references &&
|
|
@@ -3866,7 +2946,7 @@ server.registerTool('generate-video-by-ref', {
|
|
|
3866
2946
|
const character = mainCharacters.find((char) => char.name === referenceName);
|
|
3867
2947
|
if (character) {
|
|
3868
2948
|
if (!character.reference_image) {
|
|
3869
|
-
return createErrorResponse(
|
|
2949
|
+
return createErrorResponse(`分镜 ${sceneIndex} 中引用的角色 "${referenceName}" 在 main_characters 中没有 reference_image 属性`, 'generate-video-by-ref');
|
|
3870
2950
|
}
|
|
3871
2951
|
requiredImage = character.reference_image;
|
|
3872
2952
|
found = true;
|
|
@@ -3876,7 +2956,7 @@ server.registerTool('generate-video-by-ref', {
|
|
|
3876
2956
|
const refObject = referenceObjects.find((obj) => obj.name === referenceName);
|
|
3877
2957
|
if (refObject) {
|
|
3878
2958
|
if (!refObject.image) {
|
|
3879
|
-
return createErrorResponse(
|
|
2959
|
+
return createErrorResponse(`分镜 ${sceneIndex} 中引用的物品/背景 "${referenceName}" 在 reference_objects 中没有 image 属性`, 'generate-video-by-ref');
|
|
3880
2960
|
}
|
|
3881
2961
|
requiredImage = refObject.image;
|
|
3882
2962
|
found = true;
|
|
@@ -3884,11 +2964,11 @@ server.registerTool('generate-video-by-ref', {
|
|
|
3884
2964
|
}
|
|
3885
2965
|
// 如果既不在 main_characters 也不在 reference_objects 中
|
|
3886
2966
|
if (!found) {
|
|
3887
|
-
return createErrorResponse(
|
|
2967
|
+
return createErrorResponse(`分镜 ${sceneIndex} 中引用的 "${referenceName}" 在 main_characters 或 reference_objects 中都找不到对应的定义`, 'generate-video-by-ref');
|
|
3888
2968
|
}
|
|
3889
2969
|
// 检查对应的图片是否在 referenceImages 参数中
|
|
3890
2970
|
if (!referenceImages.some(ref => ref.fileName === requiredImage)) {
|
|
3891
|
-
return createErrorResponse(
|
|
2971
|
+
return createErrorResponse(`分镜 ${sceneIndex} 中引用的 "${referenceName}" 对应的图片 "${requiredImage}" 不在 referenceImages 参数中,请确保传入正确的参考图片`, 'generate-video-by-ref');
|
|
3892
2972
|
}
|
|
3893
2973
|
}
|
|
3894
2974
|
}
|