cerevox 3.0.0-beta.1 → 3.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/ai.d.ts +18 -1
- package/dist/core/ai.d.ts.map +1 -1
- package/dist/core/ai.js +52 -2
- package/dist/core/ai.js.map +1 -1
- package/dist/mcp/servers/prompts/image-prompt-optimizer bak.md +68 -0
- package/dist/mcp/servers/prompts/image-prompt-optimizer.md +14 -34
- package/dist/mcp/servers/prompts/rules/creative-ad.md +2 -2
- package/dist/mcp/servers/prompts/rules/expert.md +1 -1
- package/dist/mcp/servers/prompts/rules/freeform.md +2 -2
- package/dist/mcp/servers/prompts/rules/general-video.md +2 -2
- package/dist/mcp/servers/prompts/rules/story-telling.md +3 -3
- package/dist/mcp/servers/prompts/skills/workflows/general-video.md +2 -2
- package/dist/mcp/servers/prompts/skills/workflows/music-video.md +1 -1
- package/dist/mcp/servers/prompts/zerocut-core.md +0 -5
- package/dist/mcp/servers/zerocut.d.ts.map +1 -1
- package/dist/mcp/servers/zerocut.js +265 -141
- package/dist/mcp/servers/zerocut.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"zerocut.d.ts","sourceRoot":"","sources":["../../../src/mcp/servers/zerocut.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"zerocut.d.ts","sourceRoot":"","sources":["../../../src/mcp/servers/zerocut.ts"],"names":[],"mappings":";AA6nLA,wBAAsB,GAAG,kBAKxB"}
|
|
@@ -552,11 +552,17 @@ server.registerTool('retrieve-rules-context', {
|
|
|
552
552
|
title: 'ZeroCut Basic Rules',
|
|
553
553
|
description: `ZeroCut-${constants_1.VERSION} Basic Rules`,
|
|
554
554
|
inputSchema: {
|
|
555
|
-
request: zod_1.z.string().describe(
|
|
555
|
+
request: zod_1.z.string().describe(`根据用户的具体的指令召回规则上下文,具体指令为通常为如下格式之一:
|
|
556
|
+
|
|
557
|
+
- 启用xxx模式
|
|
558
|
+
- 使用xxx助手
|
|
559
|
+
- 启用xxx角色
|
|
560
|
+
- 切换上下文为xxx
|
|
561
|
+
- 召回原规则上下文(配合type:memo使用)
|
|
562
|
+
`),
|
|
556
563
|
type: zod_1.z
|
|
557
|
-
.enum(['
|
|
558
|
-
.
|
|
559
|
-
.describe('retrieve: 根据需要重新召回规则上下文\nswitch: 根据用户指令,切换规则上下文'),
|
|
564
|
+
.enum(['new', 'memo', 'switch'])
|
|
565
|
+
.describe('new: 新创建规则上下文\nmemo: 根据需要重新召回规则上下文(记忆)\nswitch: 根据用户指令,切换规则上下文'),
|
|
560
566
|
},
|
|
561
567
|
}, async ({ request, type }) => {
|
|
562
568
|
const currentSession = await validateSession('retrieve-rules-context', false);
|
|
@@ -580,59 +586,60 @@ server.registerTool('retrieve-rules-context', {
|
|
|
580
586
|
}
|
|
581
587
|
if (type === 'switch') {
|
|
582
588
|
(0, node_fs_1.rmSync)(projectRulesFile, { force: true });
|
|
589
|
+
(0, node_fs_1.rmSync)(customRulesFile, { force: true });
|
|
583
590
|
promptContent = '';
|
|
584
591
|
}
|
|
585
592
|
try {
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
593
|
+
if (!promptContent) {
|
|
594
|
+
const ai = currentSession.ai;
|
|
595
|
+
const { data: rules } = await ai.listContextRules();
|
|
596
|
+
const rulesList = rules.map((rule) => ({
|
|
597
|
+
name: rule.name,
|
|
598
|
+
trigger: rule.trigger,
|
|
599
|
+
}));
|
|
600
|
+
const chooseRulePrompt = `
|
|
601
|
+
请分析用户具体指令,匹配可用规则中的trigger,选择最合适的规则。
|
|
594
602
|
|
|
595
|
-
## 可用规则:
|
|
596
|
-
${JSON.stringify(rulesList, null, 2)}
|
|
603
|
+
## 可用规则:
|
|
604
|
+
${JSON.stringify(rulesList, null, 2)}
|
|
597
605
|
|
|
598
|
-
请返回一个 JSON 字符串,包含用户意图对应的规则名称。格式为:{"rule_name": "规则名称"}
|
|
599
|
-
`;
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
606
|
+
请返回一个 JSON 字符串,包含用户意图对应的规则名称。格式为:{"rule_name": "规则名称"}
|
|
607
|
+
`;
|
|
608
|
+
const schema = {
|
|
609
|
+
name: 'choose_rule',
|
|
610
|
+
schema: {
|
|
611
|
+
type: 'object',
|
|
612
|
+
properties: {
|
|
613
|
+
rule_name: {
|
|
614
|
+
type: 'string',
|
|
615
|
+
description: '用户意图对应的规则名称',
|
|
616
|
+
},
|
|
608
617
|
},
|
|
618
|
+
required: ['rule_name'],
|
|
609
619
|
},
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
620
|
+
};
|
|
621
|
+
const completion = await ai.getCompletions({
|
|
622
|
+
model: 'Doubao-Seed-1.6-flash',
|
|
623
|
+
messages: [
|
|
624
|
+
{
|
|
625
|
+
role: 'system',
|
|
626
|
+
content: chooseRulePrompt,
|
|
627
|
+
},
|
|
628
|
+
{
|
|
629
|
+
role: 'user',
|
|
630
|
+
content: request,
|
|
631
|
+
},
|
|
632
|
+
],
|
|
633
|
+
response_format: {
|
|
634
|
+
type: 'json_schema',
|
|
635
|
+
json_schema: schema,
|
|
623
636
|
},
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
const ruleName = JSON.parse(completion.choices[0].message.content).rule_name;
|
|
631
|
-
let rule = rules.find((rule) => rule.name === ruleName);
|
|
632
|
-
if (!rule) {
|
|
633
|
-
rule = rules.find((rule) => rule.name === '通用视频助手');
|
|
634
|
-
}
|
|
635
|
-
if (!promptContent) {
|
|
637
|
+
});
|
|
638
|
+
const ruleName = JSON.parse(completion.choices[0].message.content).rule_name;
|
|
639
|
+
let rule = rules.find((rule) => rule.name === ruleName);
|
|
640
|
+
if (!rule) {
|
|
641
|
+
rule = rules.find((rule) => rule.name === '通用视频助手');
|
|
642
|
+
}
|
|
636
643
|
promptContent = rule.prompt;
|
|
637
644
|
// 确保目录存在
|
|
638
645
|
await (0, promises_1.mkdir)((0, node_path_1.dirname)(projectRulesFile), { recursive: true });
|
|
@@ -645,7 +652,6 @@ ${JSON.stringify(rulesList, null, 2)}
|
|
|
645
652
|
text: JSON.stringify({
|
|
646
653
|
type: 'project_rules',
|
|
647
654
|
content: promptContent,
|
|
648
|
-
rulesList,
|
|
649
655
|
projectRulesFile,
|
|
650
656
|
}),
|
|
651
657
|
},
|
|
@@ -725,7 +731,7 @@ server.registerTool('generate-character-image', {
|
|
|
725
731
|
description: 'Generate a turnaround image or portrait for any character.',
|
|
726
732
|
inputSchema: {
|
|
727
733
|
type: zod_1.z
|
|
728
|
-
.enum(['banana', 'banana-pro', 'seedream'])
|
|
734
|
+
.enum(['banana', 'banana-pro', 'seedream', 'seedream-pro'])
|
|
729
735
|
.optional()
|
|
730
736
|
.default('banana'),
|
|
731
737
|
name: zod_1.z.string().describe('The name of the character.'),
|
|
@@ -755,7 +761,9 @@ server.registerTool('generate-character-image', {
|
|
|
755
761
|
.boolean()
|
|
756
762
|
.default(true)
|
|
757
763
|
.describe('是否生成三视图。true: 生成4096x3072的三视图,false: 生成2304x4096的竖版人物正视图'),
|
|
758
|
-
saveToFileName: zod_1.z
|
|
764
|
+
saveToFileName: zod_1.z
|
|
765
|
+
.string()
|
|
766
|
+
.describe('The filename to save. 应该是png文件'),
|
|
759
767
|
},
|
|
760
768
|
}, async ({ type, name, gender, age, appearance, clothing, personality, detail_features, style, saveToFileName, referenceImage, referenceImagePrompt, isTurnaround, }) => {
|
|
761
769
|
try {
|
|
@@ -931,7 +939,7 @@ server.registerTool('generate-line-sketch', {
|
|
|
931
939
|
prompt: zod_1.z.string().describe('The prompt to generate line sketch.'),
|
|
932
940
|
saveToFileName: zod_1.z
|
|
933
941
|
.string()
|
|
934
|
-
.describe('The filename to save the generated line sketch.'),
|
|
942
|
+
.describe('The filename to save the generated line sketch. 应该是png文件'),
|
|
935
943
|
},
|
|
936
944
|
}, async ({ prompt, saveToFileName }) => {
|
|
937
945
|
try {
|
|
@@ -1058,7 +1066,7 @@ server.registerTool('generate-image', {
|
|
|
1058
1066
|
description: `生成图片`,
|
|
1059
1067
|
inputSchema: {
|
|
1060
1068
|
type: zod_1.z
|
|
1061
|
-
.enum(['banana', 'banana-pro', 'seedream'])
|
|
1069
|
+
.enum(['banana', 'banana-pro', 'seedream', 'seedream-pro'])
|
|
1062
1070
|
.optional()
|
|
1063
1071
|
.default('seedream'),
|
|
1064
1072
|
prompt: zod_1.z
|
|
@@ -1118,17 +1126,14 @@ server.registerTool('generate-image', {
|
|
|
1118
1126
|
])
|
|
1119
1127
|
.default('720x1280')
|
|
1120
1128
|
.describe('The size of the image.'),
|
|
1121
|
-
saveToFileName: zod_1.z
|
|
1129
|
+
saveToFileName: zod_1.z
|
|
1130
|
+
.string()
|
|
1131
|
+
.describe('The filename to save. 应该是png文件'),
|
|
1122
1132
|
watermark: zod_1.z
|
|
1123
1133
|
.boolean()
|
|
1124
1134
|
.optional()
|
|
1125
1135
|
.default(false)
|
|
1126
1136
|
.describe('Whether to add watermark to the image.'),
|
|
1127
|
-
optimizePrompt: zod_1.z
|
|
1128
|
-
.boolean()
|
|
1129
|
-
.optional()
|
|
1130
|
-
.default(false)
|
|
1131
|
-
.describe('Whether to optimize the prompt.'),
|
|
1132
1137
|
referenceImages: zod_1.z
|
|
1133
1138
|
.array(zod_1.z.object({
|
|
1134
1139
|
image: zod_1.z.string().describe('Local image file path'),
|
|
@@ -1157,7 +1162,7 @@ server.registerTool('generate-image', {
|
|
|
1157
1162
|
\`\`\`
|
|
1158
1163
|
`),
|
|
1159
1164
|
},
|
|
1160
|
-
}, async ({ type = 'seedream', prompt, sceneIndex, storyBoardFile = 'storyboard.json', skipConsistencyCheck = false, size = '720x1280', saveToFileName, watermark, referenceImages,
|
|
1165
|
+
}, async ({ type = 'seedream', prompt, sceneIndex, storyBoardFile = 'storyboard.json', skipConsistencyCheck = false, size = '720x1280', saveToFileName, watermark, referenceImages, }) => {
|
|
1161
1166
|
try {
|
|
1162
1167
|
// 验证session状态
|
|
1163
1168
|
const currentSession = await validateSession('generate-image');
|
|
@@ -1251,52 +1256,70 @@ server.registerTool('generate-image', {
|
|
|
1251
1256
|
// 检查并替换英文单引号包裹的中文内容为中文双引号
|
|
1252
1257
|
// 这样才能让 seedream 生成更好的中文文字
|
|
1253
1258
|
let processedPrompt = prompt.replace(/'([^']*[\u4e00-\u9fff][^']*)'/g, '“$1”');
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1259
|
+
try {
|
|
1260
|
+
const ai = currentSession.ai;
|
|
1261
|
+
const promptOptimizer = await (0, promises_1.readFile)((0, node_path_1.resolve)(__dirname, './prompts/image-prompt-optimizer.md'), 'utf8');
|
|
1262
|
+
const schema = {
|
|
1263
|
+
name: 'optimize_image_prompt',
|
|
1264
|
+
schema: {
|
|
1265
|
+
type: 'object',
|
|
1266
|
+
properties: {
|
|
1267
|
+
prompt_optimized: {
|
|
1268
|
+
type: 'string',
|
|
1269
|
+
description: '优化后的提示词',
|
|
1264
1270
|
},
|
|
1265
|
-
{
|
|
1266
|
-
|
|
1267
|
-
|
|
1271
|
+
metaphor_modifiers: {
|
|
1272
|
+
type: 'array',
|
|
1273
|
+
description: '从 prompt_optimized 中抽取的所有比喻修饰词(字符串数组)',
|
|
1274
|
+
items: {
|
|
1275
|
+
type: 'string',
|
|
1276
|
+
description: '比喻性修饰词,例如 “如羽毛般轻盈”、“像晨雾一样柔和”',
|
|
1277
|
+
},
|
|
1268
1278
|
},
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
}
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1279
|
+
},
|
|
1280
|
+
required: ['prompt_optimized', 'metaphor_modifiers'],
|
|
1281
|
+
},
|
|
1282
|
+
};
|
|
1283
|
+
const completion = await ai.getCompletions({
|
|
1284
|
+
model: 'Doubao-Seed-1.6',
|
|
1285
|
+
messages: [
|
|
1286
|
+
{
|
|
1287
|
+
role: 'system',
|
|
1288
|
+
content: promptOptimizer,
|
|
1289
|
+
},
|
|
1290
|
+
{
|
|
1291
|
+
role: 'user',
|
|
1292
|
+
content: `## 用户指令
|
|
1293
|
+
|
|
1294
|
+
${processedPrompt.trim()}
|
|
1295
|
+
|
|
1296
|
+
## 参考图
|
|
1297
|
+
|
|
1298
|
+
${referenceImages?.map((ref, index) => `图${index + 1}:${ref.image}`).join('\n') || '无'}`,
|
|
1299
|
+
},
|
|
1300
|
+
],
|
|
1301
|
+
response_format: {
|
|
1302
|
+
type: 'json_schema',
|
|
1303
|
+
json_schema: schema,
|
|
1304
|
+
},
|
|
1305
|
+
});
|
|
1306
|
+
const optimizedPrompt = completion.choices[0]?.message?.content.trim();
|
|
1307
|
+
if (optimizedPrompt) {
|
|
1308
|
+
try {
|
|
1309
|
+
const { prompt_optimized, metaphor_modifiers } = JSON.parse(optimizedPrompt);
|
|
1310
|
+
processedPrompt = `${prompt_optimized}`;
|
|
1311
|
+
if (metaphor_modifiers?.length) {
|
|
1312
|
+
processedPrompt += `\n\n注意:下面这些是形象比喻,并不是输出内容。\n${metaphor_modifiers}`;
|
|
1294
1313
|
}
|
|
1295
1314
|
}
|
|
1315
|
+
catch (ex) {
|
|
1316
|
+
console.error('Failed to parse optimized prompt:', ex);
|
|
1317
|
+
processedPrompt = optimizedPrompt;
|
|
1318
|
+
}
|
|
1296
1319
|
}
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1320
|
+
}
|
|
1321
|
+
catch (error) {
|
|
1322
|
+
console.error('Failed to optimize prompt:', error);
|
|
1300
1323
|
}
|
|
1301
1324
|
console.log(`Generating image with prompt: ${processedPrompt.substring(0, 100)}...`);
|
|
1302
1325
|
// 处理参考图片
|
|
@@ -1426,12 +1449,14 @@ server.registerTool('edit-image', {
|
|
|
1426
1449
|
inputSchema: {
|
|
1427
1450
|
prompt: zod_1.z.string().describe('要编辑图片的中文提示词'),
|
|
1428
1451
|
type: zod_1.z
|
|
1429
|
-
.enum(['banana-pro', 'banana', 'seedream'])
|
|
1452
|
+
.enum(['banana-pro', 'banana', 'seedream', 'seedream-pro'])
|
|
1430
1453
|
.optional()
|
|
1431
1454
|
.default('seedream')
|
|
1432
1455
|
.describe('The type of image model to use.'),
|
|
1433
1456
|
sourceImageFileName: zod_1.z.string().describe('The source image file name.'),
|
|
1434
|
-
saveToFileName: zod_1.z
|
|
1457
|
+
saveToFileName: zod_1.z
|
|
1458
|
+
.string()
|
|
1459
|
+
.describe('The filename to save. 应该是png文件'),
|
|
1435
1460
|
size: zod_1.z
|
|
1436
1461
|
.enum([
|
|
1437
1462
|
'1024x1024',
|
|
@@ -1595,7 +1620,9 @@ server.registerTool('generate-video', {
|
|
|
1595
1620
|
])
|
|
1596
1621
|
.default('lite')
|
|
1597
1622
|
.describe('除非用户明确提出使用其他模型,否则一律用lite模型;zero 系列模型适合创作8-23秒带故事情节的短片'),
|
|
1598
|
-
saveToFileName: zod_1.z
|
|
1623
|
+
saveToFileName: zod_1.z
|
|
1624
|
+
.string()
|
|
1625
|
+
.describe('The filename to save. 应该是mp4文件'),
|
|
1599
1626
|
start_frame: zod_1.z
|
|
1600
1627
|
.string()
|
|
1601
1628
|
.optional()
|
|
@@ -2002,7 +2029,7 @@ server.registerTool('generate-video', {
|
|
|
2002
2029
|
console.warn('Failed to send progress update:', progressError);
|
|
2003
2030
|
}
|
|
2004
2031
|
},
|
|
2005
|
-
waitForFinish:
|
|
2032
|
+
waitForFinish: type !== 'zero',
|
|
2006
2033
|
});
|
|
2007
2034
|
if (!res) {
|
|
2008
2035
|
throw new Error('Failed to generate video: no response from AI service');
|
|
@@ -2088,7 +2115,7 @@ server.registerTool('wait-for-task-finish', {
|
|
|
2088
2115
|
.describe('The taskUrl of the video task to wait for.'),
|
|
2089
2116
|
saveToFileName: zod_1.z
|
|
2090
2117
|
.string()
|
|
2091
|
-
.describe('The file name to save the video to.'),
|
|
2118
|
+
.describe('The file name to save the video to. 应该是mp4文件'),
|
|
2092
2119
|
},
|
|
2093
2120
|
}, async ({ taskUrl, saveToFileName }, context) => {
|
|
2094
2121
|
try {
|
|
@@ -2163,7 +2190,7 @@ server.registerTool('generate-sound-effect', {
|
|
|
2163
2190
|
.describe('The duration of the sound which will be generated in seconds. Must be at least 0.5 and at most 30. If set to None we will guess the optimal duration using the prompt. Defaults to None.'),
|
|
2164
2191
|
saveToFileName: zod_1.z
|
|
2165
2192
|
.string()
|
|
2166
|
-
.describe('The filename to save.
|
|
2193
|
+
.describe('The filename to save. 应该是mp3文件'),
|
|
2167
2194
|
},
|
|
2168
2195
|
}, async ({ prompt_in_english, loop, saveToFileName, duration_seconds }) => {
|
|
2169
2196
|
try {
|
|
@@ -2210,13 +2237,22 @@ server.registerTool('generate-sound-effect', {
|
|
|
2210
2237
|
return createErrorResponse(error, 'generate-sound-effect');
|
|
2211
2238
|
}
|
|
2212
2239
|
});
|
|
2213
|
-
server.registerTool('generate-music', {
|
|
2214
|
-
title: '
|
|
2215
|
-
description: '
|
|
2240
|
+
server.registerTool('generate-music-or-mv', {
|
|
2241
|
+
title: '创作音乐(Music)或音乐视频(Music Video)',
|
|
2242
|
+
description: '生成音乐,包括MV(music video)、BGM 或 歌曲',
|
|
2216
2243
|
inputSchema: {
|
|
2217
2244
|
prompt: zod_1.z.string().describe('The prompt to generate.'),
|
|
2245
|
+
singerPhoto: zod_1.z
|
|
2246
|
+
.string()
|
|
2247
|
+
.optional()
|
|
2248
|
+
.describe('The singer photo to use. 只有type为music_video的时候才生效,也可以不传,模型会自动生成'),
|
|
2249
|
+
mvOrientation: zod_1.z
|
|
2250
|
+
.enum(['portrait', 'landscape'])
|
|
2251
|
+
.optional()
|
|
2252
|
+
.describe('The orientation of the music video. Defaults to portrait.')
|
|
2253
|
+
.default('portrait'),
|
|
2218
2254
|
type: zod_1.z
|
|
2219
|
-
.enum(['bgm', 'song'])
|
|
2255
|
+
.enum(['bgm', 'song', 'music_video'])
|
|
2220
2256
|
.describe('The type of music. Defaults to background music.')
|
|
2221
2257
|
.default('bgm'),
|
|
2222
2258
|
model: zod_1.z
|
|
@@ -2233,9 +2269,11 @@ server.registerTool('generate-music', {
|
|
|
2233
2269
|
.boolean()
|
|
2234
2270
|
.default(false)
|
|
2235
2271
|
.describe('Whether to skip copyright check.'),
|
|
2236
|
-
saveToFileName: zod_1.z
|
|
2272
|
+
saveToFileName: zod_1.z
|
|
2273
|
+
.string()
|
|
2274
|
+
.describe('The filename to save. 应该是mp3文件'),
|
|
2237
2275
|
},
|
|
2238
|
-
}, async ({ prompt, type, model, duration, skipCopyCheck, saveToFileName }, context) => {
|
|
2276
|
+
}, async ({ prompt, singerPhoto, mvOrientation, type, model, duration, skipCopyCheck, saveToFileName, }, context) => {
|
|
2239
2277
|
try {
|
|
2240
2278
|
// 验证session状态
|
|
2241
2279
|
const currentSession = await validateSession('generate-music');
|
|
@@ -2246,24 +2284,47 @@ server.registerTool('generate-music', {
|
|
|
2246
2284
|
if (type === 'bgm' && duration > 120) {
|
|
2247
2285
|
throw new Error('BGM duration must be at most 120 seconds.');
|
|
2248
2286
|
}
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2287
|
+
let res;
|
|
2288
|
+
if (type === 'music_video') {
|
|
2289
|
+
const singer_photo = singerPhoto
|
|
2290
|
+
? await getMaterialUri(currentSession, singerPhoto)
|
|
2291
|
+
: undefined;
|
|
2292
|
+
res = await ai.generateZeroCutMusicVideo({
|
|
2293
|
+
prompt: `${prompt.trim()} 音乐时长${duration}秒`,
|
|
2294
|
+
singerPhoto: singer_photo,
|
|
2295
|
+
orientation: mvOrientation,
|
|
2296
|
+
resolution: '720p',
|
|
2297
|
+
onProgress: async (metaData) => {
|
|
2298
|
+
try {
|
|
2299
|
+
await sendProgress(context, metaData.Result?.Progress ?? ++progress, metaData.Result?.Progress ? 100 : undefined, JSON.stringify(metaData));
|
|
2300
|
+
}
|
|
2301
|
+
catch (progressError) {
|
|
2302
|
+
console.warn('Failed to send progress update:', progressError);
|
|
2303
|
+
}
|
|
2304
|
+
},
|
|
2305
|
+
waitForFinish: false,
|
|
2306
|
+
});
|
|
2307
|
+
}
|
|
2308
|
+
else {
|
|
2309
|
+
const finalPrompt = `${prompt.trim()} ${type === 'bgm' ? `纯音乐无歌词,时长${duration}秒` : `时长${duration}秒,使用${model}模型`}`;
|
|
2310
|
+
res = await ai.generateMusic({
|
|
2311
|
+
prompt: finalPrompt,
|
|
2312
|
+
skipCopyCheck,
|
|
2313
|
+
onProgress: async (metaData) => {
|
|
2314
|
+
try {
|
|
2315
|
+
await sendProgress(context, metaData.Result?.Progress ?? ++progress, metaData.Result?.Progress ? 100 : undefined, JSON.stringify(metaData));
|
|
2316
|
+
}
|
|
2317
|
+
catch (progressError) {
|
|
2318
|
+
console.warn('Failed to send progress update:', progressError);
|
|
2319
|
+
}
|
|
2320
|
+
},
|
|
2321
|
+
});
|
|
2322
|
+
}
|
|
2262
2323
|
if (!res) {
|
|
2263
|
-
throw new Error('Failed to generate
|
|
2324
|
+
throw new Error('Failed to generate Music: no response from AI service');
|
|
2264
2325
|
}
|
|
2265
2326
|
if (res.url) {
|
|
2266
|
-
console.log('
|
|
2327
|
+
console.log('Music generated successfully, saving to materials...');
|
|
2267
2328
|
const uri = await saveMaterial(currentSession, res.url, validatedFileName);
|
|
2268
2329
|
const { url, duration: bgmDuration, captions, ...opts } = res;
|
|
2269
2330
|
// 保存captions到本地
|
|
@@ -2301,15 +2362,29 @@ server.registerTool('generate-music', {
|
|
|
2301
2362
|
],
|
|
2302
2363
|
};
|
|
2303
2364
|
}
|
|
2365
|
+
else if (res.taskUrl) {
|
|
2366
|
+
return {
|
|
2367
|
+
content: [
|
|
2368
|
+
{
|
|
2369
|
+
type: 'text',
|
|
2370
|
+
text: JSON.stringify({
|
|
2371
|
+
success: true,
|
|
2372
|
+
message: '该视频生成任务正在运行中,它是异步任务,且执行时间较长,你应立即调用工具 wait-for-task-finish 来等待任务结束,如该工具调用超时,你应立即再次重新调用直到任务结束。',
|
|
2373
|
+
taskUrl: res.taskUrl,
|
|
2374
|
+
}),
|
|
2375
|
+
},
|
|
2376
|
+
],
|
|
2377
|
+
};
|
|
2378
|
+
}
|
|
2304
2379
|
else {
|
|
2305
|
-
console.warn('
|
|
2380
|
+
console.warn('Music generation completed but no URL returned');
|
|
2306
2381
|
return {
|
|
2307
2382
|
content: [
|
|
2308
2383
|
{
|
|
2309
2384
|
type: 'text',
|
|
2310
2385
|
text: JSON.stringify({
|
|
2311
2386
|
success: false,
|
|
2312
|
-
error: 'No
|
|
2387
|
+
error: 'No Music URL returned from AI service',
|
|
2313
2388
|
response: res,
|
|
2314
2389
|
timestamp: new Date().toISOString(),
|
|
2315
2390
|
}),
|
|
@@ -2319,7 +2394,7 @@ server.registerTool('generate-music', {
|
|
|
2319
2394
|
}
|
|
2320
2395
|
}
|
|
2321
2396
|
catch (error) {
|
|
2322
|
-
return createErrorResponse(error, 'generate-
|
|
2397
|
+
return createErrorResponse(error, 'generate-music');
|
|
2323
2398
|
}
|
|
2324
2399
|
});
|
|
2325
2400
|
server.registerTool('generate-scene-tts', {
|
|
@@ -2346,7 +2421,9 @@ server.registerTool('generate-scene-tts', {
|
|
|
2346
2421
|
.string()
|
|
2347
2422
|
.optional()
|
|
2348
2423
|
.describe('跳过校验的理由,如果skipConsistencyCheck设为true,必须要传这个参数'),
|
|
2349
|
-
saveToFileName: zod_1.z
|
|
2424
|
+
saveToFileName: zod_1.z
|
|
2425
|
+
.string()
|
|
2426
|
+
.describe('The filename to save. 应该是mp3文件'),
|
|
2350
2427
|
speed: zod_1.z
|
|
2351
2428
|
.number()
|
|
2352
2429
|
.min(0.5)
|
|
@@ -2917,7 +2994,7 @@ server.registerTool('voice-design', {
|
|
|
2917
2994
|
previewText: zod_1.z.string().describe('The preview text to design the voice.'),
|
|
2918
2995
|
saveToFileName: zod_1.z
|
|
2919
2996
|
.string()
|
|
2920
|
-
.describe('The file name to save the designed voice.'),
|
|
2997
|
+
.describe('The file name to save the designed voice. 应该是mp3文件'),
|
|
2921
2998
|
},
|
|
2922
2999
|
}, async ({ prompt, previewText, saveToFileName }) => {
|
|
2923
3000
|
try {
|
|
@@ -3393,7 +3470,7 @@ server.registerTool('audio-video-sync', {
|
|
|
3393
3470
|
.describe('The reference photo face for lip sync.'),
|
|
3394
3471
|
saveToFileName: zod_1.z
|
|
3395
3472
|
.string()
|
|
3396
|
-
.describe('The filename to save the audio-video-synced video.'),
|
|
3473
|
+
.describe('The filename to save the audio-video-synced video. 应该是mp4文件'),
|
|
3397
3474
|
},
|
|
3398
3475
|
}, async ({ lipSync, lipSyncType, lipSyncPadAudio, videoFileName, audioFileName, audioInMs, refPhotoFileName, saveToFileName, }, context) => {
|
|
3399
3476
|
try {
|
|
@@ -3582,7 +3659,7 @@ server.registerTool('generate-video-by-ref', {
|
|
|
3582
3659
|
.describe('Whether to mute the video (effective for sora2 and veo3.1).'),
|
|
3583
3660
|
saveToFileName: zod_1.z
|
|
3584
3661
|
.string()
|
|
3585
|
-
.describe('The filename to save the generated video.'),
|
|
3662
|
+
.describe('The filename to save the generated video. 应该是mp4文件'),
|
|
3586
3663
|
sceneIndex: zod_1.z
|
|
3587
3664
|
.number()
|
|
3588
3665
|
.min(1)
|
|
@@ -3788,7 +3865,54 @@ server.registerTool('generate-video-by-ref', {
|
|
|
3788
3865
|
if (promptPrefix) {
|
|
3789
3866
|
promptPrefix += '\n';
|
|
3790
3867
|
}
|
|
3791
|
-
|
|
3868
|
+
let finalPrompt = `${promptPrefix}${prompt}`;
|
|
3869
|
+
if (type === 'pixv') {
|
|
3870
|
+
const completion = await ai.getCompletions({
|
|
3871
|
+
model: 'Doubao-Seed-1.6',
|
|
3872
|
+
messages: [
|
|
3873
|
+
{
|
|
3874
|
+
role: 'system',
|
|
3875
|
+
content: `你根据主体信息,优化用户指令,使描述中的内容正确引用主体名称。
|
|
3876
|
+
|
|
3877
|
+
具体方式为,将用户指令中引用主体信息中主体名称的部分,用 “@主体名” 的形式替代,注意它和前后内容之间也需要用**空格**分隔。
|
|
3878
|
+
|
|
3879
|
+
## 例子
|
|
3880
|
+
|
|
3881
|
+
### 输入:
|
|
3882
|
+
|
|
3883
|
+
主体信息
|
|
3884
|
+
[
|
|
3885
|
+
{"type": "subject", "fileName": "dog.png", "ref_name": "狗"},
|
|
3886
|
+
{"type": "background", "fileName": "room.png", "ref_name": "房间"}
|
|
3887
|
+
]
|
|
3888
|
+
|
|
3889
|
+
用户指令
|
|
3890
|
+
一只狗在房间里玩耍
|
|
3891
|
+
|
|
3892
|
+
### 输出:
|
|
3893
|
+
一只 @狗 在 @房间 里玩耍
|
|
3894
|
+
|
|
3895
|
+
---
|
|
3896
|
+
|
|
3897
|
+
## 要求与约束
|
|
3898
|
+
|
|
3899
|
+
只输出替换主体名后的用户指令,不要输出其他任何额外内容
|
|
3900
|
+
`,
|
|
3901
|
+
},
|
|
3902
|
+
{
|
|
3903
|
+
role: 'user',
|
|
3904
|
+
content: `## 主体信息
|
|
3905
|
+
|
|
3906
|
+
${JSON.stringify(referenceImages)}
|
|
3907
|
+
|
|
3908
|
+
## 用户指令
|
|
3909
|
+
|
|
3910
|
+
${prompt.trim()}`,
|
|
3911
|
+
},
|
|
3912
|
+
],
|
|
3913
|
+
});
|
|
3914
|
+
finalPrompt = completion.choices[0]?.message?.content.trim();
|
|
3915
|
+
}
|
|
3792
3916
|
// 调用 referencesToVideo 函数
|
|
3793
3917
|
const result = await currentSession.ai.referencesToVideo({
|
|
3794
3918
|
prompt: finalPrompt,
|
|
@@ -3892,7 +4016,7 @@ server.registerTool('extend-video-duration', {
|
|
|
3892
4016
|
.describe('Optional end frame image file name in materials directory to guide the video extension.'),
|
|
3893
4017
|
saveToFileName: zod_1.z
|
|
3894
4018
|
.string()
|
|
3895
|
-
.describe('The filename to save the extended video.'),
|
|
4019
|
+
.describe('The filename to save the extended video. 应该是mp4文件'),
|
|
3896
4020
|
},
|
|
3897
4021
|
}, async ({ videoFileName, duration, resolution, prompt, type = 'turbo', endFrame, saveToFileName, }, context) => {
|
|
3898
4022
|
try {
|
|
@@ -3988,11 +4112,11 @@ server.registerTool('use-template', {
|
|
|
3988
4112
|
.describe('Optional materials to use in the template.'),
|
|
3989
4113
|
saveToFileName: zod_1.z
|
|
3990
4114
|
.string()
|
|
3991
|
-
.describe('The filename to save the generated material.'),
|
|
4115
|
+
.describe('The filename to save the generated material. 根据用户具体需求,应该是mp4或png文件'),
|
|
3992
4116
|
},
|
|
3993
4117
|
}, async ({ user_request, saveToFileName, materials }) => {
|
|
3994
4118
|
try {
|
|
3995
|
-
const currentSession = await validateSession('
|
|
4119
|
+
const currentSession = await validateSession('use-template');
|
|
3996
4120
|
const ai = currentSession.ai;
|
|
3997
4121
|
const data = await ai.listTemplates('all');
|
|
3998
4122
|
const templates = data.map(item => ({
|
|
@@ -4007,7 +4131,7 @@ server.registerTool('use-template', {
|
|
|
4007
4131
|
messages: [
|
|
4008
4132
|
{
|
|
4009
4133
|
role: 'system',
|
|
4010
|
-
content:
|
|
4134
|
+
content: `你根据用户需求,分析需求与模板描述(description)和触发器(trigger)的匹配程度,从以下模板中选择一个匹配的模板,返回模板ID:\n\n${JSON.stringify(templates)}\n\n**约束**:只输出模板ID,不需要其他解释,如果没有匹配的模版,输出"无匹配模版"`,
|
|
4011
4135
|
},
|
|
4012
4136
|
{
|
|
4013
4137
|
role: 'user',
|