cerevox 3.0.0-beta.1 → 3.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"zerocut.d.ts","sourceRoot":"","sources":["../../../src/mcp/servers/zerocut.ts"],"names":[],"mappings":";AA4+KA,wBAAsB,GAAG,kBAKxB"}
1
+ {"version":3,"file":"zerocut.d.ts","sourceRoot":"","sources":["../../../src/mcp/servers/zerocut.ts"],"names":[],"mappings":";AA6nLA,wBAAsB,GAAG,kBAKxB"}
@@ -552,11 +552,17 @@ server.registerTool('retrieve-rules-context', {
552
552
  title: 'ZeroCut Basic Rules',
553
553
  description: `ZeroCut-${constants_1.VERSION} Basic Rules`,
554
554
  inputSchema: {
555
- request: zod_1.z.string().describe(`用户的具体的指令`),
555
+ request: zod_1.z.string().describe(`根据用户的具体的指令召回规则上下文,具体指令为通常为如下格式之一:
556
+
557
+ - 启用xxx模式
558
+ - 使用xxx助手
559
+ - 启用xxx角色
560
+ - 切换上下文为xxx
561
+ - 召回原规则上下文(配合type:memo使用)
562
+ `),
556
563
  type: zod_1.z
557
- .enum(['retrieve', 'switch'])
558
- .default('retrieve')
559
- .describe('retrieve: 根据需要重新召回规则上下文\nswitch: 根据用户指令,切换规则上下文'),
564
+ .enum(['new', 'memo', 'switch'])
565
+ .describe('new: 新创建规则上下文\nmemo: 根据需要重新召回规则上下文(记忆)\nswitch: 根据用户指令,切换规则上下文'),
560
566
  },
561
567
  }, async ({ request, type }) => {
562
568
  const currentSession = await validateSession('retrieve-rules-context', false);
@@ -580,59 +586,60 @@ server.registerTool('retrieve-rules-context', {
580
586
  }
581
587
  if (type === 'switch') {
582
588
  (0, node_fs_1.rmSync)(projectRulesFile, { force: true });
589
+ (0, node_fs_1.rmSync)(customRulesFile, { force: true });
583
590
  promptContent = '';
584
591
  }
585
592
  try {
586
- const ai = currentSession.ai;
587
- const { data: rules } = await ai.listContextRules();
588
- const rulesList = rules.map((rule) => ({
589
- name: rule.name,
590
- trigger: rule.trigger,
591
- }));
592
- const chooseRulePrompt = `
593
- 请分析用户具体指令,匹配可用规则中的trigger,选择最合适的规则。
593
+ if (!promptContent) {
594
+ const ai = currentSession.ai;
595
+ const { data: rules } = await ai.listContextRules();
596
+ const rulesList = rules.map((rule) => ({
597
+ name: rule.name,
598
+ trigger: rule.trigger,
599
+ }));
600
+ const chooseRulePrompt = `
601
+ 请分析用户具体指令,匹配可用规则中的trigger,选择最合适的规则。
594
602
 
595
- ## 可用规则:
596
- ${JSON.stringify(rulesList, null, 2)}
603
+ ## 可用规则:
604
+ ${JSON.stringify(rulesList, null, 2)}
597
605
 
598
- 请返回一个 JSON 字符串,包含用户意图对应的规则名称。格式为:{"rule_name": "规则名称"}
599
- `;
600
- const schema = {
601
- name: 'choose_rule',
602
- schema: {
603
- type: 'object',
604
- properties: {
605
- rule_name: {
606
- type: 'string',
607
- description: '用户意图对应的规则名称',
606
+ 请返回一个 JSON 字符串,包含用户意图对应的规则名称。格式为:{"rule_name": "规则名称"}
607
+ `;
608
+ const schema = {
609
+ name: 'choose_rule',
610
+ schema: {
611
+ type: 'object',
612
+ properties: {
613
+ rule_name: {
614
+ type: 'string',
615
+ description: '用户意图对应的规则名称',
616
+ },
608
617
  },
618
+ required: ['rule_name'],
609
619
  },
610
- required: ['rule_name'],
611
- },
612
- };
613
- const completion = await ai.getCompletions({
614
- model: 'Doubao-Seed-1.6-flash',
615
- messages: [
616
- {
617
- role: 'system',
618
- content: chooseRulePrompt,
619
- },
620
- {
621
- role: 'user',
622
- content: request,
620
+ };
621
+ const completion = await ai.getCompletions({
622
+ model: 'Doubao-Seed-1.6-flash',
623
+ messages: [
624
+ {
625
+ role: 'system',
626
+ content: chooseRulePrompt,
627
+ },
628
+ {
629
+ role: 'user',
630
+ content: request,
631
+ },
632
+ ],
633
+ response_format: {
634
+ type: 'json_schema',
635
+ json_schema: schema,
623
636
  },
624
- ],
625
- response_format: {
626
- type: 'json_schema',
627
- json_schema: schema,
628
- },
629
- });
630
- const ruleName = JSON.parse(completion.choices[0].message.content).rule_name;
631
- let rule = rules.find((rule) => rule.name === ruleName);
632
- if (!rule) {
633
- rule = rules.find((rule) => rule.name === '通用视频助手');
634
- }
635
- if (!promptContent) {
637
+ });
638
+ const ruleName = JSON.parse(completion.choices[0].message.content).rule_name;
639
+ let rule = rules.find((rule) => rule.name === ruleName);
640
+ if (!rule) {
641
+ rule = rules.find((rule) => rule.name === '通用视频助手');
642
+ }
636
643
  promptContent = rule.prompt;
637
644
  // 确保目录存在
638
645
  await (0, promises_1.mkdir)((0, node_path_1.dirname)(projectRulesFile), { recursive: true });
@@ -645,7 +652,6 @@ ${JSON.stringify(rulesList, null, 2)}
645
652
  text: JSON.stringify({
646
653
  type: 'project_rules',
647
654
  content: promptContent,
648
- rulesList,
649
655
  projectRulesFile,
650
656
  }),
651
657
  },
@@ -725,7 +731,7 @@ server.registerTool('generate-character-image', {
725
731
  description: 'Generate a turnaround image or portrait for any character.',
726
732
  inputSchema: {
727
733
  type: zod_1.z
728
- .enum(['banana', 'banana-pro', 'seedream'])
734
+ .enum(['banana', 'banana-pro', 'seedream', 'seedream-pro'])
729
735
  .optional()
730
736
  .default('banana'),
731
737
  name: zod_1.z.string().describe('The name of the character.'),
@@ -755,7 +761,9 @@ server.registerTool('generate-character-image', {
755
761
  .boolean()
756
762
  .default(true)
757
763
  .describe('是否生成三视图。true: 生成4096x3072的三视图,false: 生成2304x4096的竖版人物正视图'),
758
- saveToFileName: zod_1.z.string().describe('The filename to save.'),
764
+ saveToFileName: zod_1.z
765
+ .string()
766
+ .describe('The filename to save. 应该是png文件'),
759
767
  },
760
768
  }, async ({ type, name, gender, age, appearance, clothing, personality, detail_features, style, saveToFileName, referenceImage, referenceImagePrompt, isTurnaround, }) => {
761
769
  try {
@@ -931,7 +939,7 @@ server.registerTool('generate-line-sketch', {
931
939
  prompt: zod_1.z.string().describe('The prompt to generate line sketch.'),
932
940
  saveToFileName: zod_1.z
933
941
  .string()
934
- .describe('The filename to save the generated line sketch.'),
942
+ .describe('The filename to save the generated line sketch. 应该是png文件'),
935
943
  },
936
944
  }, async ({ prompt, saveToFileName }) => {
937
945
  try {
@@ -1058,7 +1066,7 @@ server.registerTool('generate-image', {
1058
1066
  description: `生成图片`,
1059
1067
  inputSchema: {
1060
1068
  type: zod_1.z
1061
- .enum(['banana', 'banana-pro', 'seedream'])
1069
+ .enum(['banana', 'banana-pro', 'seedream', 'seedream-pro'])
1062
1070
  .optional()
1063
1071
  .default('seedream'),
1064
1072
  prompt: zod_1.z
@@ -1118,17 +1126,14 @@ server.registerTool('generate-image', {
1118
1126
  ])
1119
1127
  .default('720x1280')
1120
1128
  .describe('The size of the image.'),
1121
- saveToFileName: zod_1.z.string().describe('The filename to save.'),
1129
+ saveToFileName: zod_1.z
1130
+ .string()
1131
+ .describe('The filename to save. 应该是png文件'),
1122
1132
  watermark: zod_1.z
1123
1133
  .boolean()
1124
1134
  .optional()
1125
1135
  .default(false)
1126
1136
  .describe('Whether to add watermark to the image.'),
1127
- optimizePrompt: zod_1.z
1128
- .boolean()
1129
- .optional()
1130
- .default(false)
1131
- .describe('Whether to optimize the prompt.'),
1132
1137
  referenceImages: zod_1.z
1133
1138
  .array(zod_1.z.object({
1134
1139
  image: zod_1.z.string().describe('Local image file path'),
@@ -1157,7 +1162,7 @@ server.registerTool('generate-image', {
1157
1162
  \`\`\`
1158
1163
  `),
1159
1164
  },
1160
- }, async ({ type = 'seedream', prompt, sceneIndex, storyBoardFile = 'storyboard.json', skipConsistencyCheck = false, size = '720x1280', saveToFileName, watermark, referenceImages, optimizePrompt, }) => {
1165
+ }, async ({ type = 'seedream', prompt, sceneIndex, storyBoardFile = 'storyboard.json', skipConsistencyCheck = false, size = '720x1280', saveToFileName, watermark, referenceImages, }) => {
1161
1166
  try {
1162
1167
  // 验证session状态
1163
1168
  const currentSession = await validateSession('generate-image');
@@ -1251,52 +1256,70 @@ server.registerTool('generate-image', {
1251
1256
  // 检查并替换英文单引号包裹的中文内容为中文双引号
1252
1257
  // 这样才能让 seedream 生成更好的中文文字
1253
1258
  let processedPrompt = prompt.replace(/'([^']*[\u4e00-\u9fff][^']*)'/g, '“$1”');
1254
- if (optimizePrompt) {
1255
- try {
1256
- const ai = currentSession.ai;
1257
- const promptOptimizer = await (0, promises_1.readFile)((0, node_path_1.resolve)(__dirname, './prompts/image-prompt-optimizer.md'), 'utf8');
1258
- const completion = await ai.getCompletions({
1259
- model: 'Doubao-Seed-1.6-flash',
1260
- messages: [
1261
- {
1262
- role: 'system',
1263
- content: promptOptimizer,
1259
+ try {
1260
+ const ai = currentSession.ai;
1261
+ const promptOptimizer = await (0, promises_1.readFile)((0, node_path_1.resolve)(__dirname, './prompts/image-prompt-optimizer.md'), 'utf8');
1262
+ const schema = {
1263
+ name: 'optimize_image_prompt',
1264
+ schema: {
1265
+ type: 'object',
1266
+ properties: {
1267
+ prompt_optimized: {
1268
+ type: 'string',
1269
+ description: '优化后的提示词',
1264
1270
  },
1265
- {
1266
- role: 'user',
1267
- content: processedPrompt.trim(),
1271
+ metaphor_modifiers: {
1272
+ type: 'array',
1273
+ description: '从 prompt_optimized 中抽取的所有比喻修饰词(字符串数组)',
1274
+ items: {
1275
+ type: 'string',
1276
+ description: '比喻性修饰词,例如 “如羽毛般轻盈”、“像晨雾一样柔和”',
1277
+ },
1268
1278
  },
1269
- ],
1270
- });
1271
- let optimizedPrompt = completion.choices[0]?.message?.content.trim();
1272
- if (optimizedPrompt) {
1273
- if (optimizedPrompt.startsWith('```json')) {
1274
- // 提取 JSON 代码块中的内容
1275
- const jsonMatch = optimizedPrompt.match(/```json\s*([\s\S]*?)\s*```/);
1276
- if (jsonMatch && jsonMatch[1]) {
1277
- optimizedPrompt = jsonMatch[1];
1278
- }
1279
- }
1280
- if (optimizedPrompt.startsWith('{')) {
1281
- try {
1282
- const { prompt_optimized, metaphor_modifiers } = JSON.parse(optimizedPrompt);
1283
- processedPrompt = `${prompt_optimized}`;
1284
- if (metaphor_modifiers?.length) {
1285
- processedPrompt += `\n\n注意:下面这些是形象比喻,并不是输出内容。\n${metaphor_modifiers}`;
1286
- }
1287
- }
1288
- catch (ex) {
1289
- processedPrompt = optimizedPrompt;
1290
- }
1291
- }
1292
- else {
1293
- processedPrompt = optimizedPrompt;
1279
+ },
1280
+ required: ['prompt_optimized', 'metaphor_modifiers'],
1281
+ },
1282
+ };
1283
+ const completion = await ai.getCompletions({
1284
+ model: 'Doubao-Seed-1.6',
1285
+ messages: [
1286
+ {
1287
+ role: 'system',
1288
+ content: promptOptimizer,
1289
+ },
1290
+ {
1291
+ role: 'user',
1292
+ content: `## 用户指令
1293
+
1294
+ ${processedPrompt.trim()}
1295
+
1296
+ ## 参考图
1297
+
1298
+ ${referenceImages?.map((ref, index) => `图${index + 1}:${ref.image}`).join('\n') || '无'}`,
1299
+ },
1300
+ ],
1301
+ response_format: {
1302
+ type: 'json_schema',
1303
+ json_schema: schema,
1304
+ },
1305
+ });
1306
+ const optimizedPrompt = completion.choices[0]?.message?.content.trim();
1307
+ if (optimizedPrompt) {
1308
+ try {
1309
+ const { prompt_optimized, metaphor_modifiers } = JSON.parse(optimizedPrompt);
1310
+ processedPrompt = `${prompt_optimized}`;
1311
+ if (metaphor_modifiers?.length) {
1312
+ processedPrompt += `\n\n注意:下面这些是形象比喻,并不是输出内容。\n${metaphor_modifiers}`;
1294
1313
  }
1295
1314
  }
1315
+ catch (ex) {
1316
+ console.error('Failed to parse optimized prompt:', ex);
1317
+ processedPrompt = optimizedPrompt;
1318
+ }
1296
1319
  }
1297
- catch (error) {
1298
- console.error('Failed to optimize prompt:', error);
1299
- }
1320
+ }
1321
+ catch (error) {
1322
+ console.error('Failed to optimize prompt:', error);
1300
1323
  }
1301
1324
  console.log(`Generating image with prompt: ${processedPrompt.substring(0, 100)}...`);
1302
1325
  // 处理参考图片
@@ -1426,12 +1449,14 @@ server.registerTool('edit-image', {
1426
1449
  inputSchema: {
1427
1450
  prompt: zod_1.z.string().describe('要编辑图片的中文提示词'),
1428
1451
  type: zod_1.z
1429
- .enum(['banana-pro', 'banana', 'seedream'])
1452
+ .enum(['banana-pro', 'banana', 'seedream', 'seedream-pro'])
1430
1453
  .optional()
1431
1454
  .default('seedream')
1432
1455
  .describe('The type of image model to use.'),
1433
1456
  sourceImageFileName: zod_1.z.string().describe('The source image file name.'),
1434
- saveToFileName: zod_1.z.string().describe('The filename to save.'),
1457
+ saveToFileName: zod_1.z
1458
+ .string()
1459
+ .describe('The filename to save. 应该是png文件'),
1435
1460
  size: zod_1.z
1436
1461
  .enum([
1437
1462
  '1024x1024',
@@ -1595,7 +1620,9 @@ server.registerTool('generate-video', {
1595
1620
  ])
1596
1621
  .default('lite')
1597
1622
  .describe('除非用户明确提出使用其他模型,否则一律用lite模型;zero 系列模型适合创作8-23秒带故事情节的短片'),
1598
- saveToFileName: zod_1.z.string().describe('The filename to save.'),
1623
+ saveToFileName: zod_1.z
1624
+ .string()
1625
+ .describe('The filename to save. 应该是mp4文件'),
1599
1626
  start_frame: zod_1.z
1600
1627
  .string()
1601
1628
  .optional()
@@ -2002,7 +2029,7 @@ server.registerTool('generate-video', {
2002
2029
  console.warn('Failed to send progress update:', progressError);
2003
2030
  }
2004
2031
  },
2005
- waitForFinish: true,
2032
+ waitForFinish: type !== 'zero',
2006
2033
  });
2007
2034
  if (!res) {
2008
2035
  throw new Error('Failed to generate video: no response from AI service');
@@ -2088,7 +2115,7 @@ server.registerTool('wait-for-task-finish', {
2088
2115
  .describe('The taskUrl of the video task to wait for.'),
2089
2116
  saveToFileName: zod_1.z
2090
2117
  .string()
2091
- .describe('The file name to save the video to.'),
2118
+ .describe('The file name to save the video to. 应该是mp4文件'),
2092
2119
  },
2093
2120
  }, async ({ taskUrl, saveToFileName }, context) => {
2094
2121
  try {
@@ -2163,7 +2190,7 @@ server.registerTool('generate-sound-effect', {
2163
2190
  .describe('The duration of the sound which will be generated in seconds. Must be at least 0.5 and at most 30. If set to None we will guess the optimal duration using the prompt. Defaults to None.'),
2164
2191
  saveToFileName: zod_1.z
2165
2192
  .string()
2166
- .describe('The filename to save. The mime type is audio/mpeg (mp3).'),
2193
+ .describe('The filename to save. 应该是mp3文件'),
2167
2194
  },
2168
2195
  }, async ({ prompt_in_english, loop, saveToFileName, duration_seconds }) => {
2169
2196
  try {
@@ -2210,13 +2237,22 @@ server.registerTool('generate-sound-effect', {
2210
2237
  return createErrorResponse(error, 'generate-sound-effect');
2211
2238
  }
2212
2239
  });
2213
- server.registerTool('generate-music', {
2214
- title: 'Generate Music',
2215
- description: 'Generate the music. Include background music or song.',
2240
+ server.registerTool('generate-music-or-mv', {
2241
+ title: '创作音乐(Music)或音乐视频(Music Video)',
2242
+ description: '生成音乐,包括MV(music video)、BGM 歌曲',
2216
2243
  inputSchema: {
2217
2244
  prompt: zod_1.z.string().describe('The prompt to generate.'),
2245
+ singerPhoto: zod_1.z
2246
+ .string()
2247
+ .optional()
2248
+ .describe('The singer photo to use. 只有type为music_video的时候才生效,也可以不传,模型会自动生成'),
2249
+ mvOrientation: zod_1.z
2250
+ .enum(['portrait', 'landscape'])
2251
+ .optional()
2252
+ .describe('The orientation of the music video. Defaults to portrait.')
2253
+ .default('portrait'),
2218
2254
  type: zod_1.z
2219
- .enum(['bgm', 'song'])
2255
+ .enum(['bgm', 'song', 'music_video'])
2220
2256
  .describe('The type of music. Defaults to background music.')
2221
2257
  .default('bgm'),
2222
2258
  model: zod_1.z
@@ -2233,9 +2269,11 @@ server.registerTool('generate-music', {
2233
2269
  .boolean()
2234
2270
  .default(false)
2235
2271
  .describe('Whether to skip copyright check.'),
2236
- saveToFileName: zod_1.z.string().describe('The filename to save.'),
2272
+ saveToFileName: zod_1.z
2273
+ .string()
2274
+ .describe('The filename to save. 应该是mp3文件'),
2237
2275
  },
2238
- }, async ({ prompt, type, model, duration, skipCopyCheck, saveToFileName }, context) => {
2276
+ }, async ({ prompt, singerPhoto, mvOrientation, type, model, duration, skipCopyCheck, saveToFileName, }, context) => {
2239
2277
  try {
2240
2278
  // 验证session状态
2241
2279
  const currentSession = await validateSession('generate-music');
@@ -2246,24 +2284,47 @@ server.registerTool('generate-music', {
2246
2284
  if (type === 'bgm' && duration > 120) {
2247
2285
  throw new Error('BGM duration must be at most 120 seconds.');
2248
2286
  }
2249
- const finalPrompt = `${prompt.trim()} ${type === 'bgm' ? `纯音乐无歌词,时长${duration}秒` : `时长${duration}秒,使用${model}模型`}`;
2250
- const res = await ai.generateMusic({
2251
- prompt: finalPrompt,
2252
- skipCopyCheck,
2253
- onProgress: async (metaData) => {
2254
- try {
2255
- await sendProgress(context, metaData.Result?.Progress ?? ++progress, metaData.Result?.Progress ? 100 : undefined, JSON.stringify(metaData));
2256
- }
2257
- catch (progressError) {
2258
- console.warn('Failed to send progress update:', progressError);
2259
- }
2260
- },
2261
- });
2287
+ let res;
2288
+ if (type === 'music_video') {
2289
+ const singer_photo = singerPhoto
2290
+ ? await getMaterialUri(currentSession, singerPhoto)
2291
+ : undefined;
2292
+ res = await ai.generateZeroCutMusicVideo({
2293
+ prompt: `${prompt.trim()} 音乐时长${duration}秒`,
2294
+ singerPhoto: singer_photo,
2295
+ orientation: mvOrientation,
2296
+ resolution: '720p',
2297
+ onProgress: async (metaData) => {
2298
+ try {
2299
+ await sendProgress(context, metaData.Result?.Progress ?? ++progress, metaData.Result?.Progress ? 100 : undefined, JSON.stringify(metaData));
2300
+ }
2301
+ catch (progressError) {
2302
+ console.warn('Failed to send progress update:', progressError);
2303
+ }
2304
+ },
2305
+ waitForFinish: false,
2306
+ });
2307
+ }
2308
+ else {
2309
+ const finalPrompt = `${prompt.trim()} ${type === 'bgm' ? `纯音乐无歌词,时长${duration}秒` : `时长${duration}秒,使用${model}模型`}`;
2310
+ res = await ai.generateMusic({
2311
+ prompt: finalPrompt,
2312
+ skipCopyCheck,
2313
+ onProgress: async (metaData) => {
2314
+ try {
2315
+ await sendProgress(context, metaData.Result?.Progress ?? ++progress, metaData.Result?.Progress ? 100 : undefined, JSON.stringify(metaData));
2316
+ }
2317
+ catch (progressError) {
2318
+ console.warn('Failed to send progress update:', progressError);
2319
+ }
2320
+ },
2321
+ });
2322
+ }
2262
2323
  if (!res) {
2263
- throw new Error('Failed to generate BGM: no response from AI service');
2324
+ throw new Error('Failed to generate Music: no response from AI service');
2264
2325
  }
2265
2326
  if (res.url) {
2266
- console.log('BGM generated successfully, saving to materials...');
2327
+ console.log('Music generated successfully, saving to materials...');
2267
2328
  const uri = await saveMaterial(currentSession, res.url, validatedFileName);
2268
2329
  const { url, duration: bgmDuration, captions, ...opts } = res;
2269
2330
  // 保存captions到本地
@@ -2301,15 +2362,29 @@ server.registerTool('generate-music', {
2301
2362
  ],
2302
2363
  };
2303
2364
  }
2365
+ else if (res.taskUrl) {
2366
+ return {
2367
+ content: [
2368
+ {
2369
+ type: 'text',
2370
+ text: JSON.stringify({
2371
+ success: true,
2372
+ message: '该视频生成任务正在运行中,它是异步任务,且执行时间较长,你应立即调用工具 wait-for-task-finish 来等待任务结束,如该工具调用超时,你应立即再次重新调用直到任务结束。',
2373
+ taskUrl: res.taskUrl,
2374
+ }),
2375
+ },
2376
+ ],
2377
+ };
2378
+ }
2304
2379
  else {
2305
- console.warn('BGM generation completed but no URL returned');
2380
+ console.warn('Music generation completed but no URL returned');
2306
2381
  return {
2307
2382
  content: [
2308
2383
  {
2309
2384
  type: 'text',
2310
2385
  text: JSON.stringify({
2311
2386
  success: false,
2312
- error: 'No BGM URL returned from AI service',
2387
+ error: 'No Music URL returned from AI service',
2313
2388
  response: res,
2314
2389
  timestamp: new Date().toISOString(),
2315
2390
  }),
@@ -2319,7 +2394,7 @@ server.registerTool('generate-music', {
2319
2394
  }
2320
2395
  }
2321
2396
  catch (error) {
2322
- return createErrorResponse(error, 'generate-bgm');
2397
+ return createErrorResponse(error, 'generate-music');
2323
2398
  }
2324
2399
  });
2325
2400
  server.registerTool('generate-scene-tts', {
@@ -2346,7 +2421,9 @@ server.registerTool('generate-scene-tts', {
2346
2421
  .string()
2347
2422
  .optional()
2348
2423
  .describe('跳过校验的理由,如果skipConsistencyCheck设为true,必须要传这个参数'),
2349
- saveToFileName: zod_1.z.string().describe('The filename to save.'),
2424
+ saveToFileName: zod_1.z
2425
+ .string()
2426
+ .describe('The filename to save. 应该是mp3文件'),
2350
2427
  speed: zod_1.z
2351
2428
  .number()
2352
2429
  .min(0.5)
@@ -2917,7 +2994,7 @@ server.registerTool('voice-design', {
2917
2994
  previewText: zod_1.z.string().describe('The preview text to design the voice.'),
2918
2995
  saveToFileName: zod_1.z
2919
2996
  .string()
2920
- .describe('The file name to save the designed voice.'),
2997
+ .describe('The file name to save the designed voice. 应该是mp3文件'),
2921
2998
  },
2922
2999
  }, async ({ prompt, previewText, saveToFileName }) => {
2923
3000
  try {
@@ -3393,7 +3470,7 @@ server.registerTool('audio-video-sync', {
3393
3470
  .describe('The reference photo face for lip sync.'),
3394
3471
  saveToFileName: zod_1.z
3395
3472
  .string()
3396
- .describe('The filename to save the audio-video-synced video.'),
3473
+ .describe('The filename to save the audio-video-synced video. 应该是mp4文件'),
3397
3474
  },
3398
3475
  }, async ({ lipSync, lipSyncType, lipSyncPadAudio, videoFileName, audioFileName, audioInMs, refPhotoFileName, saveToFileName, }, context) => {
3399
3476
  try {
@@ -3582,7 +3659,7 @@ server.registerTool('generate-video-by-ref', {
3582
3659
  .describe('Whether to mute the video (effective for sora2 and veo3.1).'),
3583
3660
  saveToFileName: zod_1.z
3584
3661
  .string()
3585
- .describe('The filename to save the generated video.'),
3662
+ .describe('The filename to save the generated video. 应该是mp4文件'),
3586
3663
  sceneIndex: zod_1.z
3587
3664
  .number()
3588
3665
  .min(1)
@@ -3788,7 +3865,54 @@ server.registerTool('generate-video-by-ref', {
3788
3865
  if (promptPrefix) {
3789
3866
  promptPrefix += '\n';
3790
3867
  }
3791
- const finalPrompt = `${promptPrefix}${prompt}`;
3868
+ let finalPrompt = `${promptPrefix}${prompt}`;
3869
+ if (type === 'pixv') {
3870
+ const completion = await ai.getCompletions({
3871
+ model: 'Doubao-Seed-1.6',
3872
+ messages: [
3873
+ {
3874
+ role: 'system',
3875
+ content: `你根据主体信息,优化用户指令,使描述中的内容正确引用主体名称。
3876
+
3877
+ 具体方式为,将用户指令中引用主体信息中主体名称的部分,用 “@主体名” 的形式替代,注意它和前后内容之间也需要用**空格**分隔。
3878
+
3879
+ ## 例子
3880
+
3881
+ ### 输入:
3882
+
3883
+ 主体信息
3884
+ [
3885
+ {"type": "subject", "fileName": "dog.png", "ref_name": "狗"},
3886
+ {"type": "background", "fileName": "room.png", "ref_name": "房间"}
3887
+ ]
3888
+
3889
+ 用户指令
3890
+ 一只狗在房间里玩耍
3891
+
3892
+ ### 输出:
3893
+ 一只 @狗 在 @房间 里玩耍
3894
+
3895
+ ---
3896
+
3897
+ ## 要求与约束
3898
+
3899
+ 只输出替换主体名后的用户指令,不要输出其他任何额外内容
3900
+ `,
3901
+ },
3902
+ {
3903
+ role: 'user',
3904
+ content: `## 主体信息
3905
+
3906
+ ${JSON.stringify(referenceImages)}
3907
+
3908
+ ## 用户指令
3909
+
3910
+ ${prompt.trim()}`,
3911
+ },
3912
+ ],
3913
+ });
3914
+ finalPrompt = completion.choices[0]?.message?.content.trim();
3915
+ }
3792
3916
  // 调用 referencesToVideo 函数
3793
3917
  const result = await currentSession.ai.referencesToVideo({
3794
3918
  prompt: finalPrompt,
@@ -3892,7 +4016,7 @@ server.registerTool('extend-video-duration', {
3892
4016
  .describe('Optional end frame image file name in materials directory to guide the video extension.'),
3893
4017
  saveToFileName: zod_1.z
3894
4018
  .string()
3895
- .describe('The filename to save the extended video.'),
4019
+ .describe('The filename to save the extended video. 应该是mp4文件'),
3896
4020
  },
3897
4021
  }, async ({ videoFileName, duration, resolution, prompt, type = 'turbo', endFrame, saveToFileName, }, context) => {
3898
4022
  try {
@@ -3988,11 +4112,11 @@ server.registerTool('use-template', {
3988
4112
  .describe('Optional materials to use in the template.'),
3989
4113
  saveToFileName: zod_1.z
3990
4114
  .string()
3991
- .describe('The filename to save the generated material.'),
4115
+ .describe('The filename to save the generated material. 根据用户具体需求,应该是mp4或png文件'),
3992
4116
  },
3993
4117
  }, async ({ user_request, saveToFileName, materials }) => {
3994
4118
  try {
3995
- const currentSession = await validateSession('generate-video-by-template');
4119
+ const currentSession = await validateSession('use-template');
3996
4120
  const ai = currentSession.ai;
3997
4121
  const data = await ai.listTemplates('all');
3998
4122
  const templates = data.map(item => ({
@@ -4007,7 +4131,7 @@ server.registerTool('use-template', {
4007
4131
  messages: [
4008
4132
  {
4009
4133
  role: 'system',
4010
- content: `你根据用户需求,从以下模板中选择一个匹配的模板,返回模板ID:\n\n${JSON.stringify(templates)}\n\n**约束**:只输出模板ID,不需要其他解释,如果没有匹配的模版,输出"无匹配模版"`,
4134
+ content: `你根据用户需求,分析需求与模板描述(description)和触发器(trigger)的匹配程度,从以下模板中选择一个匹配的模板,返回模板ID:\n\n${JSON.stringify(templates)}\n\n**约束**:只输出模板ID,不需要其他解释,如果没有匹配的模版,输出"无匹配模版"`,
4011
4135
  },
4012
4136
  {
4013
4137
  role: 'user',