cerevox 3.0.0-beta.9 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/ai.d.ts +25 -17
- package/dist/core/ai.d.ts.map +1 -1
- package/dist/core/ai.js +372 -151
- package/dist/core/ai.js.map +1 -1
- package/dist/mcp/servers/helper/doubao_voices_full.js +1 -1
- package/dist/mcp/servers/prompts/rules/creative-ad.md +2 -2
- package/dist/mcp/servers/prompts/rules/expert.md +1 -1
- package/dist/mcp/servers/prompts/rules/freeform.md +2 -2
- package/dist/mcp/servers/prompts/rules/general-video.md +10 -5
- package/dist/mcp/servers/prompts/rules/story-telling.md +3 -3
- package/dist/mcp/servers/prompts/skills/workflows/general-video.md +2 -2
- package/dist/mcp/servers/prompts/skills/workflows/music-video.md +1 -1
- package/dist/mcp/servers/zerocut.d.ts.map +1 -1
- package/dist/mcp/servers/zerocut.js +361 -523
- package/dist/mcp/servers/zerocut.js.map +1 -1
- package/dist/utils/coze.d.ts.map +1 -1
- package/dist/utils/coze.js +4 -1
- package/dist/utils/coze.js.map +1 -1
- package/package.json +1 -1
package/dist/core/ai.js
CHANGED
|
@@ -30,7 +30,7 @@ async function waitForWorkflowFinish(taskUrl, apiKey = process.env.CEREVOX_API_K
|
|
|
30
30
|
...resData,
|
|
31
31
|
debug_url: undefined,
|
|
32
32
|
});
|
|
33
|
-
throw new Error(`
|
|
33
|
+
throw new Error(`Creation generation failed: ${errMsg}`);
|
|
34
34
|
}
|
|
35
35
|
if (resData.status === 'Success') {
|
|
36
36
|
return {
|
|
@@ -41,7 +41,7 @@ async function waitForWorkflowFinish(taskUrl, apiKey = process.env.CEREVOX_API_K
|
|
|
41
41
|
await new Promise(resolve => setTimeout(resolve, interval || 1500));
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
|
-
async function
|
|
44
|
+
async function waitForTaskComplete(sandbox, taskUrl, logger, onProgress, interval = 1500) {
|
|
45
45
|
const headers = {
|
|
46
46
|
'Content-Type': 'application/json',
|
|
47
47
|
};
|
|
@@ -51,19 +51,19 @@ async function waitForVideoTaskComplete(sandbox, taskUrl, logger, onProgress, in
|
|
|
51
51
|
headers: headers,
|
|
52
52
|
});
|
|
53
53
|
if (!response.ok) {
|
|
54
|
-
throw new Error(`
|
|
54
|
+
throw new Error(`Creation generation failed: ${response.statusText}`);
|
|
55
55
|
}
|
|
56
56
|
const data = await response.json();
|
|
57
|
-
logger.debug(`get
|
|
57
|
+
logger.debug(`get task status: ${JSON.stringify(data)}`);
|
|
58
58
|
onProgress?.(data);
|
|
59
59
|
if (data.status === 'succeeded' || data.status === 'succeed') {
|
|
60
60
|
return data;
|
|
61
61
|
}
|
|
62
62
|
if (data.status === 'failed' || data.status === 'Fail' || !response.ok) {
|
|
63
|
-
throw new Error(`
|
|
63
|
+
throw new Error(`Creation generation failed: ${JSON.stringify(data)}`);
|
|
64
64
|
}
|
|
65
65
|
if (data.status === 'canceled') {
|
|
66
|
-
throw new Error(`
|
|
66
|
+
throw new Error(`Creation generation canceled: ${JSON.stringify(data)}`);
|
|
67
67
|
}
|
|
68
68
|
await new Promise(resolve => setTimeout(resolve, interval || 1500));
|
|
69
69
|
}
|
|
@@ -78,7 +78,7 @@ async function waitForMusicTaskComplete(sandbox, taskUrl, logger, onProgress, in
|
|
|
78
78
|
headers: headers,
|
|
79
79
|
});
|
|
80
80
|
if (!response.ok) {
|
|
81
|
-
throw new Error(`
|
|
81
|
+
throw new Error(`Creation generation failed: ${response.statusText}`);
|
|
82
82
|
}
|
|
83
83
|
const data = await response.json();
|
|
84
84
|
logger.debug(`get music task status: ${JSON.stringify(data)}`);
|
|
@@ -108,80 +108,11 @@ let AI = class AI extends base_1.BaseClass {
|
|
|
108
108
|
const data = await res.json();
|
|
109
109
|
return data;
|
|
110
110
|
}
|
|
111
|
-
async generateImageSeries(options) {
|
|
112
|
-
try {
|
|
113
|
-
if (Array.isArray(options.prompts)) {
|
|
114
|
-
const count = options.prompts.length;
|
|
115
|
-
options.prompts = options.prompts
|
|
116
|
-
.map((p, i) => `图片${i + 1}: ${p.trim()}`)
|
|
117
|
-
.join('\n');
|
|
118
|
-
options.prompts = `生成${count}张图片\n${options.prompts.trim()}`;
|
|
119
|
-
}
|
|
120
|
-
if (options.refPrefix) {
|
|
121
|
-
options.prompts = `${options.refPrefix}
|
|
122
|
-
|
|
123
|
-
参考以上图片,执行以下各场景绘图:
|
|
124
|
-
|
|
125
|
-
${options.prompts};
|
|
126
|
-
`;
|
|
127
|
-
}
|
|
128
|
-
const max_count = options.max_count ?? 15;
|
|
129
|
-
const payload = {
|
|
130
|
-
prompt: options.prompts,
|
|
131
|
-
size: options.size ?? '1280x720',
|
|
132
|
-
watermark: options.watermark ?? false,
|
|
133
|
-
image: options.image,
|
|
134
|
-
max_images: max_count,
|
|
135
|
-
};
|
|
136
|
-
// 启动心跳机制
|
|
137
|
-
let heartbeatInterval = null;
|
|
138
|
-
if (options.onProgress) {
|
|
139
|
-
heartbeatInterval = setInterval(() => {
|
|
140
|
-
options.onProgress?.({});
|
|
141
|
-
}, 60000); // 每60秒调用一次
|
|
142
|
-
}
|
|
143
|
-
try {
|
|
144
|
-
const res = await this.session.sandbox.request('/ai/image/sequential_generate', {
|
|
145
|
-
method: 'POST',
|
|
146
|
-
headers: {
|
|
147
|
-
'Content-Type': 'application/json',
|
|
148
|
-
},
|
|
149
|
-
body: JSON.stringify(payload),
|
|
150
|
-
});
|
|
151
|
-
const data = await res.json();
|
|
152
|
-
// 清除心跳定时器
|
|
153
|
-
if (heartbeatInterval) {
|
|
154
|
-
clearInterval(heartbeatInterval);
|
|
155
|
-
}
|
|
156
|
-
if (data.data) {
|
|
157
|
-
const urls = data.data
|
|
158
|
-
.map((item) => item.url)
|
|
159
|
-
.filter((url) => url?.trim());
|
|
160
|
-
for (const url of urls) {
|
|
161
|
-
this.session.track('Image Generated', { url });
|
|
162
|
-
}
|
|
163
|
-
return {
|
|
164
|
-
prompt: options.prompts,
|
|
165
|
-
data,
|
|
166
|
-
urls,
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
return data;
|
|
170
|
-
}
|
|
171
|
-
catch (error) {
|
|
172
|
-
// 确保在出错时也清除心跳定时器
|
|
173
|
-
if (heartbeatInterval) {
|
|
174
|
-
clearInterval(heartbeatInterval);
|
|
175
|
-
}
|
|
176
|
-
throw error;
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
catch (error) {
|
|
180
|
-
this.logger.error('generateImage error', error);
|
|
181
|
-
return { error: `generateImage error: ${error}` };
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
111
|
async generateImage(options) {
|
|
112
|
+
if (options.type === 'line-sketch') {
|
|
113
|
+
options.prompt = `请根据以下描述绘制一张线稿:\n\n${options.prompt}\n\n画面除了主体和提到的物品之外,不要画其他任何东西,留白`;
|
|
114
|
+
options.type = 'banana';
|
|
115
|
+
}
|
|
185
116
|
try {
|
|
186
117
|
const res = await this.session.sandbox.request(`/ai/image/generate`, {
|
|
187
118
|
method: 'POST',
|
|
@@ -191,18 +122,20 @@ ${options.prompts};
|
|
|
191
122
|
body: JSON.stringify(options),
|
|
192
123
|
});
|
|
193
124
|
const data = await res.json();
|
|
194
|
-
if (data.data
|
|
195
|
-
|
|
125
|
+
if (data.data) {
|
|
126
|
+
const urls = data.data
|
|
127
|
+
.map((item) => item.url)
|
|
128
|
+
.filter((url) => url?.trim());
|
|
129
|
+
for (const url of urls) {
|
|
130
|
+
this.session.track('Image Generated', { url });
|
|
131
|
+
}
|
|
196
132
|
return {
|
|
197
|
-
|
|
133
|
+
prompt: options.prompt,
|
|
134
|
+
// data,
|
|
135
|
+
urls,
|
|
198
136
|
};
|
|
199
137
|
}
|
|
200
|
-
|
|
201
|
-
// banana
|
|
202
|
-
this.session.track('Image Generated', data);
|
|
203
|
-
return data;
|
|
204
|
-
}
|
|
205
|
-
throw new Error(`generateImage error: ${JSON.stringify(data)}`);
|
|
138
|
+
return data;
|
|
206
139
|
}
|
|
207
140
|
catch (error) {
|
|
208
141
|
this.logger.error('generateImage error', error);
|
|
@@ -210,28 +143,6 @@ ${options.prompts};
|
|
|
210
143
|
throw error;
|
|
211
144
|
}
|
|
212
145
|
}
|
|
213
|
-
async generateLineSketch(options) {
|
|
214
|
-
try {
|
|
215
|
-
const res = await this.session.sandbox.request(`/ai/image/line_sketch`, {
|
|
216
|
-
method: 'POST',
|
|
217
|
-
headers: {
|
|
218
|
-
'Content-Type': 'application/json',
|
|
219
|
-
},
|
|
220
|
-
body: JSON.stringify(options),
|
|
221
|
-
});
|
|
222
|
-
const data = await res.json();
|
|
223
|
-
if (data.url) {
|
|
224
|
-
this.session.track('Line Sketch Generated', data);
|
|
225
|
-
return data;
|
|
226
|
-
}
|
|
227
|
-
throw new Error(`generateLineSketch error: ${JSON.stringify(data)}`);
|
|
228
|
-
}
|
|
229
|
-
catch (error) {
|
|
230
|
-
this.logger.error('generateLineSketch error', error);
|
|
231
|
-
this.session.track('Line Sketch Generated', { error: error.message });
|
|
232
|
-
throw error;
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
146
|
async extendVideo(options) {
|
|
236
147
|
try {
|
|
237
148
|
const waitForFinish = options.waitForFinish ?? true;
|
|
@@ -244,7 +155,7 @@ ${options.prompts};
|
|
|
244
155
|
});
|
|
245
156
|
const data = await res.json();
|
|
246
157
|
if (data.taskUrl && waitForFinish) {
|
|
247
|
-
return this.
|
|
158
|
+
return this.waitForTaskComplete({
|
|
248
159
|
taskUrl: data.taskUrl,
|
|
249
160
|
onProgress: options.onProgress,
|
|
250
161
|
timeout: 300000,
|
|
@@ -286,7 +197,7 @@ ${options.prompts};
|
|
|
286
197
|
});
|
|
287
198
|
const data = await res.json();
|
|
288
199
|
if (data.taskUrl && waitForFinish) {
|
|
289
|
-
return this.
|
|
200
|
+
return this.waitForTaskComplete({
|
|
290
201
|
taskUrl: data.taskUrl,
|
|
291
202
|
onProgress: options.onProgress,
|
|
292
203
|
timeout: 300000,
|
|
@@ -299,6 +210,27 @@ ${options.prompts};
|
|
|
299
210
|
return { error: error.message };
|
|
300
211
|
}
|
|
301
212
|
}
|
|
213
|
+
async generateZeroCutMusicVideo(options) {
|
|
214
|
+
try {
|
|
215
|
+
const waitForFinish = options.waitForFinish ?? true;
|
|
216
|
+
const workflowId = '7574088452263591988';
|
|
217
|
+
const parameters = {
|
|
218
|
+
prompt: options.prompt,
|
|
219
|
+
singer_apperance: options.singerPhoto,
|
|
220
|
+
orientation: options.orientation,
|
|
221
|
+
resolution: options.resolution,
|
|
222
|
+
music_duration: options.duration,
|
|
223
|
+
origin_song: options.originalSong,
|
|
224
|
+
gen_subtitles: options.genSubtitles,
|
|
225
|
+
};
|
|
226
|
+
const result = await this.runCozeWorkflow(workflowId, parameters, options.onProgress, 1500, waitForFinish);
|
|
227
|
+
return result;
|
|
228
|
+
}
|
|
229
|
+
catch (error) {
|
|
230
|
+
this.logger.error('Generate ZeroCut Video error', error);
|
|
231
|
+
return { error: error.message, content: null };
|
|
232
|
+
}
|
|
233
|
+
}
|
|
302
234
|
// 用自己的模型生成 video
|
|
303
235
|
async generateZeroCutVideo(options) {
|
|
304
236
|
try {
|
|
@@ -361,7 +293,7 @@ ${options.prompts};
|
|
|
361
293
|
});
|
|
362
294
|
const data = await res.json();
|
|
363
295
|
if (data.taskUrl && waitForFinish) {
|
|
364
|
-
return this.
|
|
296
|
+
return this.waitForTaskComplete({
|
|
365
297
|
taskUrl: data.taskUrl,
|
|
366
298
|
onProgress: options.onProgress,
|
|
367
299
|
timeout: 300000,
|
|
@@ -388,7 +320,7 @@ ${options.prompts};
|
|
|
388
320
|
});
|
|
389
321
|
const data = await res.json();
|
|
390
322
|
if (data.taskUrl && waitForFinish) {
|
|
391
|
-
return this.
|
|
323
|
+
return this.waitForTaskComplete({
|
|
392
324
|
taskUrl: data.taskUrl,
|
|
393
325
|
onProgress: options.onProgress,
|
|
394
326
|
timeout: 300000,
|
|
@@ -738,7 +670,6 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
738
670
|
async textToSpeechVolc(options) {
|
|
739
671
|
const speech_rate = options.speed ?? 0; // -50 ~ 100
|
|
740
672
|
const loudness_rate = options.volume ?? 0; // -50 ~ 100
|
|
741
|
-
const emotion = options.emotion ?? 'storytelling';
|
|
742
673
|
const explicit_language = options.explicit_language ?? 'zh';
|
|
743
674
|
// 随机停顿 300 ~ 500 ms 模拟人对话
|
|
744
675
|
const silence_duration = options.silence_duration ?? Math.floor(Math.random() * 200 + 300);
|
|
@@ -753,10 +684,10 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
753
684
|
text: options.text,
|
|
754
685
|
speech_rate,
|
|
755
686
|
loudness_rate,
|
|
756
|
-
emotion,
|
|
757
687
|
silence_duration,
|
|
758
688
|
explicit_language,
|
|
759
689
|
context_texts: options.context_texts,
|
|
690
|
+
version: options.version,
|
|
760
691
|
}),
|
|
761
692
|
});
|
|
762
693
|
const data = await res.json();
|
|
@@ -910,7 +841,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
910
841
|
});
|
|
911
842
|
const data = await res.json();
|
|
912
843
|
if (data.taskUrl && waitForFinish) {
|
|
913
|
-
return this.
|
|
844
|
+
return this.waitForTaskComplete({
|
|
914
845
|
taskUrl: data.taskUrl,
|
|
915
846
|
onProgress: options.onProgress,
|
|
916
847
|
timeout: 300000,
|
|
@@ -985,7 +916,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
985
916
|
});
|
|
986
917
|
const data = await res.json();
|
|
987
918
|
if (data.taskUrl && waitForFinish) {
|
|
988
|
-
return this.
|
|
919
|
+
return this.waitForTaskComplete({
|
|
989
920
|
taskUrl: data.taskUrl,
|
|
990
921
|
onProgress: options.onProgress,
|
|
991
922
|
timeout: 300000,
|
|
@@ -1018,7 +949,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
1018
949
|
});
|
|
1019
950
|
const data = await res.json();
|
|
1020
951
|
if (data.taskUrl && waitForFinish) {
|
|
1021
|
-
return this.
|
|
952
|
+
return this.waitForTaskComplete({
|
|
1022
953
|
taskUrl: data.taskUrl,
|
|
1023
954
|
onProgress: options.onProgress,
|
|
1024
955
|
timeout: 300000,
|
|
@@ -1065,7 +996,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
1065
996
|
});
|
|
1066
997
|
const data = await res.json();
|
|
1067
998
|
if (data.taskUrl && waitForFinish) {
|
|
1068
|
-
return this.
|
|
999
|
+
return this.waitForTaskComplete({
|
|
1069
1000
|
taskUrl: data.taskUrl,
|
|
1070
1001
|
onProgress: options.onProgress,
|
|
1071
1002
|
timeout: 300000,
|
|
@@ -1266,7 +1197,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
1266
1197
|
});
|
|
1267
1198
|
const data = await res.json();
|
|
1268
1199
|
if (data.taskUrl && waitForFinish) {
|
|
1269
|
-
return this.
|
|
1200
|
+
return this.waitForTaskComplete({
|
|
1270
1201
|
taskUrl: data.taskUrl,
|
|
1271
1202
|
onProgress: options.onProgress,
|
|
1272
1203
|
timeout: 300000,
|
|
@@ -1279,6 +1210,25 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
1279
1210
|
return { error: error.message };
|
|
1280
1211
|
}
|
|
1281
1212
|
}
|
|
1213
|
+
async splitVideoAndAudio(options) {
|
|
1214
|
+
try {
|
|
1215
|
+
const res = await this.session.sandbox.request(`/ai/video/split`, {
|
|
1216
|
+
method: 'POST',
|
|
1217
|
+
headers: {
|
|
1218
|
+
'Content-Type': 'application/json',
|
|
1219
|
+
},
|
|
1220
|
+
body: JSON.stringify({
|
|
1221
|
+
url: options.videoUrl,
|
|
1222
|
+
}),
|
|
1223
|
+
});
|
|
1224
|
+
const data = await res.json();
|
|
1225
|
+
return data;
|
|
1226
|
+
}
|
|
1227
|
+
catch (error) {
|
|
1228
|
+
this.logger.error('splitVideoAndAudio error', error);
|
|
1229
|
+
return { error: error.message };
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1282
1232
|
async getCozeWorkflowInfo(workflowId) {
|
|
1283
1233
|
return (0, coze_1.getWorkflowInfo)(workflowId, this.session.sandbox.token);
|
|
1284
1234
|
}
|
|
@@ -1317,7 +1267,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
1317
1267
|
}
|
|
1318
1268
|
async waitForWorkflowFinish(options) {
|
|
1319
1269
|
try {
|
|
1320
|
-
const { taskUrl, onProgress, interval = 1500, timeout =
|
|
1270
|
+
const { taskUrl, onProgress, interval = 1500, timeout = 1800000, } = options;
|
|
1321
1271
|
const result = await Promise.race([
|
|
1322
1272
|
waitForWorkflowFinish(taskUrl, this.session.sandbox.token, this.logger, onProgress, interval),
|
|
1323
1273
|
new Promise(resolve => {
|
|
@@ -1325,13 +1275,14 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
1325
1275
|
resolve({
|
|
1326
1276
|
status: 'running',
|
|
1327
1277
|
reason: `wait timeout ${timeout}ms`,
|
|
1278
|
+
nextAction: '可继续等待',
|
|
1328
1279
|
taskUrl,
|
|
1329
1280
|
});
|
|
1330
1281
|
}, timeout);
|
|
1331
1282
|
}),
|
|
1332
1283
|
]);
|
|
1333
1284
|
if (result?.content) {
|
|
1334
|
-
this.session.track('
|
|
1285
|
+
this.session.track('Creation Generated', {
|
|
1335
1286
|
taskUrl,
|
|
1336
1287
|
});
|
|
1337
1288
|
return {
|
|
@@ -1344,22 +1295,22 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
1344
1295
|
else if (result?.taskUrl) {
|
|
1345
1296
|
return result;
|
|
1346
1297
|
}
|
|
1347
|
-
this.session.track('
|
|
1298
|
+
this.session.track('Creation Generated Failed', {
|
|
1348
1299
|
result: JSON.stringify(result),
|
|
1349
1300
|
});
|
|
1350
|
-
throw new Error(`
|
|
1301
|
+
throw new Error(`Creation Generated Failed: ${JSON.stringify(result)}`);
|
|
1351
1302
|
}
|
|
1352
1303
|
catch (error) {
|
|
1353
1304
|
this.logger.error('waitForWorkflowFinish error', error);
|
|
1354
|
-
this.session.track('
|
|
1305
|
+
this.session.track('Creation Generated Failed', {
|
|
1355
1306
|
reason: error.message,
|
|
1356
1307
|
});
|
|
1357
1308
|
throw error;
|
|
1358
1309
|
}
|
|
1359
1310
|
}
|
|
1360
|
-
async
|
|
1311
|
+
async waitForTaskComplete(options) {
|
|
1361
1312
|
try {
|
|
1362
|
-
const { taskUrl, onProgress, interval = 1500, traceWorkflow = true, timeout =
|
|
1313
|
+
const { taskUrl, onProgress, interval = 1500, traceWorkflow = true, timeout = 1800000, } = options;
|
|
1363
1314
|
if (taskUrl.startsWith('https://api.coze.cn/v1/workflows/')) {
|
|
1364
1315
|
return this.waitForWorkflowFinish(options);
|
|
1365
1316
|
}
|
|
@@ -1372,46 +1323,59 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
1372
1323
|
});
|
|
1373
1324
|
}
|
|
1374
1325
|
const result = await Promise.race([
|
|
1375
|
-
|
|
1326
|
+
waitForTaskComplete(this.session.sandbox, taskUrl, this.logger, onProgress, interval),
|
|
1376
1327
|
new Promise(resolve => {
|
|
1377
1328
|
setTimeout(() => {
|
|
1378
1329
|
resolve({
|
|
1379
1330
|
status: 'running',
|
|
1380
1331
|
reason: `wait timeout ${timeout}ms`,
|
|
1332
|
+
nextAction: '可继续等待',
|
|
1381
1333
|
taskUrl,
|
|
1382
1334
|
});
|
|
1383
1335
|
}, timeout);
|
|
1384
1336
|
}),
|
|
1385
1337
|
]);
|
|
1386
|
-
if (result
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1338
|
+
if (result?.content) {
|
|
1339
|
+
const content = result.content;
|
|
1340
|
+
if (content.video_url) {
|
|
1341
|
+
this.session.track('Creation Generated', {
|
|
1342
|
+
id: result.id,
|
|
1343
|
+
model: result.model,
|
|
1344
|
+
fps: result.framespersecond,
|
|
1345
|
+
duration: result.duration,
|
|
1346
|
+
resolution: result.resolution,
|
|
1347
|
+
ratio: result.ratio,
|
|
1348
|
+
cost: result.usage?.total_tokens,
|
|
1349
|
+
});
|
|
1350
|
+
return {
|
|
1351
|
+
url: result.content.video_url,
|
|
1352
|
+
last_frame_url: result.content?.last_frame_url,
|
|
1353
|
+
duration: result.duration,
|
|
1354
|
+
resolution: result.resolution,
|
|
1355
|
+
ratio: result.ratio,
|
|
1356
|
+
};
|
|
1357
|
+
}
|
|
1358
|
+
else if (content?.data) {
|
|
1359
|
+
// 图片
|
|
1360
|
+
this.session.track('Creation Generated', {
|
|
1361
|
+
urls: content.data,
|
|
1362
|
+
});
|
|
1363
|
+
return {
|
|
1364
|
+
urls: content.data.map((v) => v.url),
|
|
1365
|
+
};
|
|
1366
|
+
}
|
|
1403
1367
|
}
|
|
1404
1368
|
else if (result?.taskUrl) {
|
|
1405
1369
|
return result;
|
|
1406
1370
|
}
|
|
1407
|
-
this.session.track('
|
|
1371
|
+
this.session.track('Creation Creation Failed', {
|
|
1408
1372
|
result: JSON.stringify(result),
|
|
1409
1373
|
});
|
|
1410
|
-
throw new Error(`
|
|
1374
|
+
throw new Error(`Creation Creation Failed: ${JSON.stringify(result)}`);
|
|
1411
1375
|
}
|
|
1412
1376
|
catch (error) {
|
|
1413
|
-
this.logger.error('
|
|
1414
|
-
this.session.track('
|
|
1377
|
+
this.logger.error('waitForTaskComplete error', error);
|
|
1378
|
+
this.session.track('Creation Creation Failed', {
|
|
1415
1379
|
reason: error.message,
|
|
1416
1380
|
});
|
|
1417
1381
|
throw error;
|
|
@@ -1482,6 +1446,263 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
|
|
|
1482
1446
|
async listContextRules() {
|
|
1483
1447
|
return (await this.session.sandbox.request('/ai/personas')).json();
|
|
1484
1448
|
}
|
|
1449
|
+
async listVoices() {
|
|
1450
|
+
const volcano_tts_2_voices = [
|
|
1451
|
+
{
|
|
1452
|
+
id: 'zh_female_vv_uranus_bigtts',
|
|
1453
|
+
description: ['语调平稳、咬字柔和、自带治愈安抚力的女声音色'],
|
|
1454
|
+
gender: 'female',
|
|
1455
|
+
languages: ['zh', 'en'],
|
|
1456
|
+
type: 'volcano_tts_2',
|
|
1457
|
+
},
|
|
1458
|
+
{
|
|
1459
|
+
id: 'zh_female_xiaohe_uranus_bigtts',
|
|
1460
|
+
description: ['声线甜美有活力的妹妹,活泼开朗,笑容明媚。'],
|
|
1461
|
+
gender: 'female',
|
|
1462
|
+
languages: ['zh', 'en'],
|
|
1463
|
+
type: 'volcano_tts_2',
|
|
1464
|
+
},
|
|
1465
|
+
{
|
|
1466
|
+
id: 'zh_male_m191_uranus_bigtts',
|
|
1467
|
+
description: ['声音磁性的男生,成熟理性,做事有条理,让人信赖。'],
|
|
1468
|
+
gender: 'male',
|
|
1469
|
+
languages: ['zh', 'en'],
|
|
1470
|
+
type: 'volcano_tts_2',
|
|
1471
|
+
},
|
|
1472
|
+
{
|
|
1473
|
+
id: 'zh_male_taocheng_uranus_bigtts',
|
|
1474
|
+
description: ['眉目清朗男大,清澈温润有朝气,开朗真诚。'],
|
|
1475
|
+
gender: 'male',
|
|
1476
|
+
languages: ['zh', 'en'],
|
|
1477
|
+
type: 'volcano_tts_2',
|
|
1478
|
+
},
|
|
1479
|
+
{
|
|
1480
|
+
id: 'zh_female_xueayi_saturn_bigtts',
|
|
1481
|
+
description: ['温柔又有力量的音色,陪孩子在故事里勇敢探索!'],
|
|
1482
|
+
gender: 'female',
|
|
1483
|
+
languages: ['zh', 'en'],
|
|
1484
|
+
type: 'volcano_tts_2',
|
|
1485
|
+
},
|
|
1486
|
+
{
|
|
1487
|
+
id: 'zh_male_dayi_saturn_bigtts',
|
|
1488
|
+
description: ['语气沉稳有力、咬字清晰厚重、自带可靠安全感的男声音色'],
|
|
1489
|
+
gender: 'male',
|
|
1490
|
+
languages: ['zh', 'en'],
|
|
1491
|
+
type: 'volcano_tts_2',
|
|
1492
|
+
},
|
|
1493
|
+
{
|
|
1494
|
+
id: 'zh_female_mizai_saturn_bigtts',
|
|
1495
|
+
description: ['语调沉稳、咬字扎实、自带踏实和包容感的女声音色'],
|
|
1496
|
+
gender: 'female',
|
|
1497
|
+
languages: ['zh', 'en'],
|
|
1498
|
+
type: 'volcano_tts_2',
|
|
1499
|
+
},
|
|
1500
|
+
{
|
|
1501
|
+
id: 'zh_female_jitangnv_saturn_bigtts',
|
|
1502
|
+
description: [
|
|
1503
|
+
'鸡汤女:语气轻软、尾音轻落、带有自然呼吸感且十分亲切的女声音色',
|
|
1504
|
+
],
|
|
1505
|
+
gender: 'female',
|
|
1506
|
+
languages: ['zh', 'en'],
|
|
1507
|
+
type: 'volcano_tts_2',
|
|
1508
|
+
},
|
|
1509
|
+
{
|
|
1510
|
+
id: 'zh_female_meilinvyou_saturn_bigtts',
|
|
1511
|
+
description: ['魅力女友:嗲软轻飘的轻熟美人,妩媚有耐心,温柔勾人。'],
|
|
1512
|
+
gender: 'female',
|
|
1513
|
+
languages: ['zh', 'en'],
|
|
1514
|
+
type: 'volcano_tts_2',
|
|
1515
|
+
},
|
|
1516
|
+
{
|
|
1517
|
+
id: 'zh_female_santongyongns_saturn_bigtts',
|
|
1518
|
+
description: [
|
|
1519
|
+
'流畅女声:语调顺滑自然、咬字利落不拖沓、适配多种场景的百搭女声音色',
|
|
1520
|
+
],
|
|
1521
|
+
gender: 'female',
|
|
1522
|
+
languages: ['zh', 'en'],
|
|
1523
|
+
type: 'volcano_tts_2',
|
|
1524
|
+
},
|
|
1525
|
+
{
|
|
1526
|
+
id: 'zh_male_ruyayichen_saturn_bigtts',
|
|
1527
|
+
description: [
|
|
1528
|
+
'儒雅逸辰:语气温润有礼、咬字文雅舒缓、自带书卷气的儒雅男声音色',
|
|
1529
|
+
],
|
|
1530
|
+
gender: 'male',
|
|
1531
|
+
languages: ['zh', 'en'],
|
|
1532
|
+
type: 'volcano_tts_2',
|
|
1533
|
+
},
|
|
1534
|
+
{
|
|
1535
|
+
id: 'saturn_zh_female_keainvsheng_tob',
|
|
1536
|
+
description: ['可爱女生:可爱的妹妹,性格活泼,举动讨喜。'],
|
|
1537
|
+
gender: 'female',
|
|
1538
|
+
languages: ['zh', 'en'],
|
|
1539
|
+
type: 'volcano_tts_2',
|
|
1540
|
+
},
|
|
1541
|
+
{
|
|
1542
|
+
id: 'saturn_zh_female_tiaopigongzhu_tob',
|
|
1543
|
+
description: [
|
|
1544
|
+
'调皮公主:语调灵动跳脱、尾音带娇俏、古灵精怪爱撒娇的少女音色',
|
|
1545
|
+
],
|
|
1546
|
+
gender: 'female',
|
|
1547
|
+
languages: ['zh', 'en'],
|
|
1548
|
+
type: 'volcano_tts_2',
|
|
1549
|
+
},
|
|
1550
|
+
{
|
|
1551
|
+
id: 'saturn_zh_male_shuanglangshaonian_tob',
|
|
1552
|
+
description: [
|
|
1553
|
+
'爽朗少年:语气洪亮开阔、咬字干脆爽朗、阳光率真不扭捏的少年音色',
|
|
1554
|
+
],
|
|
1555
|
+
gender: 'male',
|
|
1556
|
+
languages: ['zh', 'en'],
|
|
1557
|
+
type: 'volcano_tts_2',
|
|
1558
|
+
},
|
|
1559
|
+
{
|
|
1560
|
+
id: 'saturn_zh_male_tiancaitongzhuo_tob',
|
|
1561
|
+
description: [
|
|
1562
|
+
'天才同桌:语气明快利落、咬字清晰带活力、聪明机敏又毒舌的少年音色',
|
|
1563
|
+
],
|
|
1564
|
+
gender: 'male',
|
|
1565
|
+
languages: ['zh', 'en'],
|
|
1566
|
+
type: 'volcano_tts_2',
|
|
1567
|
+
},
|
|
1568
|
+
{
|
|
1569
|
+
id: 'saturn_zh_female_cancan_tob',
|
|
1570
|
+
description: [
|
|
1571
|
+
'知性灿灿:语气温柔平缓、带有些许软绵感却又善解人意的少女音色',
|
|
1572
|
+
],
|
|
1573
|
+
gender: 'female',
|
|
1574
|
+
languages: ['zh', 'en'],
|
|
1575
|
+
type: 'volcano_tts_2',
|
|
1576
|
+
},
|
|
1577
|
+
];
|
|
1578
|
+
const list = await (await this.session.sandbox.request('/ai/voices/minimax')).json();
|
|
1579
|
+
const minimax_voices = list
|
|
1580
|
+
.filter((v) => v.description.length > 0)
|
|
1581
|
+
.map((v) => {
|
|
1582
|
+
const description = v.description[0];
|
|
1583
|
+
let gender;
|
|
1584
|
+
let languages = ['zh'];
|
|
1585
|
+
if (description.includes('男') ||
|
|
1586
|
+
description.includes('哥') ||
|
|
1587
|
+
description.includes('弟')) {
|
|
1588
|
+
gender = 'male';
|
|
1589
|
+
}
|
|
1590
|
+
else if (description.includes('女') ||
|
|
1591
|
+
description.includes('姐') ||
|
|
1592
|
+
description.includes('妹')) {
|
|
1593
|
+
gender = 'female';
|
|
1594
|
+
}
|
|
1595
|
+
if (description.includes('英语') || v.voice_id.startsWith('English')) {
|
|
1596
|
+
languages = ['en'];
|
|
1597
|
+
}
|
|
1598
|
+
else if (description.includes('日语')) {
|
|
1599
|
+
languages = ['ja'];
|
|
1600
|
+
}
|
|
1601
|
+
else if (description.includes('韩语')) {
|
|
1602
|
+
languages = ['ko'];
|
|
1603
|
+
}
|
|
1604
|
+
else if (description.includes('西班牙语')) {
|
|
1605
|
+
languages = ['es'];
|
|
1606
|
+
}
|
|
1607
|
+
else if (description.includes('葡萄牙语')) {
|
|
1608
|
+
languages = ['pt'];
|
|
1609
|
+
}
|
|
1610
|
+
else if (description.includes('荷兰语')) {
|
|
1611
|
+
languages = ['nl'];
|
|
1612
|
+
}
|
|
1613
|
+
else if (description.includes('越南语')) {
|
|
1614
|
+
languages = ['vi'];
|
|
1615
|
+
}
|
|
1616
|
+
else if (description.includes('俄语')) {
|
|
1617
|
+
languages = ['ru'];
|
|
1618
|
+
}
|
|
1619
|
+
else if (description.includes('印度尼西亚语')) {
|
|
1620
|
+
languages = ['id'];
|
|
1621
|
+
}
|
|
1622
|
+
else if (description.includes('德语')) {
|
|
1623
|
+
languages = ['de'];
|
|
1624
|
+
}
|
|
1625
|
+
else if (description.includes('法语')) {
|
|
1626
|
+
languages = ['fr'];
|
|
1627
|
+
}
|
|
1628
|
+
else if (description.includes('意大利语')) {
|
|
1629
|
+
languages = ['it'];
|
|
1630
|
+
}
|
|
1631
|
+
else if (description.includes('阿拉伯语')) {
|
|
1632
|
+
languages = ['ar'];
|
|
1633
|
+
}
|
|
1634
|
+
else if (description.includes('土耳其语')) {
|
|
1635
|
+
languages = ['tr'];
|
|
1636
|
+
}
|
|
1637
|
+
else if (description.includes('乌克兰语')) {
|
|
1638
|
+
languages = ['uk'];
|
|
1639
|
+
}
|
|
1640
|
+
return {
|
|
1641
|
+
id: v.voice_id,
|
|
1642
|
+
gender,
|
|
1643
|
+
languages,
|
|
1644
|
+
description: v.description[0],
|
|
1645
|
+
};
|
|
1646
|
+
});
|
|
1647
|
+
return [...volcano_tts_2_voices, ...minimax_voices];
|
|
1648
|
+
}
|
|
1649
|
+
async pickVoice(options) {
|
|
1650
|
+
const voices = await this.listVoices();
|
|
1651
|
+
const prompt = `根据用户输入的语音上下文内容,从音色库数据中选择最合适的语音音色,将该音色ID和选择该音色的理由返回。
|
|
1652
|
+
|
|
1653
|
+
## 重要原则
|
|
1654
|
+
在合适的情况下,尽量优先采用 volcano_tts_2 类型的语音
|
|
1655
|
+
|
|
1656
|
+
## 音色库数据
|
|
1657
|
+
${JSON.stringify(voices)}
|
|
1658
|
+
|
|
1659
|
+
## 输出格式
|
|
1660
|
+
{
|
|
1661
|
+
"voice_id": "音色ID",
|
|
1662
|
+
"reason": "选择该音色理由"
|
|
1663
|
+
}
|
|
1664
|
+
`;
|
|
1665
|
+
const schema = {
|
|
1666
|
+
name: 'emotion_schema',
|
|
1667
|
+
schema: {
|
|
1668
|
+
type: 'object',
|
|
1669
|
+
properties: {
|
|
1670
|
+
voice_id: {
|
|
1671
|
+
type: 'string',
|
|
1672
|
+
description: '从语音列表中选出的语音ID',
|
|
1673
|
+
},
|
|
1674
|
+
reason: {
|
|
1675
|
+
type: 'string',
|
|
1676
|
+
description: '选择的理由',
|
|
1677
|
+
},
|
|
1678
|
+
},
|
|
1679
|
+
required: ['voice_id', 'reason'],
|
|
1680
|
+
},
|
|
1681
|
+
};
|
|
1682
|
+
const payload = {
|
|
1683
|
+
model: 'Doubao-Seed-1.6',
|
|
1684
|
+
messages: [
|
|
1685
|
+
{
|
|
1686
|
+
role: 'system',
|
|
1687
|
+
content: prompt,
|
|
1688
|
+
},
|
|
1689
|
+
{
|
|
1690
|
+
role: 'user',
|
|
1691
|
+
content: `## 语音上下文
|
|
1692
|
+
|
|
1693
|
+
${options.prompt}
|
|
1694
|
+
`,
|
|
1695
|
+
},
|
|
1696
|
+
],
|
|
1697
|
+
response_format: {
|
|
1698
|
+
type: 'json_schema',
|
|
1699
|
+
json_schema: schema,
|
|
1700
|
+
},
|
|
1701
|
+
};
|
|
1702
|
+
const completion = await this.getCompletions(payload);
|
|
1703
|
+
const voiceObj = JSON.parse(completion.choices[0]?.message?.content ?? '{}');
|
|
1704
|
+
return voiceObj;
|
|
1705
|
+
}
|
|
1485
1706
|
};
|
|
1486
1707
|
exports.AI = AI;
|
|
1487
1708
|
exports.AI = AI = __decorate([
|