cerevox 3.0.0-beta.3 → 3.0.0-beta.30

package/dist/core/ai.js

@@ -108,80 +108,91 @@ let AI = class AI extends base_1.BaseClass {
         const data = await res.json();
         return data;
     }
-    async generateImageSeries(options) {
-        try {
-            if (Array.isArray(options.prompts)) {
-                const count = options.prompts.length;
-                options.prompts = options.prompts
-                    .map((p, i) => `图片${i + 1}: ${p.trim()}`)
-                    .join('\n');
-                options.prompts = `生成${count}张图片\n${options.prompts.trim()}`;
-            }
-            if (options.refPrefix) {
-                options.prompts = `${options.refPrefix}
-
-参考以上图片，执行以下各场景绘图：
-
-${options.prompts};
-`;
-            }
-            const max_count = options.max_count ?? 15;
-            const payload = {
-                prompt: options.prompts,
-                size: options.size ?? '1280x720',
-                watermark: options.watermark ?? false,
-                image: options.image,
-                max_images: max_count,
-            };
-            // 启动心跳机制
-            let heartbeatInterval = null;
-            if (options.onProgress) {
-                heartbeatInterval = setInterval(() => {
-                    options.onProgress?.({});
-                }, 60000); // 每60秒调用一次
-            }
-            try {
-                const res = await this.session.sandbox.request('/ai/image/sequential_generate', {
-                    method: 'POST',
-                    headers: {
-                        'Content-Type': 'application/json',
-                    },
-                    body: JSON.stringify(payload),
-                });
-                const data = await res.json();
-                // 清除心跳定时器
-                if (heartbeatInterval) {
-                    clearInterval(heartbeatInterval);
-                }
-                if (data.data) {
-                    const urls = data.data
-                        .map((item) => item.url)
-                        .filter((url) => url?.trim());
-                    for (const url of urls) {
-                        this.session.track('Image Generated', { url });
-                    }
-                    return {
-                        prompt: options.prompts,
-                        data,
-                        urls,
-                    };
-                }
-                return data;
-            }
-            catch (error) {
-                // 确保在出错时也清除心跳定时器
-                if (heartbeatInterval) {
-                    clearInterval(heartbeatInterval);
-                }
-                throw error;
-            }
-        }
-        catch (error) {
-            this.logger.error('generateImage error', error);
-            return { error: `generateImage error: ${error}` };
-        }
-    }
+    //   async generateImageSeries(options: {
+    //     prompts: string | string[];
+    //     refPrefix?: string; // 参考图前缀，默认 ''
+    //     size?: string;
+    //     watermark?: boolean;
+    //     image?: string[]; // 参考图（可多张）
+    //     max_count?: number; // 最大生成数量，默认 15
+    //     onProgress?: (metaData: any) => void;
+    //   }) {
+    //     try {
+    //       if (Array.isArray(options.prompts)) {
+    //         const count = options.prompts.length;
+    //         options.prompts = options.prompts
+    //           .map((p, i) => `图片${i + 1}: ${p.trim()}`)
+    //           .join('\n');
+    //         options.prompts = `生成${count}张图片\n${options.prompts.trim()}`;
+    //       }
+    //       if (options.refPrefix) {
+    //         options.prompts = `${options.refPrefix}
+    // 参考以上图片，执行以下各场景绘图：
+    // ${options.prompts};
+    // `;
+    //       }
+    //       const max_count = options.max_count ?? 15;
+    //       const payload = {
+    //         prompt: options.prompts,
+    //         size: options.size ?? '1280x720',
+    //         watermark: options.watermark ?? false,
+    //         image: options.image,
+    //         max_images: max_count,
+    //       };
+    //       // 启动心跳机制
+    //       let heartbeatInterval: NodeJS.Timeout | null = null;
+    //       if (options.onProgress) {
+    //         heartbeatInterval = setInterval(() => {
+    //           options.onProgress?.({});
+    //         }, 60000); // 每60秒调用一次
+    //       }
+    //       try {
+    //         const res = await this.session.sandbox.request(
+    //           '/ai/image/sequential_generate',
+    //           {
+    //             method: 'POST',
+    //             headers: {
+    //               'Content-Type': 'application/json',
+    //             },
+    //             body: JSON.stringify(payload),
+    //           }
+    //         );
+    //         const data = await res.json();
+    //         // 清除心跳定时器
+    //         if (heartbeatInterval) {
+    //           clearInterval(heartbeatInterval);
+    //         }
+    //         if (data.data) {
+    //           const urls = data.data
+    //             .map((item: any) => item.url)
+    //             .filter((url: any) => url?.trim());
+    //           for (const url of urls) {
+    //             this.session.track('Image Generated', { url });
+    //           }
+    //           return {
+    //             prompt: options.prompts,
+    //             data,
+    //             urls,
+    //           };
+    //         }
+    //         return data;
+    //       } catch (error) {
+    //         // 确保在出错时也清除心跳定时器
+    //         if (heartbeatInterval) {
+    //           clearInterval(heartbeatInterval);
+    //         }
+    //         throw error;
+    //       }
+    //     } catch (error) {
+    //       this.logger.error('generateImage error', error);
+    //       return { error: `generateImage error: ${error}` };
+    //     }
+    //   }
     async generateImage(options) {
+        if (options.type === 'line-sketch') {
+            options.prompt = `请根据以下描述绘制一张线稿：\n\n${options.prompt}\n\n画面除了主体和提到的物品之外，不要画其他任何东西，留白`;
+            options.type = 'banana';
+        }
         try {
             const res = await this.session.sandbox.request(`/ai/image/generate`, {
                 method: 'POST',
@@ -191,18 +202,20 @@ ${options.prompts};
                 body: JSON.stringify(options),
             });
             const data = await res.json();
-            if (data.data?.[0]?.url) {
-                this.session.track('Image Generated', data);
+            if (data.data) {
+                const urls = data.data
+                    .map((item) => item.url)
+                    .filter((url) => url?.trim());
+                for (const url of urls) {
+                    this.session.track('Image Generated', { url });
+                }
                 return {
-                    url: data.data?.[0]?.url,
+                    prompt: options.prompt,
+                    // data,
+                    urls,
                 };
             }
-            else if (data.url) {
-                // banana
-                this.session.track('Image Generated', data);
-                return data;
-            }
-            throw new Error(`generateImage error: ${JSON.stringify(data)}`);
+            return data;
         }
         catch (error) {
             this.logger.error('generateImage error', error);
@@ -210,28 +223,6 @@ ${options.prompts};
             throw error;
         }
     }
-    async generateLineSketch(options) {
-        try {
-            const res = await this.session.sandbox.request(`/ai/image/line_sketch`, {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                },
-                body: JSON.stringify(options),
-            });
-            const data = await res.json();
-            if (data.url) {
-                this.session.track('Line Sketch Generated', data);
-                return data;
-            }
-            throw new Error(`generateLineSketch error: ${JSON.stringify(data)}`);
-        }
-        catch (error) {
-            this.logger.error('generateLineSketch error', error);
-            this.session.track('Line Sketch Generated', { error: error.message });
-            throw error;
-        }
-    }
     async extendVideo(options) {
         try {
             const waitForFinish = options.waitForFinish ?? true;
@@ -299,6 +290,27 @@ ${options.prompts};
             return { error: error.message };
         }
     }
+    async generateZeroCutMusicVideo(options) {
+        try {
+            const waitForFinish = options.waitForFinish ?? true;
+            const workflowId = '7574088452263591988';
+            const parameters = {
+                prompt: options.prompt,
+                singer_apperance: options.singerPhoto,
+                orientation: options.orientation,
+                resolution: options.resolution,
+                music_duration: options.duration,
+                origin_song: options.originalSong,
+                gen_subtitles: options.genSubtitles,
+            };
+            const result = await this.runCozeWorkflow(workflowId, parameters, options.onProgress, 1500, waitForFinish);
+            return result;
+        }
+        catch (error) {
+            this.logger.error('Generate ZeroCut Video error', error);
+            return { error: error.message, content: null };
+        }
+    }
     // 用自己的模型生成 video
     async generateZeroCutVideo(options) {
         try {
@@ -738,7 +750,6 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
     async textToSpeechVolc(options) {
         const speech_rate = options.speed ?? 0; // -50 ~ 100
         const loudness_rate = options.volume ?? 0; // -50 ~ 100
-        const emotion = options.emotion ?? 'storytelling';
         const explicit_language = options.explicit_language ?? 'zh';
         // 随机停顿 300 ～ 500 ms 模拟人对话
         const silence_duration = options.silence_duration ?? Math.floor(Math.random() * 200 + 300);
@@ -753,10 +764,10 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
                     text: options.text,
                     speech_rate,
                     loudness_rate,
-                    emotion,
                     silence_duration,
                     explicit_language,
                     context_texts: options.context_texts,
+                    version: options.version,
                 }),
             });
             const data = await res.json();
@@ -1279,6 +1290,25 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
             return { error: error.message };
         }
     }
+    async splitVideoAndAudio(options) {
+        try {
+            const res = await this.session.sandbox.request(`/ai/video/split`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                    url: options.videoUrl,
+                }),
+            });
+            const data = await res.json();
+            return data;
+        }
+        catch (error) {
+            this.logger.error('splitVideoAndAudio error', error);
+            return { error: error.message };
+        }
+    }
     async getCozeWorkflowInfo(workflowId) {
         return (0, coze_1.getWorkflowInfo)(workflowId, this.session.sandbox.token);
     }
@@ -1317,7 +1347,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
     }
     async waitForWorkflowFinish(options) {
         try {
-            const { taskUrl, onProgress, interval = 1500, timeout = 900000, } = options;
+            const { taskUrl, onProgress, interval = 1500, timeout = 1800000, } = options;
             const result = await Promise.race([
                 waitForWorkflowFinish(taskUrl, this.session.sandbox.token, this.logger, onProgress, interval),
                 new Promise(resolve => {
@@ -1325,6 +1355,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
                         resolve({
                             status: 'running',
                             reason: `wait timeout ${timeout}ms`,
+                            nextAction: '可继续等待',
                             taskUrl,
                         });
                     }, timeout);
@@ -1359,7 +1390,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
     }
     async waitForVideoTaskComplete(options) {
         try {
-            const { taskUrl, onProgress, interval = 1500, traceWorkflow = true, timeout = 900000, } = options;
+            const { taskUrl, onProgress, interval = 1500, traceWorkflow = true, timeout = 1800000, } = options;
             if (taskUrl.startsWith('https://api.coze.cn/v1/workflows/')) {
                 return this.waitForWorkflowFinish(options);
             }
@@ -1378,6 +1409,7 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
                         resolve({
                             status: 'running',
                             reason: `wait timeout ${timeout}ms`,
+                            nextAction: '可继续等待',
                             taskUrl,
                         });
                     }, timeout);
@@ -1482,6 +1514,263 @@ ${timbre ? `timbre: ${timbre}` : ''}`;
     async listContextRules() {
         return (await this.session.sandbox.request('/ai/personas')).json();
     }
+    async listVoices() {
+        const volcano_tts_2_voices = [
+            {
+                id: 'zh_female_vv_uranus_bigtts',
+                description: ['语调平稳、咬字柔和、自带治愈安抚力的女声音色'],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_female_xiaohe_uranus_bigtts',
+                description: ['声线甜美有活力的妹妹，活泼开朗，笑容明媚。'],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_male_m191_uranus_bigtts',
+                description: ['声音磁性的男生，成熟理性，做事有条理，让人信赖。'],
+                gender: 'male',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_male_taocheng_uranus_bigtts',
+                description: ['眉目清朗男大，清澈温润有朝气，开朗真诚。'],
+                gender: 'male',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_female_xueayi_saturn_bigtts',
+                description: ['温柔又有力量的音色，陪孩子在故事里勇敢探索！'],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_male_dayi_saturn_bigtts',
+                description: ['语气沉稳有力、咬字清晰厚重、自带可靠安全感的男声音色'],
+                gender: 'male',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_female_mizai_saturn_bigtts',
+                description: ['语调沉稳、咬字扎实、自带踏实和包容感的女声音色'],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_female_jitangnv_saturn_bigtts',
+                description: [
+                    '鸡汤女：语气轻软、尾音轻落、带有自然呼吸感且十分亲切的女声音色',
+                ],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_female_meilinvyou_saturn_bigtts',
+                description: ['魅力女友：嗲软轻飘的轻熟美人，妩媚有耐心，温柔勾人。'],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_female_santongyongns_saturn_bigtts',
+                description: [
+                    '流畅女声：语调顺滑自然、咬字利落不拖沓、适配多种场景的百搭女声音色',
+                ],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'zh_male_ruyayichen_saturn_bigtts',
+                description: [
+                    '儒雅逸辰：语气温润有礼、咬字文雅舒缓、自带书卷气的儒雅男声音色',
+                ],
+                gender: 'male',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'saturn_zh_female_keainvsheng_tob',
+                description: ['可爱女生：可爱的妹妹，性格活泼，举动讨喜。'],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'saturn_zh_female_tiaopigongzhu_tob',
+                description: [
+                    '调皮公主：语调灵动跳脱、尾音带娇俏、古灵精怪爱撒娇的少女音色',
+                ],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'saturn_zh_male_shuanglangshaonian_tob',
+                description: [
+                    '爽朗少年：语气洪亮开阔、咬字干脆爽朗、阳光率真不扭捏的少年音色',
+                ],
+                gender: 'male',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'saturn_zh_male_tiancaitongzhuo_tob',
+                description: [
+                    '天才同桌：语气明快利落、咬字清晰带活力、聪明机敏又毒舌的少年音色',
+                ],
+                gender: 'male',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+            {
+                id: 'saturn_zh_female_cancan_tob',
+                description: [
+                    '知性灿灿：语气温柔平缓、带有些许软绵感却又善解人意的少女音色',
+                ],
+                gender: 'female',
+                languages: ['zh', 'en'],
+                type: 'volcano_tts_2',
+            },
+        ];
+        const list = await (await this.session.sandbox.request('/ai/voices/minimax')).json();
+        const minimax_voices = list
+            .filter((v) => v.description.length > 0)
+            .map((v) => {
+            const description = v.description[0];
+            let gender;
+            let languages = ['zh'];
+            if (description.includes('男') ||
+                description.includes('哥') ||
+                description.includes('弟')) {
+                gender = 'male';
+            }
+            else if (description.includes('女') ||
+                description.includes('姐') ||
+                description.includes('妹')) {
+                gender = 'female';
+            }
+            if (description.includes('英语') || v.voice_id.startsWith('English')) {
+                languages = ['en'];
+            }
+            else if (description.includes('日语')) {
+                languages = ['ja'];
+            }
+            else if (description.includes('韩语')) {
+                languages = ['ko'];
+            }
+            else if (description.includes('西班牙语')) {
+                languages = ['es'];
+            }
+            else if (description.includes('葡萄牙语')) {
+                languages = ['pt'];
+            }
+            else if (description.includes('荷兰语')) {
+                languages = ['nl'];
+            }
+            else if (description.includes('越南语')) {
+                languages = ['vi'];
+            }
+            else if (description.includes('俄语')) {
+                languages = ['ru'];
+            }
+            else if (description.includes('印度尼西亚语')) {
+                languages = ['id'];
+            }
+            else if (description.includes('德语')) {
+                languages = ['de'];
+            }
+            else if (description.includes('法语')) {
+                languages = ['fr'];
+            }
+            else if (description.includes('意大利语')) {
+                languages = ['it'];
+            }
+            else if (description.includes('阿拉伯语')) {
+                languages = ['ar'];
+            }
+            else if (description.includes('土耳其语')) {
+                languages = ['tr'];
+            }
+            else if (description.includes('乌克兰语')) {
+                languages = ['uk'];
+            }
+            return {
+                id: v.voice_id,
+                gender,
+                languages,
+                description: v.description[0],
+            };
+        });
+        return [...volcano_tts_2_voices, ...minimax_voices];
+    }
+    async pickVoice(options) {
+        const voices = await this.listVoices();
+        const prompt = `根据用户输入的语音上下文内容，从音色库数据中选择最合适的语音音色，将该音色ID和选择该音色的理由返回。
+
+## 重要原则
+在合适的情况下，尽量优先采用 volcano_tts_2 类型的语音
+
+## 音色库数据
+${JSON.stringify(voices)}
+
+## 输出格式
+{
+  "voice_id": "音色ID",
+  "reason": "选择该音色理由"
+}
+`;
+        const schema = {
+            name: 'emotion_schema',
+            schema: {
+                type: 'object',
+                properties: {
+                    voice_id: {
+                        type: 'string',
+                        description: '从语音列表中选出的语音ID',
+                    },
+                    reason: {
+                        type: 'string',
+                        description: '选择的理由',
+                    },
+                },
+                required: ['voice_id', 'reason'],
+            },
+        };
+        const payload = {
+            model: 'Doubao-Seed-1.6',
+            messages: [
+                {
+                    role: 'system',
+                    content: prompt,
+                },
+                {
+                    role: 'user',
+                    content: `## 语音上下文
+
+${options.prompt}
+`,
+                },
+            ],
+            response_format: {
+                type: 'json_schema',
+                json_schema: schema,
+            },
+        };
+        const completion = await this.getCompletions(payload);
+        const voiceObj = JSON.parse(completion.choices[0]?.message?.content ?? '{}');
+        return voiceObj;
+    }
 };
 exports.AI = AI;
 exports.AI = AI = __decorate([