@remixmate/cli 0.13.3 → 0.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handlers/gen-digital-human.js +29 -23
- package/dist/handlers/gen-voice.js +8 -8
- package/dist/manifest.json +8 -3
- package/package.json +1 -1
- package/skills/gen-digital-human/SKILL.md +7 -4
- package/skills/gen-digital-human/skill.json +6 -1
- package/skills/gen-digital-human/version.json +1 -1
- package/skills/render-video/scripts/_continuous_narration.py +468 -0
- package/skills/render-video/scripts/render_video.py +321 -31
- package/skills/render-video/version.json +1 -1
- package/skills/template-registry/SKILL.md +1 -0
- package/skills/template-registry/video_dsl/schema/video-dsl-v1alpha1.json +4 -1
|
@@ -1,33 +1,39 @@
|
|
|
1
|
-
var
|
|
2
|
-
`);return}if(
|
|
3
|
-
|
|
1
|
+
var k=Object.defineProperty;var f=(t,e)=>k(t,"name",{value:e,configurable:!0});import{mmPost as p,pollUntil as y,resolveHttpContext as U,SkillError as i}from"../http.js";import{getCapabilities as b}from"../capabilities.js";import{emitProgress as m}from"../progress.js";import{isTrue as w,toNumber as M}from"./shared.js";const E=["video","image"],S="hifly";function l(t){return t==null?"":String(t).trim()}f(l,"toId");async function N(t,e){const r=w(e.mine),a=!r&&w(e.all),n=!r&&!a,s={current:1,pageSize:100};n?s.featured=!0:s.source=S,r&&(s.mine=!0),e.gender&&(s.gender=String(e.gender)),e.name&&(s.name=String(e.name));const g=await p(t,"/digital-human/avatar/page",s,{timeoutMs:3e4}),o=g.list??[];if(w(e.json_output)){process.stdout.write(JSON.stringify({avatars:o})+`
|
|
2
|
+
`);return}if(o.length===0){process.stdout.write(n?`⚠️ No curated public avatars match. Try --all for the full inventory, or --mine for your own.
|
|
3
|
+
`:`⚠️ No avatars available
|
|
4
|
+
`);return}if(n){process.stdout.write(`
|
|
5
|
+
${"ID".padEnd(24)} ${"Name".padEnd(16)} ${"Gender".padEnd(8)} ${"Ratio".padEnd(6)} ${"Scenes".padEnd(22)} Voice
|
|
6
|
+
`),process.stdout.write(`${"-".repeat(24)} ${"-".repeat(16)} ${"-".repeat(8)} ${"-".repeat(6)} ${"-".repeat(22)} ${"-".repeat(12)}
|
|
7
|
+
`);for(const d of o)process.stdout.write(`${l(d.id).padEnd(24)} ${String(d.name??"").padEnd(16)} ${String(d.gender??"").padEnd(8)} ${String(d.aspectRatio??"").padEnd(6)} ${(d.tags??[]).join(",").padEnd(22)} ${d.defaultVoiceId||"-"}
|
|
8
|
+
`)}else{process.stdout.write(`
|
|
4
9
|
${"ID".padEnd(24)} ${"Name".padEnd(16)} ${"Source".padEnd(14)} ${"Gender".padEnd(8)} ${"Ratio".padEnd(6)} Clone
|
|
5
10
|
`),process.stdout.write(`${"-".repeat(24)} ${"-".repeat(16)} ${"-".repeat(14)} ${"-".repeat(8)} ${"-".repeat(6)} ${"-".repeat(8)}
|
|
6
|
-
`);for(const d of
|
|
7
|
-
`)
|
|
8
|
-
${
|
|
9
|
-
`),process.stdout.write(
|
|
10
|
-
|
|
11
|
+
`);for(const d of o)process.stdout.write(`${l(d.id).padEnd(24)} ${String(d.name??"").padEnd(16)} ${String(d.source??"").padEnd(14)} ${String(d.gender??"").padEnd(8)} ${String(d.aspectRatio??"").padEnd(6)} ${d.cloneStatus||"-"}
|
|
12
|
+
`)}const u=g.pager?.total??o.length;process.stdout.write(`
|
|
13
|
+
${o.length} avatar(s) shown${u>o.length?` of ${u}; narrow with --name`:""}
|
|
14
|
+
`),process.stdout.write(n?`💡 Use --avatar-id <ID> to pick an avatar; pass its Voice as --voice-id for a matching voice
|
|
15
|
+
`:`💡 Use --avatar-id <ID> to pick an avatar for generation
|
|
16
|
+
`)}f(N,"listAvatars");async function I(t,e){return((await p(t,"/digital-human/avatar/page",{current:1,pageSize:100,mine:!0,source:S},{timeoutMs:3e4})).list??[]).find(n=>l(n.id)===e)??{id:e,source:S}}f(I,"fetchAvatarInfo");function O(t,e){const r=(t.source??"").trim()||S;if(!e.includes(r))throw new i(`❌ avatar uses provider "${r}", which is not available (supported: ${e.join("/")}). Pick another avatar from --list-avatars.`);return r}f(O,"resolveSource");async function T(t,e){const r=l(e.clone_url);if(!/^https?:\/\//i.test(r))throw new i("❌ --clone needs --clone-url <public http(s) URL of a video or portrait image> (local files are not uploaded)");const a=l(e.name);if(!a)throw new i("❌ --clone needs --name <avatar name>");const n=l(e.clone_type)||"video";if(!E.includes(n))throw new i(`❌ --clone-type must be one of ${E.join(" / ")}, received: ${n}`);const s={name:a,sourceType:n,fileUrl:r};e.thumbnail_url&&(s.thumbnailUrl=l(e.thumbnail_url)),e.gender&&(s.gender=l(e.gender)),m({phase:"gen-digital-human:clone-request",sourceType:n});const g=await p(t,"/digital-human/avatar/clone",s,{timeoutMs:6e4}),o=l(g.id);if(!o)throw new i(`❌ could not parse the new avatar id; API returned: ${JSON.stringify(g)}`);m({phase:"gen-digital-human:clone-async",avatarId:o});const u=w(e.json_output);if(e.wait!==void 0&&!w(e.wait)){if(u){process.stdout.write(JSON.stringify({avatarId:o,cloneStatus:g.cloneStatus??"cloning"})+`
|
|
11
17
|
`);return}process.stdout.write(`
|
|
12
|
-
🧬 Clone submitted, avatar id: ${
|
|
13
|
-
`),process.stdout.write(`💡 Check progress with --clone-status --avatar-id ${
|
|
14
|
-
`);return}let
|
|
18
|
+
🧬 Clone submitted, avatar id: ${o}
|
|
19
|
+
`),process.stdout.write(`💡 Check progress with --clone-status --avatar-id ${o}
|
|
20
|
+
`);return}let d;try{d=await y(async()=>{const c=await p(t,"/digital-human/avatar/clone-status",{id:o},{timeoutMs:3e4}),h=c.cloneStatus??"unknown";if(m({phase:"gen-digital-human:clone-poll",avatarId:o,status:h}),h==="done")return c;if(h==="failed")throw new i(`❌ avatar clone failed: ${c.cloneError||"unknown error"}`);return null},{intervalMs:1e4,timeoutMs:20*6e4})}catch(c){throw c instanceof i&&/polling timed out/.test(c.message)?new i(`❌ avatar ${o} is still cloning; rerun --clone-status --avatar-id ${o} later`):c}if(u){process.stdout.write(JSON.stringify({avatarId:o,cloneStatus:d.cloneStatus})+`
|
|
15
21
|
`);return}process.stdout.write(`
|
|
16
|
-
✅ Avatar cloned, id: ${
|
|
17
|
-
`),process.stdout.write(`💡 Generate with --avatar-id ${
|
|
18
|
-
`)}
|
|
22
|
+
✅ Avatar cloned, id: ${o}
|
|
23
|
+
`),process.stdout.write(`💡 Generate with --avatar-id ${o} --text "..."
|
|
24
|
+
`)}f(T,"cloneAvatar");async function R(t,e,r){const a=await p(t,"/digital-human/avatar/clone-status",{id:e},{timeoutMs:3e4}),n=a.cloneStatus||"unknown";if(r){process.stdout.write(JSON.stringify({avatarId:e,cloneStatus:n,error:a.cloneError||null})+`
|
|
19
25
|
`);return}process.stdout.write(`
|
|
20
|
-
clone status: ${
|
|
21
|
-
`),
|
|
22
|
-
`)}
|
|
26
|
+
clone status: ${n}
|
|
27
|
+
`),n==="failed"&&process.stdout.write(` failure reason: ${a.cloneError||"unknown error"}
|
|
28
|
+
`)}f(R,"checkCloneStatus");async function C(t,e,r){const a=await p(t,"/digital-human/video/status",{generationId:e},{timeoutMs:3e4}),n=a.status??"unknown";if(r){process.stdout.write(JSON.stringify({status:n,url:a.videoUrl??null,generationId:e})+`
|
|
23
29
|
`);return}if(process.stdout.write(`
|
|
24
|
-
task status: ${
|
|
25
|
-
`),
|
|
26
|
-
`),
|
|
27
|
-
`)}}
|
|
30
|
+
task status: ${n}
|
|
31
|
+
`),a.videoUrl&&process.stdout.write(` video URL: ${a.videoUrl}
|
|
32
|
+
`),n==="failed"){const s=a.errorMessage??a.error??a.message??"unknown error";process.stdout.write(` failure reason: ${s}
|
|
33
|
+
`)}}f(C,"checkStatus");const j=30*6e4;async function J(t,e){return await y(async()=>{const a=await p(t,"/digital-human/video/status",{generationId:e},{timeoutMs:3e4}),n=a.status??"unknown";if(m({phase:"gen-digital-human:poll",generationId:e,status:n}),n==="completed")return a.videoUrl??"";if(n==="failed"){const s=a.errorMessage??a.error??a.message??"unknown error";throw new i(`❌ video generation failed: ${s}`)}return null},{intervalMs:5e3,timeoutMs:j}).catch(a=>{throw a instanceof i&&/polling timed out/.test(a.message)?new i(`${a.message}; still generating on the server — resume with --check-status --generation-id ${e}`):a})}f(J,"pollVideoStatus");async function x(t,e){const r=await U(e.skillName,{apiBaseUrl:t.api_base_url,privateToken:t.priv_token,preflightToken:e.auth?.token}),a=w(t.json_output);if(w(t.list_avatars))return N(r,t);if(w(t.clone))return T(r,t);if(w(t.clone_status)){const $=l(t.avatar_id);if(!$)throw new i("❌ clone-status mode requires --avatar-id");return R(r,$,a)}if(w(t.check_status)){const $=M(t.generation_id,"generation_id");if($===void 0)throw new i("❌ check-status mode requires --generation-id");return C(r,$,a)}const n=l(t.avatar_id);if(!n)throw new i("❌ provide --avatar-id to pick the avatar (use --list-avatars to see available avatars)");const s=(t.audio_url??"").trim(),g=t.text??"";if(s&&!/^https?:\/\//i.test(s))throw new i(`❌ --audio-url needs a public http(s) URL (local files are not uploaded), received: ${s}`);if(!s&&!g.trim())throw new i("❌ in TTS mode provide --text, or use --audio-url for audio-driven mode");const o=(await b(t.api_base_url)).digitalHuman;m({phase:"gen-digital-human:resolve-avatar",avatarId:n});const u=await I(r,n);if(u.cloneStatus==="cloning")throw new i(`❌ avatar ${n} is still cloning; check with --clone-status --avatar-id ${n}`);if(u.cloneStatus==="failed")throw new i(`❌ avatar ${n} failed to clone (${u.cloneError||"unknown error"}); clone it again`);const d=O(u,o.sources),c={avatarId:n,source:d};s?c.audioUrl=s:(c.text=g,c.voiceId=t.voice_id??o.defaultVoiceId,t.voice_name&&(c.voiceName=String(t.voice_name))),m({phase:"gen-digital-human:request",avatarId:n,source:d,mode:s?"audio":"tts"});const h=await p(r,"/digital-human/video/generate",c,{timeoutMs:6e4}),v=h.generationId;if(!v)throw new i(`❌ could not parse generationId; API returned: ${JSON.stringify(h)}`);m({phase:"gen-digital-human:async",generationId:v});const _=await J(r,v);if(!_)throw new i(`❌ video generated but no URL was returned; rerun --check-status --generation-id ${v} later`);if(a){process.stdout.write(JSON.stringify({url:_,generationId:v})+`
|
|
28
34
|
`);return}process.stdout.write(`
|
|
29
35
|
🔗 Video URL (renders directly, no download needed):
|
|
30
|
-
${
|
|
36
|
+
${_}
|
|
31
37
|
`),process.stdout.write(`
|
|
32
38
|
🎉 Digital-human video generation done!
|
|
33
|
-
`)}
|
|
39
|
+
`)}f(x,"genDigitalHuman");export{x as genDigitalHuman};
|
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
var w=Object.defineProperty;var
|
|
1
|
+
var w=Object.defineProperty;var d=(e,i)=>w(e,"name",{value:i,configurable:!0});import{mmPost as m,resolveHttpContext as v,SkillError as c}from"../http.js";import{getCapabilities as y}from"../capabilities.js";import{emitProgress as x}from"../progress.js";import{isTrue as u,toNumber as _}from"./shared.js";const b=[{id:"Chinese (Mandarin)_Male_Announcer",language:"zh",name:"Mandarin Male Announcer (legacy default)"},{id:"english-male-friendly-01",language:"en",name:"Friendly Male (English, TBD)"}];async function $(e){const o=(await m(e,"/voice/page",{current:1,pageSize:100},{timeoutMs:3e4})).list??[];if(o.length===0){process.stdout.write(`⚠️ No voices available
|
|
2
2
|
`);return}process.stdout.write(`
|
|
3
3
|
${"Voice ID".padEnd(30)} Name
|
|
4
4
|
`),process.stdout.write(`${"-".repeat(30)} ${"-".repeat(20)}
|
|
5
|
-
`);for(const s of
|
|
5
|
+
`);for(const s of o)process.stdout.write(`${(s.uniqId??"").padEnd(30)} ${s.name??""}
|
|
6
6
|
`);process.stdout.write(`
|
|
7
|
-
${
|
|
7
|
+
${o.length} voice(s) available
|
|
8
8
|
`),process.stdout.write(`💡 Use --voice-id <id> to select a voice for synthesis
|
|
9
|
-
`)}
|
|
9
|
+
`)}d($,"listVoicesRemote");function k(){process.stderr.write(`⚠️ --local prints a hardcoded 2-entry fallback list, NOT the available voices. Drop --local to query the live catalog.
|
|
10
10
|
`);for(const e of b)process.stdout.write(`${e.id} ${e.language} ${e.name}
|
|
11
|
-
`)}
|
|
12
|
-
`);return}const f=
|
|
11
|
+
`)}d(k,"listVoicesLocal");async function L(e,i){if(u(e.list_voices)){if(u(e.local))return k();const t=await v(i.skillName,{apiBaseUrl:e.api_base_url,privateToken:e.priv_token,preflightToken:i.auth?.token});return $(t)}const o=e.text;if(!o||typeof o!="string"||!o.trim())throw new c("❌ Please provide --text to specify the text to synthesize");const s=e.api_base_url,n=(await y(s)).voice,p=e.voice_id??n.defaultVoiceId,r=_(e.speed,"speed")??1;if(r<n.speedMin||r>n.speedMax)throw new c(`❌ speed must be in the range ${n.speedMin}~${n.speedMax}`);const g=e.language_boost?.trim()||void 0,h=await v(i.skillName,{apiBaseUrl:s,privateToken:e.priv_token,preflightToken:i.auth?.token});x({phase:"gen-voice:request",voiceId:p,speed:r});const a=await m(h,"/tool/tts",{provider:n.provider,text:o,voiceSetting:{voiceId:p,speed:r},outputFormat:"url",...g?{languageBoost:g}:{}},{timeoutMs:12e4}),l=a.audio;if(!l)throw new c(`❌ API response did not contain an audio URL: ${JSON.stringify(a)}`);if(u(e.json_output)){process.stdout.write(JSON.stringify({url:l,audio_length_ms:a.extraInfo?.audioLength??null,subtitles:(a.subtitles??[]).map(t=>({text:t.text??"",timeBegin:t.timeBegin??0,timeEnd:t.timeEnd??0,...typeof t.textBegin=="number"&&typeof t.textEnd=="number"?{textBegin:t.textBegin,textEnd:t.textEnd}:{}}))})+`
|
|
12
|
+
`);return}const f=a.extraInfo?.audioLength;typeof f=="number"&&process.stdout.write(`
|
|
13
13
|
🕐 Audio length: ${(f/1e3).toFixed(1)} s
|
|
14
14
|
`),process.stdout.write(`
|
|
15
15
|
🔗 Audio URL (plays directly, no download needed):
|
|
16
|
-
${
|
|
16
|
+
${l}
|
|
17
17
|
`),process.stdout.write(`
|
|
18
18
|
🎉 TTS synthesis done!
|
|
19
|
-
`)}
|
|
19
|
+
`)}d(L,"genVoice");export{L as genVoice};
|
package/dist/manifest.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": 1,
|
|
3
|
-
"version": "0.13.
|
|
4
|
-
"generatedAt": "2026-10-
|
|
3
|
+
"version": "0.13.5",
|
|
4
|
+
"generatedAt": "2026-10-28T14:51:31.746Z",
|
|
5
5
|
"skills": [
|
|
6
6
|
{
|
|
7
7
|
"id": "doc-parse",
|
|
@@ -195,12 +195,16 @@
|
|
|
195
195
|
"properties": {
|
|
196
196
|
"list_avatars": {
|
|
197
197
|
"type": "boolean",
|
|
198
|
-
"description": "List available digital-human avatars"
|
|
198
|
+
"description": "List available digital-human avatars. By default lists the curated public avatars (with scene tags and a suggested voice); see mine / all for other scopes."
|
|
199
199
|
},
|
|
200
200
|
"mine": {
|
|
201
201
|
"type": "boolean",
|
|
202
202
|
"description": "With list_avatars=true: list the caller's own custom avatars instead of the public catalog. Custom avatars do not appear without this."
|
|
203
203
|
},
|
|
204
|
+
"all": {
|
|
205
|
+
"type": "boolean",
|
|
206
|
+
"description": "With list_avatars=true: list the full public HiFly inventory (500+, mostly uncurated, no scene tags) instead of the curated public avatars. Only for when the curated set has nothing suitable."
|
|
207
|
+
},
|
|
204
208
|
"name": {
|
|
205
209
|
"type": "string",
|
|
206
210
|
"description": "With list_avatars=true: fuzzy-filter avatars by name. With clone=true: the new avatar's name (required)."
|
|
@@ -294,6 +298,7 @@
|
|
|
294
298
|
"wait",
|
|
295
299
|
"clone_status",
|
|
296
300
|
"mine",
|
|
301
|
+
"all",
|
|
297
302
|
"name",
|
|
298
303
|
"gender",
|
|
299
304
|
"check_status",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remixmate/cli",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.5",
|
|
4
4
|
"description": "AI media generation skills for Claude Code / Codex — 12 skills covering image, video, voice, digital human, web screenshot, web recording, script, template registry, rendering, Jianying export, and video deconstruction.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -25,7 +25,7 @@ Wraps ab-api's digital-human endpoints (the same ones the web studio uses), auth
|
|
|
25
25
|
|
|
26
26
|
| Capability | Description | API |
|
|
27
27
|
|------------|-------------|-----|
|
|
28
|
-
| List avatars |
|
|
28
|
+
| List avatars | Curated public avatars (default), your own with `--mine`, or the full inventory with `--all` | `POST /digital-human/avatar/page` |
|
|
29
29
|
| Text-driven | avatar + text + voice → speech is synthesized, then drives the avatar | `POST /digital-human/video/generate` |
|
|
30
30
|
| Audio-driven | avatar + existing audio URL → talking-head video | `POST /digital-human/video/generate` |
|
|
31
31
|
| Check status | Poll a generation job | `POST /digital-human/video/status` |
|
|
@@ -53,11 +53,14 @@ Generation typically takes 1–5 minutes; the handler polls and emits `__progres
|
|
|
53
53
|
### 1. List avatars
|
|
54
54
|
|
|
55
55
|
```bash
|
|
56
|
-
remixmate gen-digital-human --list-avatars
|
|
57
|
-
remixmate gen-digital-human --list-avatars --
|
|
58
|
-
remixmate gen-digital-human --list-avatars --
|
|
56
|
+
remixmate gen-digital-human --list-avatars # curated public avatars (default)
|
|
57
|
+
remixmate gen-digital-human --list-avatars --gender female # curated, filtered by gender
|
|
58
|
+
remixmate gen-digital-human --list-avatars --mine # your cloned avatars
|
|
59
|
+
remixmate gen-digital-human --list-avatars --all --name "教师" # full HiFly inventory (500+; always narrow with --name)
|
|
59
60
|
```
|
|
60
61
|
|
|
62
|
+
The default list is the **curated public library**: avatars an admin picked, with a cover, scene tags (`education` / `medical` / `news` / `business` / `culture` / `outdoor` / `lifestyle` / `festival` / `ecommerce`) and usually a suggested voice. Prefer it — pick the avatar whose scenes match the script, and pass its `Voice` column as `--voice-id` unless the user chose a voice. Fall back to `--all` only when nothing curated fits; those entries have no tags or suggested voice.
|
|
63
|
+
|
|
61
64
|
### 2. Generate a video
|
|
62
65
|
|
|
63
66
|
Pick exactly one driver: **text** (+ voice) or **audio**. With `--audio-url`, text and voice are ignored.
|
|
@@ -21,12 +21,16 @@
|
|
|
21
21
|
"properties": {
|
|
22
22
|
"list_avatars": {
|
|
23
23
|
"type": "boolean",
|
|
24
|
-
"description": "List available digital-human avatars"
|
|
24
|
+
"description": "List available digital-human avatars. By default lists the curated public avatars (with scene tags and a suggested voice); see mine / all for other scopes."
|
|
25
25
|
},
|
|
26
26
|
"mine": {
|
|
27
27
|
"type": "boolean",
|
|
28
28
|
"description": "With list_avatars=true: list the caller's own custom avatars instead of the public catalog. Custom avatars do not appear without this."
|
|
29
29
|
},
|
|
30
|
+
"all": {
|
|
31
|
+
"type": "boolean",
|
|
32
|
+
"description": "With list_avatars=true: list the full public HiFly inventory (500+, mostly uncurated, no scene tags) instead of the curated public avatars. Only for when the curated set has nothing suitable."
|
|
33
|
+
},
|
|
30
34
|
"name": {
|
|
31
35
|
"type": "string",
|
|
32
36
|
"description": "With list_avatars=true: fuzzy-filter avatars by name. With clone=true: the new avatar's name (required)."
|
|
@@ -120,6 +124,7 @@
|
|
|
120
124
|
"wait",
|
|
121
125
|
"clone_status",
|
|
122
126
|
"mine",
|
|
127
|
+
"all",
|
|
123
128
|
"name",
|
|
124
129
|
"gender",
|
|
125
130
|
"check_status",
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "gen-digital-human",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "336",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V11",
|
|
6
6
|
"skillDescription": "Digital-human (talking-head) skill backed by HiFly: list avatars, clone a custom avatar from a video or photo, drive an avatar with text (TTS) or an existing audio URL, and check the status of pending jobs.\n\nUse this skill immediately whenever the user asks for any of:\n- Digital human, talking-head video, AI presenter, virtual host\n- \"Make a talking-head video out of this script / this audio file\"\n- List or browse digital-human avatars\n- Clone / create a custom avatar from a video or photo\n\nEven without an explicit \"use AI\", any request that turns text or audio into a synthesized presenter video should route here."
|
|
7
7
|
}
|
|
@@ -0,0 +1,468 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
import re
|
|
26
|
+
import subprocess
|
|
27
|
+
import unicodedata
|
|
28
|
+
from typing import Optional
|
|
29
|
+
|
|
30
|
+
MODE_CONTINUOUS = "continuous"
|
|
31
|
+
MODE_PER_SCENE = "perScene"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
DEFAULT_SCENE_PAUSE_SEC = 0.8
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
MAX_RUN_CHARS = 1500
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
ESTIMATE_WINDOW_SEC = 2.0
|
|
45
|
+
_ESTIMATE_WINDOW_RATIO = 0.2
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
_NATURAL_PAUSE_SEC = 0.45
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
_MIN_BOUNDARY_PAUSE_RATIO = 0.75
|
|
52
|
+
|
|
53
|
+
_SILENCE_NOISE_DB = -40
|
|
54
|
+
_SILENCE_MIN_SEC = 0.25
|
|
55
|
+
|
|
56
|
+
PAUSE_TAG_RE = re.compile(r"<#\s*(\d+(?:\.\d+)?)\s*#>")
|
|
57
|
+
_LEADING_TAG_RE = re.compile(r"^\s*<#\s*(\d+(?:\.\d+)?)\s*#>\s*")
|
|
58
|
+
_TRAILING_TAG_RE = re.compile(r"\s*<#\s*(\d+(?:\.\d+)?)\s*#>\s*$")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _pause_value(raw) -> Optional[float]:
|
|
64
|
+
if isinstance(raw, bool) or not isinstance(raw, (int, float)):
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
return min(max(float(raw), 0.01), 99.99) if raw > 0 else None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def resolve_config(template: dict, override: Optional[dict]) -> dict:
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
declared = ((template or {}).get("capabilities") or {}).get("narration") or {}
|
|
76
|
+
override = override or {}
|
|
77
|
+
mode = MODE_PER_SCENE
|
|
78
|
+
for source in (declared, override):
|
|
79
|
+
if source.get("mode") in (MODE_CONTINUOUS, MODE_PER_SCENE):
|
|
80
|
+
mode = source["mode"]
|
|
81
|
+
pause = DEFAULT_SCENE_PAUSE_SEC
|
|
82
|
+
for source in (declared, override):
|
|
83
|
+
value = _pause_value(source.get("scenePauseSec"))
|
|
84
|
+
if value is not None:
|
|
85
|
+
pause = value
|
|
86
|
+
return {"mode": mode, "scenePauseSec": pause}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def group_runs(candidates: list[Optional[dict]], max_chars: int = MAX_RUN_CHARS) -> list[list[dict]]:
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
runs: list[list[dict]] = []
|
|
99
|
+
current: list[dict] = []
|
|
100
|
+
chars = 0
|
|
101
|
+
|
|
102
|
+
def flush() -> None:
|
|
103
|
+
if len(current) >= 2:
|
|
104
|
+
runs.append(list(current))
|
|
105
|
+
current.clear()
|
|
106
|
+
|
|
107
|
+
for cand in candidates:
|
|
108
|
+
if cand is None:
|
|
109
|
+
flush()
|
|
110
|
+
chars = 0
|
|
111
|
+
continue
|
|
112
|
+
size = len(cand["text"])
|
|
113
|
+
if current and (
|
|
114
|
+
cand.get("break")
|
|
115
|
+
or cand["voiceKey"] != current[-1]["voiceKey"]
|
|
116
|
+
or chars + size > max_chars
|
|
117
|
+
):
|
|
118
|
+
flush()
|
|
119
|
+
chars = 0
|
|
120
|
+
current.append(cand)
|
|
121
|
+
chars += size
|
|
122
|
+
flush()
|
|
123
|
+
return runs
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def format_pause(sec: float) -> str:
|
|
129
|
+
return f"{sec:.2f}".rstrip("0").rstrip(".")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def compose_run_text(texts: list[str], pause_sec: float) -> tuple[str, list[tuple[int, int]]]:
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
parts: list[str] = []
|
|
138
|
+
for i, text in enumerate(texts):
|
|
139
|
+
text = text.strip()
|
|
140
|
+
if i > 0:
|
|
141
|
+
text = _LEADING_TAG_RE.sub("", text, count=1)
|
|
142
|
+
if i < len(texts) - 1:
|
|
143
|
+
text = _TRAILING_TAG_RE.sub("", text, count=1)
|
|
144
|
+
parts.append(text)
|
|
145
|
+
|
|
146
|
+
pauses: list[float] = []
|
|
147
|
+
for i in range(len(texts) - 1):
|
|
148
|
+
candidates = [pause_sec]
|
|
149
|
+
tail = _TRAILING_TAG_RE.search(texts[i].strip())
|
|
150
|
+
head = _LEADING_TAG_RE.search(texts[i + 1].strip())
|
|
151
|
+
for m in (tail, head):
|
|
152
|
+
if m:
|
|
153
|
+
candidates.append(_pause_value(float(m.group(1))) or 0)
|
|
154
|
+
pauses.append(max(candidates))
|
|
155
|
+
|
|
156
|
+
out = ""
|
|
157
|
+
ranges: list[tuple[int, int]] = []
|
|
158
|
+
for i, part in enumerate(parts):
|
|
159
|
+
if i > 0:
|
|
160
|
+
out += f"\n<#{format_pause(pauses[i - 1])}#>\n"
|
|
161
|
+
start = len(out)
|
|
162
|
+
out += part
|
|
163
|
+
ranges.append((start, len(out)))
|
|
164
|
+
return out, ranges
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def normalize(text: str) -> str:
|
|
168
|
+
|
|
169
|
+
text = PAUSE_TAG_RE.sub("", text or "")
|
|
170
|
+
return "".join(
|
|
171
|
+
ch for ch in text
|
|
172
|
+
if not ch.isspace() and not unicodedata.category(ch).startswith(("P", "S"))
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def estimate_durations(texts: list[str], total_ms: float, pause_sec: float) -> list[float]:
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
counts = [max(len(normalize(t)), 1) for t in texts]
|
|
182
|
+
boundaries = len(texts) - 1
|
|
183
|
+
gap_ms = (pause_sec + _NATURAL_PAUSE_SEC) * 1000
|
|
184
|
+
speech_ms = total_ms - boundaries * gap_ms
|
|
185
|
+
if speech_ms <= 0:
|
|
186
|
+
gap_ms, speech_ms = 0.0, total_ms
|
|
187
|
+
per_char = speech_ms / sum(counts)
|
|
188
|
+
durations: list[float] = []
|
|
189
|
+
for k, count in enumerate(counts):
|
|
190
|
+
edges = (k > 0) + (k < boundaries)
|
|
191
|
+
durations.append(count * per_char + edges * gap_ms / 2)
|
|
192
|
+
return durations
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _sub_mid(subs: list[dict], i: int) -> float:
|
|
198
|
+
return (float(subs[i]["timeEnd"]) + float(subs[i + 1]["timeBegin"])) / 2
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def cuts_from_text_offsets(subs: list[dict], ranges: list[tuple[int, int]]) -> Optional[list[float]]:
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
if len(subs) < 2 or not all(
|
|
205
|
+
isinstance(s.get("textBegin"), int) and isinstance(s.get("textEnd"), int) for s in subs
|
|
206
|
+
):
|
|
207
|
+
return None
|
|
208
|
+
cuts: list[float] = []
|
|
209
|
+
for k in range(1, len(ranges)):
|
|
210
|
+
prev_end, next_start = ranges[k - 1][1], ranges[k][0]
|
|
211
|
+
hit = next(
|
|
212
|
+
(
|
|
213
|
+
i for i in range(len(subs) - 1)
|
|
214
|
+
if subs[i]["textEnd"] <= next_start
|
|
215
|
+
and subs[i + 1]["textBegin"] >= prev_end
|
|
216
|
+
and subs[i + 1]["textBegin"] >= subs[i]["textEnd"]
|
|
217
|
+
),
|
|
218
|
+
None,
|
|
219
|
+
)
|
|
220
|
+
if hit is None:
|
|
221
|
+
return None
|
|
222
|
+
cuts.append(_sub_mid(subs, hit))
|
|
223
|
+
return cuts if _monotonic(cuts) else None
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def speakable_ranges(run_text: str, ranges: list[tuple[int, int]]) -> list[tuple[int, int]]:
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
keep = [ch not in "\r\n" for ch in run_text]
|
|
234
|
+
for m in PAUSE_TAG_RE.finditer(run_text):
|
|
235
|
+
for i in range(m.start(), m.end()):
|
|
236
|
+
keep[i] = False
|
|
237
|
+
prefix = [0]
|
|
238
|
+
for k in keep:
|
|
239
|
+
prefix.append(prefix[-1] + k)
|
|
240
|
+
return [(prefix[a], prefix[b]) for a, b in ranges]
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def cuts_from_text_alignment(subs: list[dict], texts: list[str]) -> Optional[list[float]]:
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
if len(subs) < 2:
|
|
247
|
+
return None
|
|
248
|
+
sub_acc: list[int] = []
|
|
249
|
+
acc = 0
|
|
250
|
+
for s in subs:
|
|
251
|
+
acc += len(normalize(s.get("text") or ""))
|
|
252
|
+
sub_acc.append(acc)
|
|
253
|
+
if acc != sum(len(normalize(t)) for t in texts):
|
|
254
|
+
return None
|
|
255
|
+
cuts: list[float] = []
|
|
256
|
+
scene_acc = 0
|
|
257
|
+
for k in range(len(texts) - 1):
|
|
258
|
+
scene_acc += len(normalize(texts[k]))
|
|
259
|
+
try:
|
|
260
|
+
i = sub_acc.index(scene_acc)
|
|
261
|
+
except ValueError:
|
|
262
|
+
return None
|
|
263
|
+
if i >= len(subs) - 1:
|
|
264
|
+
return None
|
|
265
|
+
cuts.append(_sub_mid(subs, i))
|
|
266
|
+
return cuts if _monotonic(cuts) else None
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def cuts_from_silences(
|
|
270
|
+
silences: list[tuple[float, float]],
|
|
271
|
+
texts: list[str],
|
|
272
|
+
total_ms: float,
|
|
273
|
+
pause_sec: float,
|
|
274
|
+
) -> Optional[list[float]]:
|
|
275
|
+
|
|
276
|
+
need = len(texts) - 1
|
|
277
|
+
if need <= 0:
|
|
278
|
+
return []
|
|
279
|
+
|
|
280
|
+
inner = [s for s in silences if 0 < s[0] and s[1] < total_ms]
|
|
281
|
+
if len(inner) < need:
|
|
282
|
+
return None
|
|
283
|
+
longest = sorted(inner, key=lambda s: s[1] - s[0], reverse=True)[:need]
|
|
284
|
+
longest.sort(key=lambda s: s[0])
|
|
285
|
+
min_len = pause_sec * _MIN_BOUNDARY_PAUSE_RATIO * 1000
|
|
286
|
+
if any(end - start < min_len for start, end in longest):
|
|
287
|
+
return None
|
|
288
|
+
cuts = [(start + end) / 2 for start, end in longest]
|
|
289
|
+
if not _durations_plausible(cuts, texts, total_ms, pause_sec):
|
|
290
|
+
return None
|
|
291
|
+
return cuts if _monotonic(cuts) else None
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _monotonic(cuts: list[float]) -> bool:
|
|
295
|
+
return all(b > a for a, b in zip(cuts, cuts[1:])) and all(c > 0 for c in cuts)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _durations_plausible(cuts: list[float], texts: list[str], total_ms: float, pause_sec: float) -> bool:
|
|
299
|
+
|
|
300
|
+
edges = [0.0, *cuts, float(total_ms)]
|
|
301
|
+
for k, predicted in enumerate(estimate_durations(texts, total_ms, pause_sec)):
|
|
302
|
+
actual = edges[k + 1] - edges[k]
|
|
303
|
+
if abs(actual - predicted) > max(ESTIMATE_WINDOW_SEC * 1000, predicted * _ESTIMATE_WINDOW_RATIO):
|
|
304
|
+
return False
|
|
305
|
+
return True
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _cuts_on_boundary_pauses(cuts: list[float], silences: list[tuple[float, float]], pause_sec: float) -> bool:
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
min_len = pause_sec * _MIN_BOUNDARY_PAUSE_RATIO * 1000
|
|
313
|
+
tol = 100.0
|
|
314
|
+
return all(
|
|
315
|
+
any(end - start >= min_len and start - tol <= c <= end + tol for start, end in silences)
|
|
316
|
+
for c in cuts
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def find_cuts(
|
|
321
|
+
texts: list[str],
|
|
322
|
+
ranges: list[tuple[int, int]],
|
|
323
|
+
subs: list[dict],
|
|
324
|
+
silences: Optional[list[tuple[float, float]]],
|
|
325
|
+
total_ms: float,
|
|
326
|
+
pause_sec: float,
|
|
327
|
+
run_text: Optional[str] = None,
|
|
328
|
+
) -> tuple[Optional[list[float]], str]:
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def trusted(cuts: Optional[list[float]]) -> bool:
|
|
334
|
+
return (
|
|
335
|
+
cuts is not None
|
|
336
|
+
and _durations_plausible(cuts, texts, total_ms, pause_sec)
|
|
337
|
+
and (silences is None or _cuts_on_boundary_pauses(cuts, silences, pause_sec))
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
candidates = [speakable_ranges(run_text, ranges)] if run_text is not None else []
|
|
341
|
+
for rs in [*candidates, ranges]:
|
|
342
|
+
cuts = cuts_from_text_offsets(subs, rs)
|
|
343
|
+
if trusted(cuts):
|
|
344
|
+
return cuts, "subtitle-offsets"
|
|
345
|
+
cuts = cuts_from_text_alignment(subs, texts)
|
|
346
|
+
if trusted(cuts):
|
|
347
|
+
return cuts, "subtitle-text"
|
|
348
|
+
if silences is not None:
|
|
349
|
+
cuts = cuts_from_silences(silences, texts, total_ms, pause_sec)
|
|
350
|
+
if cuts is not None:
|
|
351
|
+
return cuts, "silence"
|
|
352
|
+
return None, "none"
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def slice_bounds(cuts: list[float], total_ms: float) -> list[tuple[float, float]]:
|
|
358
|
+
edges = [0.0, *cuts, float(total_ms)]
|
|
359
|
+
return list(zip(edges, edges[1:]))
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def speech_span(bounds: tuple[float, float], silences: list[tuple[float, float]]) -> tuple[int, int]:
|
|
363
|
+
|
|
364
|
+
start, end = bounds
|
|
365
|
+
lo, hi = start, end
|
|
366
|
+
for s_start, s_end in silences:
|
|
367
|
+
if s_start <= lo < s_end:
|
|
368
|
+
lo = min(s_end, end)
|
|
369
|
+
if s_start < hi <= s_end:
|
|
370
|
+
hi = max(s_start, lo)
|
|
371
|
+
return int(round(lo - start)), int(round(hi - start))
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def split_subtitles(subs: list[dict], bounds: list[tuple[float, float]]) -> list[list[dict]]:
|
|
375
|
+
|
|
376
|
+
per_scene: list[list[dict]] = [[] for _ in bounds]
|
|
377
|
+
for s in subs:
|
|
378
|
+
mid = (float(s.get("timeBegin", 0)) + float(s.get("timeEnd", 0))) / 2
|
|
379
|
+
for k, (start, end) in enumerate(bounds):
|
|
380
|
+
if start <= mid < end or (k == len(bounds) - 1 and mid >= start):
|
|
381
|
+
length = end - start
|
|
382
|
+
per_scene[k].append({
|
|
383
|
+
"text": s.get("text", ""),
|
|
384
|
+
"timeBegin": int(round(min(max(float(s["timeBegin"]) - start, 0), length))),
|
|
385
|
+
"timeEnd": int(round(min(max(float(s["timeEnd"]) - start, 0), length))),
|
|
386
|
+
})
|
|
387
|
+
break
|
|
388
|
+
return per_scene
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def silencedetect_command(src: str, ffmpeg: str = "ffmpeg") -> list[str]:
|
|
394
|
+
return [
|
|
395
|
+
ffmpeg, "-hide_banner", "-nostats", "-i", src,
|
|
396
|
+
"-af", f"silencedetect=noise={_SILENCE_NOISE_DB}dB:d={_SILENCE_MIN_SEC}",
|
|
397
|
+
"-f", "null", "-",
|
|
398
|
+
]
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
_SILENCE_START_RE = re.compile(r"silence_start:\s*(-?\d+(?:\.\d+)?)")
|
|
402
|
+
_SILENCE_END_RE = re.compile(r"silence_end:\s*(-?\d+(?:\.\d+)?)")
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def parse_silencedetect(stderr: str, total_ms: float) -> list[tuple[float, float]]:
|
|
406
|
+
|
|
407
|
+
silences: list[tuple[float, float]] = []
|
|
408
|
+
start: Optional[float] = None
|
|
409
|
+
for line in (stderr or "").splitlines():
|
|
410
|
+
m = _SILENCE_START_RE.search(line)
|
|
411
|
+
if m:
|
|
412
|
+
start = max(float(m.group(1)) * 1000, 0.0)
|
|
413
|
+
continue
|
|
414
|
+
m = _SILENCE_END_RE.search(line)
|
|
415
|
+
if m and start is not None:
|
|
416
|
+
silences.append((start, float(m.group(1)) * 1000))
|
|
417
|
+
start = None
|
|
418
|
+
if start is not None:
|
|
419
|
+
silences.append((start, float(total_ms)))
|
|
420
|
+
return silences
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def detect_silences(src: str, total_ms: float, ffmpeg: str, timeout: int = 120) -> list[tuple[float, float]]:
|
|
424
|
+
try:
|
|
425
|
+
result = subprocess.run(
|
|
426
|
+
silencedetect_command(src, ffmpeg),
|
|
427
|
+
capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL,
|
|
428
|
+
)
|
|
429
|
+
except subprocess.TimeoutExpired as exc:
|
|
430
|
+
raise RuntimeError(f"ffmpeg silencedetect timed out after {timeout}s") from exc
|
|
431
|
+
if result.returncode != 0:
|
|
432
|
+
raise RuntimeError(f"ffmpeg silencedetect failed: {(result.stderr or '').strip()[-500:]}")
|
|
433
|
+
return parse_silencedetect(result.stderr, total_ms)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def slice_command(src: str, start_ms: float, end_ms: Optional[float], out_path: str, ffmpeg: str = "ffmpeg") -> list[str]:
|
|
437
|
+
|
|
438
|
+
cmd = [ffmpeg, "-y", "-loglevel", "error", "-i", src, "-ss", f"{start_ms / 1000:.3f}"]
|
|
439
|
+
if end_ms is not None:
|
|
440
|
+
cmd += ["-to", f"{end_ms / 1000:.3f}"]
|
|
441
|
+
cmd += ["-c:a", "libmp3lame", "-b:a", "128k", out_path]
|
|
442
|
+
return cmd
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def slice_audio(
|
|
446
|
+
src: str,
|
|
447
|
+
bounds: list[tuple[float, float]],
|
|
448
|
+
out_dir: str,
|
|
449
|
+
names: list[str],
|
|
450
|
+
ffmpeg: str,
|
|
451
|
+
timeout: int = 120,
|
|
452
|
+
) -> list[str]:
|
|
453
|
+
|
|
454
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
455
|
+
paths: list[str] = []
|
|
456
|
+
for k, ((start, end), name) in enumerate(zip(bounds, names)):
|
|
457
|
+
safe = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in name)
|
|
458
|
+
out_path = os.path.join(out_dir, f"narration-{safe}.mp3")
|
|
459
|
+
|
|
460
|
+
cmd = slice_command(src, start, None if k == len(bounds) - 1 else end, out_path, ffmpeg)
|
|
461
|
+
try:
|
|
462
|
+
result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL)
|
|
463
|
+
except subprocess.TimeoutExpired as exc:
|
|
464
|
+
raise RuntimeError(f"ffmpeg slice timed out after {timeout}s") from exc
|
|
465
|
+
if result.returncode != 0 or not os.path.exists(out_path):
|
|
466
|
+
raise RuntimeError(f"ffmpeg slice failed: {(result.stderr or '').strip()[:500]}")
|
|
467
|
+
paths.append(out_path)
|
|
468
|
+
return paths
|
|
@@ -70,6 +70,8 @@ from video_dsl.runtime.timeline_compiler import (
|
|
|
70
70
|
|
|
71
71
|
_TRAILING_PUNCT_RE = re.compile(r"[。!?;,,、::..…]+$")
|
|
72
72
|
|
|
73
|
+
_PAUSE_TAG_RE = re.compile(r"<#\s*\d+(?:\.\d+)?\s*#>")
|
|
74
|
+
|
|
73
75
|
def _strip_subtitle_trailing_punct(text: str) -> str:
|
|
74
76
|
|
|
75
77
|
return _TRAILING_PUNCT_RE.sub("", text)
|
|
@@ -1001,12 +1003,15 @@ def build_render_plan(dsl: dict, binding: dict) -> dict:
|
|
|
1001
1003
|
|
|
1002
1004
|
if isinstance(scene.get("textLayers"), list) and scene["textLayers"]:
|
|
1003
1005
|
entry["textLayers"] = scene["textLayers"]
|
|
1006
|
+
|
|
1007
|
+
if narration.get("break") is True:
|
|
1008
|
+
entry["narrationBreak"] = True
|
|
1004
1009
|
timeline.append(entry)
|
|
1005
1010
|
current_frame += duration_frames
|
|
1006
1011
|
|
|
1007
1012
|
total_frames = current_frame
|
|
1008
1013
|
|
|
1009
|
-
|
|
1014
|
+
plan = {
|
|
1010
1015
|
"version": "v1alpha1",
|
|
1011
1016
|
"createdAt": now_iso(),
|
|
1012
1017
|
"status": "planning",
|
|
@@ -1047,6 +1052,13 @@ def build_render_plan(dsl: dict, binding: dict) -> dict:
|
|
|
1047
1052
|
],
|
|
1048
1053
|
}
|
|
1049
1054
|
|
|
1055
|
+
narration_override = {
|
|
1056
|
+
k: global_narration[k] for k in ("mode", "scenePauseSec") if k in global_narration
|
|
1057
|
+
}
|
|
1058
|
+
if narration_override:
|
|
1059
|
+
plan["narration"] = narration_override
|
|
1060
|
+
return plan
|
|
1061
|
+
|
|
1050
1062
|
|
|
1051
1063
|
def resolve_asset_image(asset: dict, private_token: str, timeout: int) -> dict:
|
|
1052
1064
|
|
|
@@ -1099,24 +1111,32 @@ def resolve_asset_audio(asset: dict, private_token: str, timeout: int) -> dict:
|
|
|
1099
1111
|
|
|
1100
1112
|
|
|
1101
1113
|
payload = asset.get("payload", {})
|
|
1114
|
+
tts_text = _tts_text_from_payload(payload)
|
|
1115
|
+
if not tts_text:
|
|
1116
|
+
return {"status": "failed", "error": "No text provided for TTS"}
|
|
1117
|
+
return _call_gen_voice(tts_text, payload, private_token, timeout)
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
def _tts_text_from_payload(payload: dict) -> str:
|
|
1102
1121
|
|
|
1103
|
-
try:
|
|
1104
|
-
cmd = build_skill_command("gen-voice")
|
|
1105
|
-
except RuntimeError as e:
|
|
1106
|
-
return {"status": "failed", "error": str(e)}
|
|
1107
1122
|
|
|
1108
1123
|
narration_items = payload.get("narrationItems")
|
|
1109
1124
|
if isinstance(narration_items, list) and narration_items:
|
|
1110
1125
|
cleaned = [str(s).strip() for s in narration_items if str(s).strip()]
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1126
|
+
return "\n".join(cleaned)
|
|
1127
|
+
text = payload.get("text", "")
|
|
1128
|
+
if not text:
|
|
1129
|
+
return ""
|
|
1130
|
+
segments = segment_narration(text)
|
|
1131
|
+
return "\n".join(s.strip() for s in segments) if segments else text
|
|
1132
|
+
|
|
1133
|
+
|
|
1134
|
+
def _call_gen_voice(tts_text: str, payload: dict, private_token: str, timeout: int) -> dict:
|
|
1135
|
+
|
|
1136
|
+
try:
|
|
1137
|
+
cmd = build_skill_command("gen-voice")
|
|
1138
|
+
except RuntimeError as e:
|
|
1139
|
+
return {"status": "failed", "error": str(e)}
|
|
1120
1140
|
|
|
1121
1141
|
cmd.extend(["--text", tts_text, "--json-output"])
|
|
1122
1142
|
if payload.get("voiceId"):
|
|
@@ -1196,6 +1216,11 @@ def resolve_asset_video(asset: dict, private_token: str, timeout: int) -> dict:
|
|
|
1196
1216
|
|
|
1197
1217
|
def resolve_asset_digital_human(asset: dict, private_token: str, timeout: int) -> dict:
|
|
1198
1218
|
|
|
1219
|
+
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
|
|
1199
1224
|
payload = asset.get("payload", {})
|
|
1200
1225
|
|
|
1201
1226
|
try:
|
|
@@ -1203,6 +1228,9 @@ def resolve_asset_digital_human(asset: dict, private_token: str, timeout: int) -
|
|
|
1203
1228
|
except RuntimeError as e:
|
|
1204
1229
|
return {"status": "failed", "error": str(e)}
|
|
1205
1230
|
|
|
1231
|
+
if asset.get("_dhGenerationId"):
|
|
1232
|
+
return _wait_digital_human_generation(cmd, int(asset["_dhGenerationId"]), private_token, max(timeout, DH_POLL_TIMEOUT_SEC))
|
|
1233
|
+
|
|
1206
1234
|
if payload.get("avatarId"):
|
|
1207
1235
|
cmd.extend(["--avatar-id", str(payload["avatarId"])])
|
|
1208
1236
|
if payload.get("text"):
|
|
@@ -1218,7 +1246,8 @@ def resolve_asset_digital_human(asset: dict, private_token: str, timeout: int) -
|
|
|
1218
1246
|
if private_token:
|
|
1219
1247
|
cmd.extend(["--priv-token", private_token])
|
|
1220
1248
|
|
|
1221
|
-
|
|
1249
|
+
|
|
1250
|
+
dh_timeout = max(timeout, DH_POLL_TIMEOUT_SEC + 120)
|
|
1222
1251
|
try:
|
|
1223
1252
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=dh_timeout, stdin=subprocess.DEVNULL)
|
|
1224
1253
|
absorb_child_billing(result.stdout)
|
|
@@ -1226,23 +1255,64 @@ def resolve_asset_digital_human(asset: dict, private_token: str, timeout: int) -
|
|
|
1226
1255
|
for line in reversed(result.stdout.strip().split("\n")):
|
|
1227
1256
|
line = line.strip()
|
|
1228
1257
|
if line.startswith("http"):
|
|
1229
|
-
|
|
1258
|
+
return _digital_human_result(line)
|
|
1259
|
+
stderr = result.stderr.strip()
|
|
1260
|
+
resume = _DH_RESUME_RE.search(stderr)
|
|
1261
|
+
if resume:
|
|
1230
1262
|
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
meta = probe_video_meta(line)
|
|
1234
|
-
if meta.get("width") and meta.get("height"):
|
|
1235
|
-
out["width"], out["height"] = meta["width"], meta["height"]
|
|
1236
|
-
if meta.get("duration_ms"):
|
|
1237
|
-
out["duration_ms"] = meta["duration_ms"]
|
|
1238
|
-
return out
|
|
1239
|
-
return {"status": "failed", "error": result.stderr.strip()[:1000]}
|
|
1263
|
+
asset["_dhGenerationId"] = int(resume.group(1))
|
|
1264
|
+
return {"status": "failed", "error": stderr[:1000]}
|
|
1240
1265
|
except subprocess.TimeoutExpired:
|
|
1241
1266
|
return {"status": "failed", "error": f"Digital human generation timed out after {dh_timeout}s"}
|
|
1242
1267
|
except Exception as e:
|
|
1243
1268
|
return {"status": "failed", "error": str(e)[:1000]}
|
|
1244
1269
|
|
|
1245
1270
|
|
|
1271
|
+
DH_POLL_TIMEOUT_SEC = 30 * 60
|
|
1272
|
+
DH_STATUS_POLL_INTERVAL_SEC = 10
|
|
1273
|
+
|
|
1274
|
+
_DH_RESUME_RE = re.compile(r"--check-status --generation-id (\d+)")
|
|
1275
|
+
|
|
1276
|
+
|
|
1277
|
+
def _digital_human_result(url: str) -> dict:
|
|
1278
|
+
out = {"status": "generated", "url": url}
|
|
1279
|
+
|
|
1280
|
+
|
|
1281
|
+
from _video_probe import probe_video_meta
|
|
1282
|
+
meta = probe_video_meta(url)
|
|
1283
|
+
if meta.get("width") and meta.get("height"):
|
|
1284
|
+
out["width"], out["height"] = meta["width"], meta["height"]
|
|
1285
|
+
if meta.get("duration_ms"):
|
|
1286
|
+
out["duration_ms"] = meta["duration_ms"]
|
|
1287
|
+
return out
|
|
1288
|
+
|
|
1289
|
+
|
|
1290
|
+
def _wait_digital_human_generation(base_cmd: list, generation_id: int, private_token: str, timeout: int) -> dict:
|
|
1291
|
+
|
|
1292
|
+
cmd = [*base_cmd, "--check-status", "--generation-id", str(generation_id), "--json-output"]
|
|
1293
|
+
if private_token:
|
|
1294
|
+
cmd.extend(["--priv-token", private_token])
|
|
1295
|
+
deadline = time.time() + timeout
|
|
1296
|
+
while time.time() < deadline:
|
|
1297
|
+
try:
|
|
1298
|
+
result = subprocess.run(cmd, capture_output=True, text=True, timeout=60, stdin=subprocess.DEVNULL)
|
|
1299
|
+
line = (result.stdout.strip().split("\n") or [""])[-1]
|
|
1300
|
+
data = json.loads(line) if line.startswith("{") else {}
|
|
1301
|
+
except (subprocess.TimeoutExpired, json.JSONDecodeError):
|
|
1302
|
+
data = {}
|
|
1303
|
+
status = data.get("status")
|
|
1304
|
+
if status == "completed" and data.get("url"):
|
|
1305
|
+
return _digital_human_result(data["url"])
|
|
1306
|
+
if status == "failed":
|
|
1307
|
+
return {"status": "failed", "error": f"digital-human generation {generation_id} failed"}
|
|
1308
|
+
time.sleep(DH_STATUS_POLL_INTERVAL_SEC)
|
|
1309
|
+
return {
|
|
1310
|
+
"status": "failed",
|
|
1311
|
+
"error": f"digital-human generation {generation_id} still not done after {timeout}s; "
|
|
1312
|
+
f"check later with gen-digital-human --check-status --generation-id {generation_id}",
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
|
|
1246
1316
|
ASSET_RESOLVERS = {
|
|
1247
1317
|
"gen-image": resolve_asset_image,
|
|
1248
1318
|
"gen-voice": resolve_asset_audio,
|
|
@@ -1391,6 +1461,178 @@ def apply_stub_urls(render_plan: dict, stub_image_url: str = "", stub_video_url:
|
|
|
1391
1461
|
return stubbed
|
|
1392
1462
|
|
|
1393
1463
|
|
|
1464
|
+
def plan_narration_runs(render_plan: dict) -> tuple[list[list[dict]], dict]:
|
|
1465
|
+
|
|
1466
|
+
|
|
1467
|
+
|
|
1468
|
+
|
|
1469
|
+
import _continuous_narration as cn
|
|
1470
|
+
|
|
1471
|
+
cfg = cn.resolve_config(_timing_template(render_plan.get("templateId", "")), render_plan.get("narration"))
|
|
1472
|
+
if cfg["mode"] != cn.MODE_CONTINUOUS:
|
|
1473
|
+
return [], cfg
|
|
1474
|
+
assets = {a.get("assetId"): a for a in render_plan.get("resolvedAssets", [])}
|
|
1475
|
+
payloads = {
|
|
1476
|
+
a.get("assetId"): a.get("payload") or {}
|
|
1477
|
+
for a in render_plan.get("_dsl_assets", [])
|
|
1478
|
+
if a.get("assetId")
|
|
1479
|
+
}
|
|
1480
|
+
timeline = render_plan.get("timeline", [])
|
|
1481
|
+
refs: dict[str, int] = {}
|
|
1482
|
+
for entry in timeline:
|
|
1483
|
+
nar_id = (entry.get("props") or {}).get("narrationAssetId")
|
|
1484
|
+
if nar_id:
|
|
1485
|
+
refs[nar_id] = refs.get(nar_id, 0) + 1
|
|
1486
|
+
|
|
1487
|
+
candidates: list[Optional[dict]] = []
|
|
1488
|
+
for entry in timeline:
|
|
1489
|
+
nar_id = (entry.get("props") or {}).get("narrationAssetId")
|
|
1490
|
+
asset = assets.get(nar_id)
|
|
1491
|
+
payload = payloads.get(nar_id) or {}
|
|
1492
|
+
text = _tts_text_from_payload(payload) if asset else ""
|
|
1493
|
+
if (
|
|
1494
|
+
not asset
|
|
1495
|
+
or asset.get("source") != "gen-voice"
|
|
1496
|
+
or asset.get("status") != "pending"
|
|
1497
|
+
or refs.get(nar_id) != 1
|
|
1498
|
+
or not text
|
|
1499
|
+
):
|
|
1500
|
+
candidates.append(None)
|
|
1501
|
+
continue
|
|
1502
|
+
candidates.append({
|
|
1503
|
+
"assetId": nar_id,
|
|
1504
|
+
"sceneId": entry.get("sceneId"),
|
|
1505
|
+
"text": text,
|
|
1506
|
+
"voiceKey": (_narration_voice(payload), _narration_speed(payload), _narration_language_boost(payload)),
|
|
1507
|
+
"break": bool(entry.get("narrationBreak")),
|
|
1508
|
+
})
|
|
1509
|
+
return cn.group_runs(candidates), cfg
|
|
1510
|
+
|
|
1511
|
+
|
|
1512
|
+
def _download_to(url: str, path: str, timeout: int) -> str:
|
|
1513
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
1514
|
+
with urllib.request.urlopen(url, timeout=timeout) as resp, open(path, "wb") as f:
|
|
1515
|
+
while True:
|
|
1516
|
+
chunk = resp.read(1 << 16)
|
|
1517
|
+
if not chunk:
|
|
1518
|
+
break
|
|
1519
|
+
f.write(chunk)
|
|
1520
|
+
return path
|
|
1521
|
+
|
|
1522
|
+
|
|
1523
|
+
def _upload_tts_clip(path: str, title: str) -> str:
|
|
1524
|
+
|
|
1525
|
+
|
|
1526
|
+
|
|
1527
|
+
|
|
1528
|
+
|
|
1529
|
+
|
|
1530
|
+
from upload_video import upload_media_file
|
|
1531
|
+
data = upload_media_file(path, title=title, role="tts_clip", retention="intermediate")
|
|
1532
|
+
url = data.get("signedUrl") or data["fileUrl"]
|
|
1533
|
+
if not data.get("signedUrl"):
|
|
1534
|
+
req = urllib.request.Request(url, headers={"Range": "bytes=0-0"})
|
|
1535
|
+
with urllib.request.urlopen(req, timeout=15):
|
|
1536
|
+
pass
|
|
1537
|
+
return url
|
|
1538
|
+
|
|
1539
|
+
|
|
1540
|
+
def resolve_narration_run(
|
|
1541
|
+
run: list[dict],
|
|
1542
|
+
cfg: dict,
|
|
1543
|
+
render_plan: dict,
|
|
1544
|
+
private_token: str,
|
|
1545
|
+
max_retries: int,
|
|
1546
|
+
timeout: int,
|
|
1547
|
+
work_dir: Optional[str] = None,
|
|
1548
|
+
) -> bool:
|
|
1549
|
+
|
|
1550
|
+
|
|
1551
|
+
import _continuous_narration as cn
|
|
1552
|
+
from _digital_human_segments import find_ffmpeg
|
|
1553
|
+
|
|
1554
|
+
scene_ids = [c.get("sceneId") for c in run]
|
|
1555
|
+
|
|
1556
|
+
def log(message: str, level: str = "info") -> None:
|
|
1557
|
+
render_plan.setdefault("logs", []).append({
|
|
1558
|
+
"phase": "narration-run",
|
|
1559
|
+
"level": level,
|
|
1560
|
+
"message": message,
|
|
1561
|
+
"timestamp": now_iso(),
|
|
1562
|
+
})
|
|
1563
|
+
LogPrint(f" 🎙️ {message}", file=sys.stderr)
|
|
1564
|
+
|
|
1565
|
+
ffmpeg = find_ffmpeg()
|
|
1566
|
+
if not ffmpeg:
|
|
1567
|
+
log(f"ffmpeg not found; scenes {scene_ids} fall back to per-scene TTS", level="warn")
|
|
1568
|
+
return False
|
|
1569
|
+
|
|
1570
|
+
pause = cfg["scenePauseSec"]
|
|
1571
|
+
texts = [c["text"] for c in run]
|
|
1572
|
+
run_text, ranges = cn.compose_run_text(texts, pause)
|
|
1573
|
+
payloads = {
|
|
1574
|
+
a.get("assetId"): a.get("payload") or {}
|
|
1575
|
+
for a in render_plan.get("_dsl_assets", [])
|
|
1576
|
+
if a.get("assetId")
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
voice_payload = payloads.get(run[0]["assetId"]) or {}
|
|
1580
|
+
|
|
1581
|
+
result: dict = {}
|
|
1582
|
+
for attempt in range(max_retries):
|
|
1583
|
+
LogPrint(f" 🔄 Generating continuous narration for {len(run)} scenes (attempt {attempt + 1}/{max_retries})...", file=sys.stderr)
|
|
1584
|
+
result = _call_gen_voice(run_text, voice_payload, private_token, timeout)
|
|
1585
|
+
if result.get("status") == "generated":
|
|
1586
|
+
break
|
|
1587
|
+
if attempt < max_retries - 1:
|
|
1588
|
+
time.sleep(2)
|
|
1589
|
+
if result.get("status") != "generated":
|
|
1590
|
+
log(f"continuous TTS failed for scenes {scene_ids}: {str(result.get('error'))[:300]}; falling back to per-scene TTS", level="warn")
|
|
1591
|
+
return False
|
|
1592
|
+
|
|
1593
|
+
try:
|
|
1594
|
+
total_ms = float(result.get("audio_length_ms") or 0)
|
|
1595
|
+
if total_ms <= 0:
|
|
1596
|
+
raise RuntimeError("TTS returned no audio length")
|
|
1597
|
+
out_dir = work_dir or os.path.join(OUTPUT_DIR, "narration")
|
|
1598
|
+
run_name = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in run[0]["assetId"])
|
|
1599
|
+
local = _download_to(result["url"], os.path.join(out_dir, f"run-{run_name}.mp3"), timeout)
|
|
1600
|
+
subs = result.get("subtitles") or []
|
|
1601
|
+
silences = cn.detect_silences(local, total_ms, ffmpeg)
|
|
1602
|
+
cuts, method = cn.find_cuts(texts, ranges, subs, silences, total_ms, pause, run_text)
|
|
1603
|
+
if cuts is None:
|
|
1604
|
+
log(f"no reliable cut points for scenes {scene_ids}; falling back to per-scene TTS", level="warn")
|
|
1605
|
+
return False
|
|
1606
|
+
bounds = cn.slice_bounds(cuts, total_ms)
|
|
1607
|
+
paths = cn.slice_audio(local, bounds, out_dir, [c["assetId"] for c in run], ffmpeg)
|
|
1608
|
+
urls = [_upload_tts_clip(path, f"narration {c['assetId']}") for path, c in zip(paths, run)]
|
|
1609
|
+
except Exception as exc:
|
|
1610
|
+
log(f"continuous narration split failed for scenes {scene_ids}: {exc}; falling back to per-scene TTS", level="warn")
|
|
1611
|
+
return False
|
|
1612
|
+
|
|
1613
|
+
per_scene_subs = cn.split_subtitles(subs, bounds)
|
|
1614
|
+
assets = {a.get("assetId"): a for a in render_plan.get("resolvedAssets", [])}
|
|
1615
|
+
run_id = run[0]["assetId"]
|
|
1616
|
+
for k, c in enumerate(run):
|
|
1617
|
+
asset = assets[c["assetId"]]
|
|
1618
|
+
start, end = bounds[k]
|
|
1619
|
+
asset["status"] = "generated"
|
|
1620
|
+
asset["url"] = urls[k]
|
|
1621
|
+
asset["duration"] = int(round(end - start))
|
|
1622
|
+
asset["generatedBy"] = {"skill": "gen-voice", "mode": "continuous", "startedAt": now_iso(), "completedAt": now_iso()}
|
|
1623
|
+
|
|
1624
|
+
asset["narrationRun"] = {"id": run_id, "index": k, "count": len(run), "method": method}
|
|
1625
|
+
asset["speechSpanMs"] = list(cn.speech_span(bounds[k], silences))
|
|
1626
|
+
if per_scene_subs[k]:
|
|
1627
|
+
asset["ttsSubtitles"] = per_scene_subs[k]
|
|
1628
|
+
payload = payloads.get(c["assetId"]) or {}
|
|
1629
|
+
if isinstance(payload.get("narrationItems"), list):
|
|
1630
|
+
asset["narrationLineCount"] = len(payload["narrationItems"])
|
|
1631
|
+
asset["narrationIntroLines"] = int(payload.get("narrationIntroLines") or 0)
|
|
1632
|
+
log(f"continuous narration: {len(run)} scenes in one TTS call, {total_ms / 1000:.1f}s, cut by {method}")
|
|
1633
|
+
return True
|
|
1634
|
+
|
|
1635
|
+
|
|
1394
1636
|
def resolve_assets(
|
|
1395
1637
|
render_plan: dict,
|
|
1396
1638
|
private_token: str,
|
|
@@ -1430,21 +1672,56 @@ def resolve_assets(
|
|
|
1430
1672
|
failed = 0
|
|
1431
1673
|
|
|
1432
1674
|
|
|
1675
|
+
runs: list[list[dict]] = []
|
|
1676
|
+
narration_cfg: dict = {}
|
|
1677
|
+
if phase1:
|
|
1678
|
+
runs, narration_cfg = plan_narration_runs(render_plan)
|
|
1679
|
+
run_asset_ids = {c["assetId"] for run in runs for c in run}
|
|
1680
|
+
singles = [a for a in phase1 if a.get("assetId") not in run_asset_ids]
|
|
1681
|
+
assets_by_id = {a.get("assetId"): a for a in phase1}
|
|
1682
|
+
|
|
1683
|
+
def resolve_run(run: list[dict]) -> tuple[int, int]:
|
|
1684
|
+
if resolve_narration_run(run, narration_cfg, render_plan, private_token, max_retries, timeout):
|
|
1685
|
+
return len(run), 0
|
|
1686
|
+
ok_count = fail_count = 0
|
|
1687
|
+
for c in run:
|
|
1688
|
+
asset = assets_by_id[c["assetId"]]
|
|
1689
|
+
if _resolve_single_asset(asset, render_plan, private_token, max_retries, timeout):
|
|
1690
|
+
ok_count += 1
|
|
1691
|
+
elif asset["status"] == "failed":
|
|
1692
|
+
fail_count += 1
|
|
1693
|
+
return ok_count, fail_count
|
|
1694
|
+
|
|
1695
|
+
|
|
1433
1696
|
if phase1:
|
|
1434
1697
|
default_parallelism = 8
|
|
1435
1698
|
max_workers = min(
|
|
1436
1699
|
int(os.environ.get("REMOTION_ASSET_PARALLELISM", str(default_parallelism))),
|
|
1437
|
-
len(
|
|
1700
|
+
len(singles) + len(runs),
|
|
1438
1701
|
)
|
|
1439
1702
|
LogPrint(f" ⚡ Phase 1: generating {len(phase1)} asset(s) in parallel (concurrency={max_workers})", file=sys.stderr)
|
|
1703
|
+
if runs:
|
|
1704
|
+
LogPrint(f" 🎙️ {len(runs)} continuous narration run(s) covering {len(run_asset_ids)} scene(s)", file=sys.stderr)
|
|
1440
1705
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
1441
1706
|
future_to_asset = {
|
|
1442
1707
|
executor.submit(
|
|
1443
1708
|
_resolve_single_asset, asset, render_plan, private_token, max_retries, timeout
|
|
1444
1709
|
): asset
|
|
1445
|
-
for asset in
|
|
1710
|
+
for asset in singles
|
|
1446
1711
|
}
|
|
1447
|
-
for
|
|
1712
|
+
future_to_run = {executor.submit(resolve_run, run): run for run in runs}
|
|
1713
|
+
for future in as_completed([*future_to_asset, *future_to_run]):
|
|
1714
|
+
if future in future_to_run:
|
|
1715
|
+
try:
|
|
1716
|
+
ok_count, fail_count = future.result()
|
|
1717
|
+
generated += ok_count
|
|
1718
|
+
failed += fail_count
|
|
1719
|
+
except Exception as exc:
|
|
1720
|
+
for c in future_to_run[future]:
|
|
1721
|
+
assets_by_id[c["assetId"]]["status"] = "failed"
|
|
1722
|
+
failed += 1
|
|
1723
|
+
LogPrint(f" ❌ continuous narration run exception: {exc}", file=sys.stderr)
|
|
1724
|
+
continue
|
|
1448
1725
|
asset = future_to_asset[future]
|
|
1449
1726
|
try:
|
|
1450
1727
|
ok = future.result()
|
|
@@ -1751,10 +2028,17 @@ def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
|
1751
2028
|
|
|
1752
2029
|
|
|
1753
2030
|
tail_pad_override = entry.get("tailPadFrames")
|
|
2031
|
+
|
|
2032
|
+
|
|
2033
|
+
run = narration_asset.get("narrationRun") or {}
|
|
2034
|
+
scene_default_tail = (
|
|
2035
|
+
0 if run and int(run.get("index", 0)) < int(run.get("count", 0)) - 1
|
|
2036
|
+
else default_tail_pad
|
|
2037
|
+
)
|
|
1754
2038
|
tail_pad = (
|
|
1755
2039
|
int(tail_pad_override)
|
|
1756
2040
|
if isinstance(tail_pad_override, int) and tail_pad_override >= 0
|
|
1757
|
-
else
|
|
2041
|
+
else scene_default_tail
|
|
1758
2042
|
)
|
|
1759
2043
|
min_dur = int(fps * 2)
|
|
1760
2044
|
|
|
@@ -1778,12 +2062,13 @@ def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
|
1778
2062
|
)
|
|
1779
2063
|
entry["subtitleSegments"] = [
|
|
1780
2064
|
{
|
|
1781
|
-
"text": _strip_subtitle_trailing_punct(s["text"].strip()),
|
|
2065
|
+
"text": _strip_subtitle_trailing_punct(_PAUSE_TAG_RE.sub("", s["text"]).strip()),
|
|
1782
2066
|
"startFrame": current_frame + int(s["timeBegin"] / 1000.0 * fps),
|
|
1783
2067
|
"endFrame": current_frame + int(s["timeEnd"] / 1000.0 * fps),
|
|
1784
2068
|
"sentenceIndex": sentence_indices[i],
|
|
1785
2069
|
}
|
|
1786
2070
|
for i, s in enumerate(tts_subs)
|
|
2071
|
+
if _PAUSE_TAG_RE.sub("", s.get("text") or "").strip()
|
|
1787
2072
|
]
|
|
1788
2073
|
_warn_step_sentence_mismatch(entry, sentence_indices, step_field)
|
|
1789
2074
|
else:
|
|
@@ -1837,13 +2122,18 @@ def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
|
1837
2122
|
entry["subtitleSegments"] = new_subs
|
|
1838
2123
|
elif old_subs:
|
|
1839
2124
|
sub_span = audio_frames
|
|
2125
|
+
cursor = 0
|
|
2126
|
+
|
|
2127
|
+
span = narration_asset.get("speechSpanMs")
|
|
2128
|
+
if isinstance(span, list) and len(span) == 2 and 0 <= span[0] < span[1]:
|
|
2129
|
+
cursor = int(span[0] / 1000.0 * fps)
|
|
2130
|
+
sub_span = max(math.ceil(span[1] / 1000.0 * fps) - cursor, fps // 2)
|
|
1840
2131
|
|
|
1841
2132
|
|
|
1842
2133
|
def _plain(t: str) -> int:
|
|
1843
2134
|
return len(re.sub(r"\*+", "", t or ""))
|
|
1844
2135
|
total_chars = max(sum(_plain(s.get("text", "")) for s in old_subs), 1)
|
|
1845
2136
|
new_subs = []
|
|
1846
|
-
cursor = 0
|
|
1847
2137
|
for s in old_subs:
|
|
1848
2138
|
text = s.get("text", "")
|
|
1849
2139
|
ratio = _plain(text) / total_chars
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "render-video",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "473",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V40",
|
|
6
6
|
"skillDescription": "Final-render skill: loads a persisted RenderPlan by `job_id` and drives the Remotion engine to produce the final video.\n\nUse this skill as soon as the user mentions any of these intents (after assets are already prepared):\n- Render the video, composite the video, export the video\n- Turn the prepared assets into the final clip\n- Render with Remotion\n\nPrerequisite: assets must already be generated via `prepare_video_assets`. This skill never resolves or regenerates assets — pass it a `job_id` from a previous `prepare_video_assets` call.\n\n⚠️ Stop-and-confirm gate: never call this skill until the user has explicitly confirmed the assets prepared by `prepare_video_assets`. If those assets were prepared in the current turn and the user has not replied since, stop and ask instead of rendering."
|
|
7
7
|
}
|
|
@@ -199,6 +199,7 @@ Rules:
|
|
|
199
199
|
- The legacy flat `narration.text` form still works but the author then has to keep `highlightMap` in sync with the subtitle segmentation (error-prone, not recommended).
|
|
200
200
|
- `narration.languageBoost` (optional, also accepted at `global.narration`) pins the TTS language hint (`auto` / `Chinese` / `English` / …). Leave it unset unless auto-detection gets it wrong — the service applies its own default, and a scene-level value overrides the global one, exactly like `speed`.
|
|
201
201
|
- Voice: to change the narration voice, set `global.narration.voiceId` (whole video) or `narration.voiceId` on a scene (that scene only). Precedence when synthesizing is scene > global > the audio asset's `payload.voiceId`, which is only a fallback — editing the asset payload alone is not how you change a voice.
|
|
202
|
+
- Continuous narration: templates that declare `capabilities.narration.mode: "continuous"` get adjacent scenes with the same voice / speed / languageBoost synthesized in one TTS call and cut back per scene, so pace and loudness stay even. Nothing to author for it. Set `narration.break: true` on a scene to start a new run there (a chapter change that wants a longer pause); `global.narration.mode` (`continuous` / `perScene`) overrides the template only when the user asks for it.
|
|
202
203
|
|
|
203
204
|
## Image sources
|
|
204
205
|
|
|
@@ -109,7 +109,9 @@
|
|
|
109
109
|
"voiceId": { "type": "string", "description": "Minimax 音色 ID" },
|
|
110
110
|
"voiceName": { "type": "string" },
|
|
111
111
|
"speed": { "type": "number", "default": 1.0 },
|
|
112
|
-
"languageBoost": { "type": "string", "description": "TTS 语种提示(auto / Chinese / English …)。中英混排的旁白不给提示时,英文词容易被按中文音系读出来;留空由服务端取默认值。" }
|
|
112
|
+
"languageBoost": { "type": "string", "description": "TTS 语种提示(auto / Chinese / English …)。中英混排的旁白不给提示时,英文词容易被按中文音系读出来;留空由服务端取默认值。" },
|
|
113
|
+
"mode": { "type": "string", "enum": ["continuous", "perScene"], "description": "旁白合成方式,覆盖模板 capabilities.narration.mode。continuous = 相邻、音色语速语种相同的场景整段合成一次再按场景切开(语速响度一致);perScene = 每个场景单独合成。留空跟随模板,模板没声明即 perScene。" },
|
|
114
|
+
"scenePauseSec": { "type": "number", "minimum": 0.01, "maximum": 99.99, "description": "continuous 模式下场景之间插入的停顿秒数(MiniMax <#x#>),覆盖模板声明;默认 0.8。" }
|
|
113
115
|
}
|
|
114
116
|
},
|
|
115
117
|
"bgm": {
|
|
@@ -268,6 +270,7 @@
|
|
|
268
270
|
"voiceId": { "type": "string", "description": "该场旁白音色,覆盖 global.narration.voiceId;留空跟随全局。合成时优先级:场景 > 全局 > 音频资产 payload.voiceId(仅作兜底)。" },
|
|
269
271
|
"voiceName": { "type": "string", "description": "voiceId 的展示名(仅展示用,不参与合成)" },
|
|
270
272
|
"speed": { "type": "number", "description": "该场旁白语速,覆盖 global.narration.speed;留空跟随全局。" },
|
|
273
|
+
"break": { "type": "boolean", "description": "continuous 旁白模式下从这一幕起另起一段整段合成(例如换章节、需要更长停顿)。perScene 模式下无效。" },
|
|
271
274
|
"languageBoost": { "type": "string", "description": "TTS 语种提示(auto / Chinese / English …)。中英混排的旁白不给提示时,英文词容易被按中文音系读出来;覆盖 global.narration 的同名字段;留空则跟随全局/服务端默认。" }
|
|
272
275
|
}
|
|
273
276
|
},
|