@remixmate/cli 0.13.4 → 0.13.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handlers/gen-voice.js +8 -8
- package/dist/manifest.json +2 -2
- package/package.json +1 -1
- package/skills/gen-script/SKILL.md +2 -0
- package/skills/gen-script/scripts/gen_script.py +114 -4
- package/skills/gen-script/version.json +1 -1
- package/skills/prepare-video-assets/SKILL.md +2 -0
- package/skills/prepare-video-assets/version.json +1 -1
- package/skills/render-video/scripts/_continuous_narration.py +476 -0
- package/skills/render-video/scripts/_narration_alignment.py +88 -0
- package/skills/render-video/scripts/_scene_assets.py +39 -0
- package/skills/render-video/scripts/render_video.py +324 -28
- package/skills/render-video/version.json +1 -1
- package/skills/template-registry/SKILL.md +1 -0
- package/skills/template-registry/video_dsl/schema/video-dsl-v1alpha1.json +4 -1
|
@@ -27,6 +27,7 @@ import argparse
|
|
|
27
27
|
import atexit
|
|
28
28
|
import builtins
|
|
29
29
|
import json
|
|
30
|
+
import math
|
|
30
31
|
import os
|
|
31
32
|
import re
|
|
32
33
|
import signal
|
|
@@ -70,6 +71,8 @@ from video_dsl.runtime.timeline_compiler import (
|
|
|
70
71
|
|
|
71
72
|
_TRAILING_PUNCT_RE = re.compile(r"[。!?;,,、::..…]+$")
|
|
72
73
|
|
|
74
|
+
_PAUSE_TAG_RE = re.compile(r"<#\s*\d+(?:\.\d+)?\s*#>")
|
|
75
|
+
|
|
73
76
|
def _strip_subtitle_trailing_punct(text: str) -> str:
|
|
74
77
|
|
|
75
78
|
return _TRAILING_PUNCT_RE.sub("", text)
|
|
@@ -919,7 +922,7 @@ def build_render_plan(dsl: dict, binding: dict) -> dict:
|
|
|
919
922
|
for idx, scene in enumerate(dsl.get("scenes", [])):
|
|
920
923
|
scene_id = scene.get("id") or scene.get("sceneId") or f"scene-{idx:03d}"
|
|
921
924
|
duration = scene.get("duration", 5)
|
|
922
|
-
duration_frames =
|
|
925
|
+
duration_frames = math.floor(duration * fps + 0.5)
|
|
923
926
|
|
|
924
927
|
scene_binding = binding_map.get(scene_id, {})
|
|
925
928
|
|
|
@@ -1001,12 +1004,15 @@ def build_render_plan(dsl: dict, binding: dict) -> dict:
|
|
|
1001
1004
|
|
|
1002
1005
|
if isinstance(scene.get("textLayers"), list) and scene["textLayers"]:
|
|
1003
1006
|
entry["textLayers"] = scene["textLayers"]
|
|
1007
|
+
|
|
1008
|
+
if narration.get("break") is True:
|
|
1009
|
+
entry["narrationBreak"] = True
|
|
1004
1010
|
timeline.append(entry)
|
|
1005
1011
|
current_frame += duration_frames
|
|
1006
1012
|
|
|
1007
1013
|
total_frames = current_frame
|
|
1008
1014
|
|
|
1009
|
-
|
|
1015
|
+
plan = {
|
|
1010
1016
|
"version": "v1alpha1",
|
|
1011
1017
|
"createdAt": now_iso(),
|
|
1012
1018
|
"status": "planning",
|
|
@@ -1047,6 +1053,13 @@ def build_render_plan(dsl: dict, binding: dict) -> dict:
|
|
|
1047
1053
|
],
|
|
1048
1054
|
}
|
|
1049
1055
|
|
|
1056
|
+
narration_override = {
|
|
1057
|
+
k: global_narration[k] for k in ("mode", "scenePauseSec") if k in global_narration
|
|
1058
|
+
}
|
|
1059
|
+
if narration_override:
|
|
1060
|
+
plan["narration"] = narration_override
|
|
1061
|
+
return plan
|
|
1062
|
+
|
|
1050
1063
|
|
|
1051
1064
|
def resolve_asset_image(asset: dict, private_token: str, timeout: int) -> dict:
|
|
1052
1065
|
|
|
@@ -1099,24 +1112,32 @@ def resolve_asset_audio(asset: dict, private_token: str, timeout: int) -> dict:
|
|
|
1099
1112
|
|
|
1100
1113
|
|
|
1101
1114
|
payload = asset.get("payload", {})
|
|
1115
|
+
tts_text = _tts_text_from_payload(payload)
|
|
1116
|
+
if not tts_text:
|
|
1117
|
+
return {"status": "failed", "error": "No text provided for TTS"}
|
|
1118
|
+
return _call_gen_voice(tts_text, payload, private_token, timeout)
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def _tts_text_from_payload(payload: dict) -> str:
|
|
1102
1122
|
|
|
1103
|
-
try:
|
|
1104
|
-
cmd = build_skill_command("gen-voice")
|
|
1105
|
-
except RuntimeError as e:
|
|
1106
|
-
return {"status": "failed", "error": str(e)}
|
|
1107
1123
|
|
|
1108
1124
|
narration_items = payload.get("narrationItems")
|
|
1109
1125
|
if isinstance(narration_items, list) and narration_items:
|
|
1110
1126
|
cleaned = [str(s).strip() for s in narration_items if str(s).strip()]
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1127
|
+
return "\n".join(cleaned)
|
|
1128
|
+
text = payload.get("text", "")
|
|
1129
|
+
if not text:
|
|
1130
|
+
return ""
|
|
1131
|
+
segments = segment_narration(text)
|
|
1132
|
+
return "\n".join(s.strip() for s in segments) if segments else text
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
def _call_gen_voice(tts_text: str, payload: dict, private_token: str, timeout: int) -> dict:
|
|
1136
|
+
|
|
1137
|
+
try:
|
|
1138
|
+
cmd = build_skill_command("gen-voice")
|
|
1139
|
+
except RuntimeError as e:
|
|
1140
|
+
return {"status": "failed", "error": str(e)}
|
|
1120
1141
|
|
|
1121
1142
|
cmd.extend(["--text", tts_text, "--json-output"])
|
|
1122
1143
|
if payload.get("voiceId"):
|
|
@@ -1379,6 +1400,7 @@ def _resolve_single_asset(
|
|
|
1379
1400
|
if isinstance(src_payload.get("narrationItems"), list):
|
|
1380
1401
|
asset["narrationLineCount"] = len(src_payload["narrationItems"])
|
|
1381
1402
|
asset["narrationIntroLines"] = int(src_payload.get("narrationIntroLines") or 0)
|
|
1403
|
+
asset["narrationLines"] = list(src_payload["narrationItems"])
|
|
1382
1404
|
LogPrint(f" ✅ {asset['assetId']} generated", file=sys.stderr)
|
|
1383
1405
|
return True
|
|
1384
1406
|
else:
|
|
@@ -1441,6 +1463,179 @@ def apply_stub_urls(render_plan: dict, stub_image_url: str = "", stub_video_url:
|
|
|
1441
1463
|
return stubbed
|
|
1442
1464
|
|
|
1443
1465
|
|
|
1466
|
+
def plan_narration_runs(render_plan: dict) -> tuple[list[list[dict]], dict]:
|
|
1467
|
+
|
|
1468
|
+
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
import _continuous_narration as cn
|
|
1472
|
+
|
|
1473
|
+
cfg = cn.resolve_config(_timing_template(render_plan.get("templateId", "")), render_plan.get("narration"))
|
|
1474
|
+
if cfg["mode"] != cn.MODE_CONTINUOUS:
|
|
1475
|
+
return [], cfg
|
|
1476
|
+
assets = {a.get("assetId"): a for a in render_plan.get("resolvedAssets", [])}
|
|
1477
|
+
payloads = {
|
|
1478
|
+
a.get("assetId"): a.get("payload") or {}
|
|
1479
|
+
for a in render_plan.get("_dsl_assets", [])
|
|
1480
|
+
if a.get("assetId")
|
|
1481
|
+
}
|
|
1482
|
+
timeline = render_plan.get("timeline", [])
|
|
1483
|
+
refs: dict[str, int] = {}
|
|
1484
|
+
for entry in timeline:
|
|
1485
|
+
nar_id = (entry.get("props") or {}).get("narrationAssetId")
|
|
1486
|
+
if nar_id:
|
|
1487
|
+
refs[nar_id] = refs.get(nar_id, 0) + 1
|
|
1488
|
+
|
|
1489
|
+
candidates: list[Optional[dict]] = []
|
|
1490
|
+
for entry in timeline:
|
|
1491
|
+
nar_id = (entry.get("props") or {}).get("narrationAssetId")
|
|
1492
|
+
asset = assets.get(nar_id)
|
|
1493
|
+
payload = payloads.get(nar_id) or {}
|
|
1494
|
+
text = _tts_text_from_payload(payload) if asset else ""
|
|
1495
|
+
if (
|
|
1496
|
+
not asset
|
|
1497
|
+
or asset.get("source") != "gen-voice"
|
|
1498
|
+
or asset.get("status") != "pending"
|
|
1499
|
+
or refs.get(nar_id) != 1
|
|
1500
|
+
or not text
|
|
1501
|
+
):
|
|
1502
|
+
candidates.append(None)
|
|
1503
|
+
continue
|
|
1504
|
+
candidates.append({
|
|
1505
|
+
"assetId": nar_id,
|
|
1506
|
+
"sceneId": entry.get("sceneId"),
|
|
1507
|
+
"text": text,
|
|
1508
|
+
"voiceKey": (_narration_voice(payload), _narration_speed(payload), _narration_language_boost(payload)),
|
|
1509
|
+
"break": bool(entry.get("narrationBreak")),
|
|
1510
|
+
})
|
|
1511
|
+
return cn.group_runs(candidates), cfg
|
|
1512
|
+
|
|
1513
|
+
|
|
1514
|
+
def _download_to(url: str, path: str, timeout: int) -> str:
|
|
1515
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
1516
|
+
with urllib.request.urlopen(url, timeout=timeout) as resp, open(path, "wb") as f:
|
|
1517
|
+
while True:
|
|
1518
|
+
chunk = resp.read(1 << 16)
|
|
1519
|
+
if not chunk:
|
|
1520
|
+
break
|
|
1521
|
+
f.write(chunk)
|
|
1522
|
+
return path
|
|
1523
|
+
|
|
1524
|
+
|
|
1525
|
+
def _upload_tts_clip(path: str, title: str) -> str:
|
|
1526
|
+
|
|
1527
|
+
|
|
1528
|
+
|
|
1529
|
+
|
|
1530
|
+
|
|
1531
|
+
|
|
1532
|
+
from upload_video import upload_media_file
|
|
1533
|
+
data = upload_media_file(path, title=title, role="tts_clip", retention="intermediate")
|
|
1534
|
+
url = data.get("signedUrl") or data["fileUrl"]
|
|
1535
|
+
if not data.get("signedUrl"):
|
|
1536
|
+
req = urllib.request.Request(url, headers={"Range": "bytes=0-0"})
|
|
1537
|
+
with urllib.request.urlopen(req, timeout=15):
|
|
1538
|
+
pass
|
|
1539
|
+
return url
|
|
1540
|
+
|
|
1541
|
+
|
|
1542
|
+
def resolve_narration_run(
|
|
1543
|
+
run: list[dict],
|
|
1544
|
+
cfg: dict,
|
|
1545
|
+
render_plan: dict,
|
|
1546
|
+
private_token: str,
|
|
1547
|
+
max_retries: int,
|
|
1548
|
+
timeout: int,
|
|
1549
|
+
work_dir: Optional[str] = None,
|
|
1550
|
+
) -> bool:
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
import _continuous_narration as cn
|
|
1554
|
+
from _digital_human_segments import find_ffmpeg
|
|
1555
|
+
|
|
1556
|
+
scene_ids = [c.get("sceneId") for c in run]
|
|
1557
|
+
|
|
1558
|
+
def log(message: str, level: str = "info") -> None:
|
|
1559
|
+
render_plan.setdefault("logs", []).append({
|
|
1560
|
+
"phase": "narration-run",
|
|
1561
|
+
"level": level,
|
|
1562
|
+
"message": message,
|
|
1563
|
+
"timestamp": now_iso(),
|
|
1564
|
+
})
|
|
1565
|
+
LogPrint(f" 🎙️ {message}", file=sys.stderr)
|
|
1566
|
+
|
|
1567
|
+
ffmpeg = find_ffmpeg()
|
|
1568
|
+
if not ffmpeg:
|
|
1569
|
+
log(f"ffmpeg not found; scenes {scene_ids} fall back to per-scene TTS", level="warn")
|
|
1570
|
+
return False
|
|
1571
|
+
|
|
1572
|
+
pause = cfg["scenePauseSec"]
|
|
1573
|
+
texts = [c["text"] for c in run]
|
|
1574
|
+
run_text, ranges = cn.compose_run_text(texts, pause)
|
|
1575
|
+
payloads = {
|
|
1576
|
+
a.get("assetId"): a.get("payload") or {}
|
|
1577
|
+
for a in render_plan.get("_dsl_assets", [])
|
|
1578
|
+
if a.get("assetId")
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1581
|
+
voice_payload = payloads.get(run[0]["assetId"]) or {}
|
|
1582
|
+
|
|
1583
|
+
result: dict = {}
|
|
1584
|
+
for attempt in range(max_retries):
|
|
1585
|
+
LogPrint(f" 🔄 Generating continuous narration for {len(run)} scenes (attempt {attempt + 1}/{max_retries})...", file=sys.stderr)
|
|
1586
|
+
result = _call_gen_voice(run_text, voice_payload, private_token, timeout)
|
|
1587
|
+
if result.get("status") == "generated":
|
|
1588
|
+
break
|
|
1589
|
+
if attempt < max_retries - 1:
|
|
1590
|
+
time.sleep(2)
|
|
1591
|
+
if result.get("status") != "generated":
|
|
1592
|
+
log(f"continuous TTS failed for scenes {scene_ids}: {str(result.get('error'))[:300]}; falling back to per-scene TTS", level="warn")
|
|
1593
|
+
return False
|
|
1594
|
+
|
|
1595
|
+
try:
|
|
1596
|
+
total_ms = float(result.get("audio_length_ms") or 0)
|
|
1597
|
+
if total_ms <= 0:
|
|
1598
|
+
raise RuntimeError("TTS returned no audio length")
|
|
1599
|
+
out_dir = work_dir or os.path.join(OUTPUT_DIR, "narration")
|
|
1600
|
+
run_name = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in run[0]["assetId"])
|
|
1601
|
+
local = _download_to(result["url"], os.path.join(out_dir, f"run-{run_name}.mp3"), timeout)
|
|
1602
|
+
subs = result.get("subtitles") or []
|
|
1603
|
+
silences = cn.detect_silences(local, total_ms, ffmpeg)
|
|
1604
|
+
cuts, method = cn.find_cuts(texts, ranges, subs, silences, total_ms, pause, run_text)
|
|
1605
|
+
if cuts is None:
|
|
1606
|
+
log(f"no reliable cut points for scenes {scene_ids}; falling back to per-scene TTS", level="warn")
|
|
1607
|
+
return False
|
|
1608
|
+
bounds = cn.slice_bounds(cuts, total_ms)
|
|
1609
|
+
paths = cn.slice_audio(local, bounds, out_dir, [c["assetId"] for c in run], ffmpeg)
|
|
1610
|
+
urls = [_upload_tts_clip(path, f"narration {c['assetId']}") for path, c in zip(paths, run)]
|
|
1611
|
+
except Exception as exc:
|
|
1612
|
+
log(f"continuous narration split failed for scenes {scene_ids}: {exc}; falling back to per-scene TTS", level="warn")
|
|
1613
|
+
return False
|
|
1614
|
+
|
|
1615
|
+
per_scene_subs = cn.split_subtitles(subs, bounds)
|
|
1616
|
+
assets = {a.get("assetId"): a for a in render_plan.get("resolvedAssets", [])}
|
|
1617
|
+
run_id = run[0]["assetId"]
|
|
1618
|
+
for k, c in enumerate(run):
|
|
1619
|
+
asset = assets[c["assetId"]]
|
|
1620
|
+
start, end = bounds[k]
|
|
1621
|
+
asset["status"] = "generated"
|
|
1622
|
+
asset["url"] = urls[k]
|
|
1623
|
+
asset["duration"] = int(round(end - start))
|
|
1624
|
+
asset["generatedBy"] = {"skill": "gen-voice", "mode": "continuous", "startedAt": now_iso(), "completedAt": now_iso()}
|
|
1625
|
+
|
|
1626
|
+
asset["narrationRun"] = {"id": run_id, "index": k, "count": len(run), "method": method}
|
|
1627
|
+
asset["speechSpanMs"] = list(cn.speech_span(bounds[k], silences))
|
|
1628
|
+
if per_scene_subs[k]:
|
|
1629
|
+
asset["ttsSubtitles"] = per_scene_subs[k]
|
|
1630
|
+
payload = payloads.get(c["assetId"]) or {}
|
|
1631
|
+
if isinstance(payload.get("narrationItems"), list):
|
|
1632
|
+
asset["narrationLineCount"] = len(payload["narrationItems"])
|
|
1633
|
+
asset["narrationIntroLines"] = int(payload.get("narrationIntroLines") or 0)
|
|
1634
|
+
asset["narrationLines"] = list(payload["narrationItems"])
|
|
1635
|
+
log(f"continuous narration: {len(run)} scenes in one TTS call, {total_ms / 1000:.1f}s, cut by {method}")
|
|
1636
|
+
return True
|
|
1637
|
+
|
|
1638
|
+
|
|
1444
1639
|
def resolve_assets(
|
|
1445
1640
|
render_plan: dict,
|
|
1446
1641
|
private_token: str,
|
|
@@ -1480,21 +1675,56 @@ def resolve_assets(
|
|
|
1480
1675
|
failed = 0
|
|
1481
1676
|
|
|
1482
1677
|
|
|
1678
|
+
runs: list[list[dict]] = []
|
|
1679
|
+
narration_cfg: dict = {}
|
|
1680
|
+
if phase1:
|
|
1681
|
+
runs, narration_cfg = plan_narration_runs(render_plan)
|
|
1682
|
+
run_asset_ids = {c["assetId"] for run in runs for c in run}
|
|
1683
|
+
singles = [a for a in phase1 if a.get("assetId") not in run_asset_ids]
|
|
1684
|
+
assets_by_id = {a.get("assetId"): a for a in phase1}
|
|
1685
|
+
|
|
1686
|
+
def resolve_run(run: list[dict]) -> tuple[int, int]:
|
|
1687
|
+
if resolve_narration_run(run, narration_cfg, render_plan, private_token, max_retries, timeout):
|
|
1688
|
+
return len(run), 0
|
|
1689
|
+
ok_count = fail_count = 0
|
|
1690
|
+
for c in run:
|
|
1691
|
+
asset = assets_by_id[c["assetId"]]
|
|
1692
|
+
if _resolve_single_asset(asset, render_plan, private_token, max_retries, timeout):
|
|
1693
|
+
ok_count += 1
|
|
1694
|
+
elif asset["status"] == "failed":
|
|
1695
|
+
fail_count += 1
|
|
1696
|
+
return ok_count, fail_count
|
|
1697
|
+
|
|
1698
|
+
|
|
1483
1699
|
if phase1:
|
|
1484
1700
|
default_parallelism = 8
|
|
1485
1701
|
max_workers = min(
|
|
1486
1702
|
int(os.environ.get("REMOTION_ASSET_PARALLELISM", str(default_parallelism))),
|
|
1487
|
-
len(
|
|
1703
|
+
len(singles) + len(runs),
|
|
1488
1704
|
)
|
|
1489
1705
|
LogPrint(f" ⚡ Phase 1: generating {len(phase1)} asset(s) in parallel (concurrency={max_workers})", file=sys.stderr)
|
|
1706
|
+
if runs:
|
|
1707
|
+
LogPrint(f" 🎙️ {len(runs)} continuous narration run(s) covering {len(run_asset_ids)} scene(s)", file=sys.stderr)
|
|
1490
1708
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
1491
1709
|
future_to_asset = {
|
|
1492
1710
|
executor.submit(
|
|
1493
1711
|
_resolve_single_asset, asset, render_plan, private_token, max_retries, timeout
|
|
1494
1712
|
): asset
|
|
1495
|
-
for asset in
|
|
1713
|
+
for asset in singles
|
|
1496
1714
|
}
|
|
1497
|
-
for
|
|
1715
|
+
future_to_run = {executor.submit(resolve_run, run): run for run in runs}
|
|
1716
|
+
for future in as_completed([*future_to_asset, *future_to_run]):
|
|
1717
|
+
if future in future_to_run:
|
|
1718
|
+
try:
|
|
1719
|
+
ok_count, fail_count = future.result()
|
|
1720
|
+
generated += ok_count
|
|
1721
|
+
failed += fail_count
|
|
1722
|
+
except Exception as exc:
|
|
1723
|
+
for c in future_to_run[future]:
|
|
1724
|
+
assets_by_id[c["assetId"]]["status"] = "failed"
|
|
1725
|
+
failed += 1
|
|
1726
|
+
LogPrint(f" ❌ continuous narration run exception: {exc}", file=sys.stderr)
|
|
1727
|
+
continue
|
|
1498
1728
|
asset = future_to_asset[future]
|
|
1499
1729
|
try:
|
|
1500
1730
|
ok = future.result()
|
|
@@ -1687,9 +1917,8 @@ def _auto_highlight_map(
|
|
|
1687
1917
|
return
|
|
1688
1918
|
|
|
1689
1919
|
props = entry.get("props") or {}
|
|
1690
|
-
|
|
1691
|
-
if
|
|
1692
|
-
return
|
|
1920
|
+
nested = isinstance(props.get("templateData"), dict)
|
|
1921
|
+
tdata = props["templateData"] if nested else props
|
|
1693
1922
|
if tdata.get("highlightMap"):
|
|
1694
1923
|
return
|
|
1695
1924
|
|
|
@@ -1721,7 +1950,8 @@ def _auto_highlight_map(
|
|
|
1721
1950
|
|
|
1722
1951
|
highlight_map = {str(intro_lines + i): i for i in range(item_count) if item_count > 0}
|
|
1723
1952
|
tdata["highlightMap"] = highlight_map
|
|
1724
|
-
|
|
1953
|
+
if nested:
|
|
1954
|
+
props["templateData"] = tdata
|
|
1725
1955
|
entry["props"] = props
|
|
1726
1956
|
|
|
1727
1957
|
render_plan.setdefault("logs", []).append({
|
|
@@ -1758,6 +1988,49 @@ def _template_default_tail_pad_frames(tpl: dict, fps: int) -> int | None:
|
|
|
1758
1988
|
return int(round(float(value) * fps))
|
|
1759
1989
|
|
|
1760
1990
|
|
|
1991
|
+
def _structured_narration_subtitles(entry: dict, asset: dict, render_plan: dict) -> list[dict]:
|
|
1992
|
+
|
|
1993
|
+
from _narration_alignment import align_narration_lines
|
|
1994
|
+
|
|
1995
|
+
subtitles = asset.get("ttsSubtitles") or []
|
|
1996
|
+
line_count = asset.get("narrationLineCount")
|
|
1997
|
+
if not isinstance(line_count, int) or isinstance(line_count, bool) or line_count <= 0:
|
|
1998
|
+
return subtitles
|
|
1999
|
+
|
|
2000
|
+
|
|
2001
|
+
old_subs = entry.get("subtitleSegments") or []
|
|
2002
|
+
at_sec = entry.get("subtitleAtSec") or []
|
|
2003
|
+
if len(at_sec) == len(old_subs) and any(isinstance(v, (int, float)) for v in at_sec):
|
|
2004
|
+
return []
|
|
2005
|
+
|
|
2006
|
+
lines = asset.get("narrationLines")
|
|
2007
|
+
if not isinstance(lines, list) or len(lines) != line_count:
|
|
2008
|
+
lines = [s.get("text") for s in old_subs] if len(old_subs) == line_count else []
|
|
2009
|
+
if len(lines) != line_count or any(not isinstance(v, str) or not v.strip() for v in lines):
|
|
2010
|
+
method = "authored-lines-unavailable"
|
|
2011
|
+
message = "Authored narration lines unavailable; cannot restore per-line timing"
|
|
2012
|
+
else:
|
|
2013
|
+
|
|
2014
|
+
asset["narrationLines"] = list(lines)
|
|
2015
|
+
subtitles, method = align_narration_lines(
|
|
2016
|
+
lines, subtitles, asset["duration"], asset.get("speechSpanMs"),
|
|
2017
|
+
)
|
|
2018
|
+
message = f"Preserved {line_count} narration lines using {method} timing"
|
|
2019
|
+
|
|
2020
|
+
logs = render_plan.setdefault("logs", [])
|
|
2021
|
+
if not any(l.get("phase") == "narration-align" and l.get("sceneId") == entry.get("sceneId")
|
|
2022
|
+
and l.get("method") == method for l in logs):
|
|
2023
|
+
logs.append({
|
|
2024
|
+
"phase": "narration-align",
|
|
2025
|
+
"sceneId": entry.get("sceneId"),
|
|
2026
|
+
"method": method,
|
|
2027
|
+
"level": "info" if method == "tts-boundaries" else "warn",
|
|
2028
|
+
"message": message,
|
|
2029
|
+
"timestamp": now_iso(),
|
|
2030
|
+
})
|
|
2031
|
+
return subtitles
|
|
2032
|
+
|
|
2033
|
+
|
|
1761
2034
|
def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
1762
2035
|
|
|
1763
2036
|
|
|
@@ -1801,10 +2074,17 @@ def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
|
1801
2074
|
|
|
1802
2075
|
|
|
1803
2076
|
tail_pad_override = entry.get("tailPadFrames")
|
|
2077
|
+
|
|
2078
|
+
|
|
2079
|
+
run = narration_asset.get("narrationRun") or {}
|
|
2080
|
+
scene_default_tail = (
|
|
2081
|
+
0 if run and int(run.get("index", 0)) < int(run.get("count", 0)) - 1
|
|
2082
|
+
else default_tail_pad
|
|
2083
|
+
)
|
|
1804
2084
|
tail_pad = (
|
|
1805
2085
|
int(tail_pad_override)
|
|
1806
2086
|
if isinstance(tail_pad_override, int) and tail_pad_override >= 0
|
|
1807
|
-
else
|
|
2087
|
+
else scene_default_tail
|
|
1808
2088
|
)
|
|
1809
2089
|
min_dur = int(fps * 2)
|
|
1810
2090
|
|
|
@@ -1816,7 +2096,7 @@ def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
|
1816
2096
|
entry["durationFrames"] = new_dur
|
|
1817
2097
|
adjusted += 1
|
|
1818
2098
|
|
|
1819
|
-
tts_subs =
|
|
2099
|
+
tts_subs = _structured_narration_subtitles(entry, narration_asset, render_plan)
|
|
1820
2100
|
if tts_subs:
|
|
1821
2101
|
|
|
1822
2102
|
|
|
@@ -1828,12 +2108,13 @@ def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
|
1828
2108
|
)
|
|
1829
2109
|
entry["subtitleSegments"] = [
|
|
1830
2110
|
{
|
|
1831
|
-
"text": _strip_subtitle_trailing_punct(s["text"].strip()),
|
|
2111
|
+
"text": _strip_subtitle_trailing_punct(_PAUSE_TAG_RE.sub("", s["text"]).strip()),
|
|
1832
2112
|
"startFrame": current_frame + int(s["timeBegin"] / 1000.0 * fps),
|
|
1833
2113
|
"endFrame": current_frame + int(s["timeEnd"] / 1000.0 * fps),
|
|
1834
2114
|
"sentenceIndex": sentence_indices[i],
|
|
1835
2115
|
}
|
|
1836
2116
|
for i, s in enumerate(tts_subs)
|
|
2117
|
+
if _PAUSE_TAG_RE.sub("", s.get("text") or "").strip()
|
|
1837
2118
|
]
|
|
1838
2119
|
_warn_step_sentence_mismatch(entry, sentence_indices, step_field)
|
|
1839
2120
|
else:
|
|
@@ -1887,13 +2168,18 @@ def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
|
1887
2168
|
entry["subtitleSegments"] = new_subs
|
|
1888
2169
|
elif old_subs:
|
|
1889
2170
|
sub_span = audio_frames
|
|
2171
|
+
cursor = 0
|
|
2172
|
+
|
|
2173
|
+
span = narration_asset.get("speechSpanMs")
|
|
2174
|
+
if isinstance(span, list) and len(span) == 2 and 0 <= span[0] < span[1]:
|
|
2175
|
+
cursor = int(span[0] / 1000.0 * fps)
|
|
2176
|
+
sub_span = max(math.ceil(span[1] / 1000.0 * fps) - cursor, fps // 2)
|
|
1890
2177
|
|
|
1891
2178
|
|
|
1892
2179
|
def _plain(t: str) -> int:
|
|
1893
2180
|
return len(re.sub(r"\*+", "", t or ""))
|
|
1894
2181
|
total_chars = max(sum(_plain(s.get("text", "")) for s in old_subs), 1)
|
|
1895
2182
|
new_subs = []
|
|
1896
|
-
cursor = 0
|
|
1897
2183
|
for s in old_subs:
|
|
1898
2184
|
text = s.get("text", "")
|
|
1899
2185
|
ratio = _plain(text) / total_chars
|
|
@@ -3143,7 +3429,7 @@ def _persist_render_plan(render_plan: dict, args, private_token: str, dsl: dict
|
|
|
3143
3429
|
args.save_job = False
|
|
3144
3430
|
|
|
3145
3431
|
|
|
3146
|
-
def _print_resolve_only_summary(render_plan: dict) -> int:
|
|
3432
|
+
def _print_resolve_only_summary(render_plan: dict, dsl: dict | None = None) -> int:
|
|
3147
3433
|
|
|
3148
3434
|
generated = sum(1 for a in render_plan["resolvedAssets"] if a["status"] == "generated")
|
|
3149
3435
|
failed = sum(1 for a in render_plan["resolvedAssets"] if a["status"] == "failed")
|
|
@@ -3202,6 +3488,16 @@ def _print_resolve_only_summary(render_plan: dict) -> int:
|
|
|
3202
3488
|
print(f" {asset['assetId']:<25} {asset.get('status', '?'):<10} scenes: {scenes_on} {asset.get('url', '')}")
|
|
3203
3489
|
else:
|
|
3204
3490
|
print(f"\n🧑 Digital-human segments: none (this render has no presenter)")
|
|
3491
|
+
from _scene_assets import scene_assets
|
|
3492
|
+
narration_texts = {}
|
|
3493
|
+
for scene in (dsl or {}).get("scenes", []):
|
|
3494
|
+
authored = (scene.get("audio") or {}).get("narration") or {}
|
|
3495
|
+
lines = extract_narration_lines(authored)
|
|
3496
|
+
if lines:
|
|
3497
|
+
narration_texts[scene["id"]] = "\n".join(lines[0])
|
|
3498
|
+
elif isinstance(authored.get("text"), str):
|
|
3499
|
+
narration_texts[scene["id"]] = authored["text"]
|
|
3500
|
+
print("__scene_assets__ " + json.dumps(scene_assets(render_plan, narration_texts), ensure_ascii=False))
|
|
3205
3501
|
|
|
3206
3502
|
return failed
|
|
3207
3503
|
|
|
@@ -3427,7 +3723,7 @@ def main():
|
|
|
3427
3723
|
LogPrint(f"💾 RenderPlan saved: {rp_output}", file=sys.stderr)
|
|
3428
3724
|
|
|
3429
3725
|
if args.resolve_only:
|
|
3430
|
-
if _print_resolve_only_summary(render_plan) > 0:
|
|
3726
|
+
if _print_resolve_only_summary(render_plan, dsl) > 0:
|
|
3431
3727
|
sys.exit(1)
|
|
3432
3728
|
return
|
|
3433
3729
|
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "render-video",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "473",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V42",
|
|
6
6
|
"skillDescription": "Final-render skill: loads a persisted RenderPlan by `job_id` and drives the Remotion engine to produce the final video.\n\nUse this skill as soon as the user mentions any of these intents (after assets are already prepared):\n- Render the video, composite the video, export the video\n- Turn the prepared assets into the final clip\n- Render with Remotion\n\nPrerequisite: assets must already be generated via `prepare_video_assets`. This skill never resolves or regenerates assets — pass it a `job_id` from a previous `prepare_video_assets` call.\n\n⚠️ Stop-and-confirm gate: never call this skill until the user has explicitly confirmed the assets prepared by `prepare_video_assets`. If those assets were prepared in the current turn and the user has not replied since, stop and ask instead of rendering."
|
|
7
7
|
}
|
|
@@ -199,6 +199,7 @@ Rules:
|
|
|
199
199
|
- The legacy flat `narration.text` form still works but the author then has to keep `highlightMap` in sync with the subtitle segmentation (error-prone, not recommended).
|
|
200
200
|
- `narration.languageBoost` (optional, also accepted at `global.narration`) pins the TTS language hint (`auto` / `Chinese` / `English` / …). Leave it unset unless auto-detection gets it wrong — the service applies its own default, and a scene-level value overrides the global one, exactly like `speed`.
|
|
201
201
|
- Voice: to change the narration voice, set `global.narration.voiceId` (whole video) or `narration.voiceId` on a scene (that scene only). Precedence when synthesizing is scene > global > the audio asset's `payload.voiceId`, which is only a fallback — editing the asset payload alone is not how you change a voice.
|
|
202
|
+
- Continuous narration: templates that declare `capabilities.narration.mode: "continuous"` get adjacent scenes with the same voice / speed / languageBoost synthesized in one TTS call and cut back per scene, so pace and loudness stay even. Nothing to author for it. Set `narration.break: true` on a scene to start a new run there (a chapter change that wants a longer pause); `global.narration.mode` (`continuous` / `perScene`) overrides the template only when the user asks for it.
|
|
202
203
|
|
|
203
204
|
## Image sources
|
|
204
205
|
|
|
@@ -109,7 +109,9 @@
|
|
|
109
109
|
"voiceId": { "type": "string", "description": "Minimax 音色 ID" },
|
|
110
110
|
"voiceName": { "type": "string" },
|
|
111
111
|
"speed": { "type": "number", "default": 1.0 },
|
|
112
|
-
"languageBoost": { "type": "string", "description": "TTS 语种提示(auto / Chinese / English …)。中英混排的旁白不给提示时,英文词容易被按中文音系读出来;留空由服务端取默认值。" }
|
|
112
|
+
"languageBoost": { "type": "string", "description": "TTS 语种提示(auto / Chinese / English …)。中英混排的旁白不给提示时,英文词容易被按中文音系读出来;留空由服务端取默认值。" },
|
|
113
|
+
"mode": { "type": "string", "enum": ["continuous", "perScene"], "description": "旁白合成方式,覆盖模板 capabilities.narration.mode。continuous = 相邻、音色语速语种相同的场景整段合成一次再按场景切开(语速响度一致);perScene = 每个场景单独合成。留空跟随模板,模板没声明即 perScene。" },
|
|
114
|
+
"scenePauseSec": { "type": "number", "minimum": 0.01, "maximum": 99.99, "description": "continuous 模式下场景之间插入的停顿秒数(MiniMax <#x#>),覆盖模板声明;默认 0.8。" }
|
|
113
115
|
}
|
|
114
116
|
},
|
|
115
117
|
"bgm": {
|
|
@@ -268,6 +270,7 @@
|
|
|
268
270
|
"voiceId": { "type": "string", "description": "该场旁白音色,覆盖 global.narration.voiceId;留空跟随全局。合成时优先级:场景 > 全局 > 音频资产 payload.voiceId(仅作兜底)。" },
|
|
269
271
|
"voiceName": { "type": "string", "description": "voiceId 的展示名(仅展示用,不参与合成)" },
|
|
270
272
|
"speed": { "type": "number", "description": "该场旁白语速,覆盖 global.narration.speed;留空跟随全局。" },
|
|
273
|
+
"break": { "type": "boolean", "description": "continuous 旁白模式下从这一幕起另起一段整段合成(例如换章节、需要更长停顿)。perScene 模式下无效。" },
|
|
271
274
|
"languageBoost": { "type": "string", "description": "TTS 语种提示(auto / Chinese / English …)。中英混排的旁白不给提示时,英文词容易被按中文音系读出来;覆盖 global.narration 的同名字段;留空则跟随全局/服务端默认。" }
|
|
272
275
|
}
|
|
273
276
|
},
|