@remixmate/cli 0.13.5 → 0.13.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/manifest.json +2 -2
- package/package.json +1 -1
- package/skills/gen-script/SKILL.md +2 -0
- package/skills/gen-script/scripts/gen_script.py +114 -4
- package/skills/gen-script/version.json +1 -1
- package/skills/prepare-video-assets/SKILL.md +2 -0
- package/skills/prepare-video-assets/version.json +1 -1
- package/skills/render-video/scripts/_continuous_narration.py +11 -3
- package/skills/render-video/scripts/_narration_alignment.py +88 -0
- package/skills/render-video/scripts/_scene_assets.py +39 -0
- package/skills/render-video/scripts/render_video.py +64 -8
- package/skills/render-video/version.json +1 -1
package/dist/manifest.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remixmate/cli",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.6",
|
|
4
4
|
"description": "AI media generation skills for Claude Code / Codex — 12 skills covering image, video, voice, digital human, web screenshot, web recording, script, template registry, rendering, Jianying export, and video deconstruction.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -32,6 +32,8 @@ Turns a user-supplied **topic** into a **Video DSL v1alpha1** JSON describing wh
|
|
|
32
32
|
|
|
33
33
|
- **Video DSL**: a declarative description language for videos — describes the final structure, not the orchestration calls.
|
|
34
34
|
- **AssetRef**: an asset reference declaring which images / audio / video clips are needed; the downstream Render Skill is responsible for producing them.
|
|
35
|
+
|
|
36
|
+
Templates may declare `capabilities.mediaBinding.sceneMode=per-item`: the returned skeleton then contains one scene per input media item, with durations allocated from the complete sequence's existing timing model. Keep those scenes, media references and timing values; fill each scene's narration separately. Captions describe the visuals and are not automatically narration. Continuous TTS can synthesize adjacent scenes together and cut the result back into their separate audio assets; it does not merge the script's scenes.
|
|
35
37
|
- **Scene**: the unit cell of the video timeline.
|
|
36
38
|
- **Image `payload.model`**: must match the `gen-image` skill and the backend `ModelGenImgDTO`. **Only** values from the LiteLLM `model` allowlist below may be used. When editing narration or prompts you **must not** rewrite this field.
|
|
37
39
|
|
|
@@ -1129,8 +1129,8 @@ def _sequence_floor_frames(sequence: dict, preset_timing: dict, text: str, fps:
|
|
|
1129
1129
|
return math.floor(min_sec * fps + 0.5)
|
|
1130
1130
|
|
|
1131
1131
|
|
|
1132
|
-
def
|
|
1133
|
-
|
|
1132
|
+
def _fit_images_hold_frames(custom_payload: dict, model: dict | None, fps: int = 30,
|
|
1133
|
+
headline: str = "") -> list[int]:
|
|
1134
1134
|
|
|
1135
1135
|
|
|
1136
1136
|
|
|
@@ -1183,8 +1183,46 @@ def _estimate_fit_images_duration(custom_payload: dict, model: dict | None, fps:
|
|
|
1183
1183
|
if rate is None:
|
|
1184
1184
|
raise ValueError(f"fit-images with a floor requires durationModel.{key}.floorMinVideoRate")
|
|
1185
1185
|
floor_frames = min(floor_frames, sum(math.floor(f / rate) for f in frames))
|
|
1186
|
-
|
|
1187
|
-
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
target = max(natural, floor_frames)
|
|
1190
|
+
original = list(frames)
|
|
1191
|
+
images = [item.get("type") != "video" for item in sequence["items"]]
|
|
1192
|
+
|
|
1193
|
+
def grow(eligible, caps):
|
|
1194
|
+
need = target - sum(frames)
|
|
1195
|
+
room = [max(0, caps[i] - frames[i]) if eligible[i] else 0 for i in range(len(frames))]
|
|
1196
|
+
capacity = sum(room)
|
|
1197
|
+
if need > 0 and capacity > 0:
|
|
1198
|
+
take = min(need, capacity)
|
|
1199
|
+
for i, r in enumerate(room):
|
|
1200
|
+
frames[i] += math.floor(r * take / capacity)
|
|
1201
|
+
|
|
1202
|
+
def uncapped(eligible):
|
|
1203
|
+
need = target - sum(frames)
|
|
1204
|
+
base = sum(f for i, f in enumerate(frames) if eligible[i])
|
|
1205
|
+
if need > 0 and base > 0:
|
|
1206
|
+
for i, f in enumerate(list(frames)):
|
|
1207
|
+
if eligible[i]:
|
|
1208
|
+
frames[i] += math.floor(f * need / base)
|
|
1209
|
+
|
|
1210
|
+
videos = [not image for image in images]
|
|
1211
|
+
grow(images, [math.floor(f * 2.5 + 0.5) for f in original])
|
|
1212
|
+
if any(videos):
|
|
1213
|
+
rate = timing.get("floorMinVideoRate")
|
|
1214
|
+
if not rate:
|
|
1215
|
+
raise ValueError(f"fit-images with a floor requires durationModel.{key}.floorMinVideoRate")
|
|
1216
|
+
grow(videos, [math.floor(f / rate) for f in original])
|
|
1217
|
+
uncapped(images)
|
|
1218
|
+
uncapped(videos)
|
|
1219
|
+
frames[-1] += target - sum(frames)
|
|
1220
|
+
return frames
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def _estimate_fit_images_duration(custom_payload: dict, model: dict | None, fps: int = 30,
|
|
1224
|
+
headline: str = "") -> float:
|
|
1225
|
+
return sum(_fit_images_hold_frames(custom_payload, model, fps, headline)) / fps
|
|
1188
1226
|
|
|
1189
1227
|
|
|
1190
1228
|
def _probe_sequence_video_duration(url: str) -> float | None:
|
|
@@ -1535,6 +1573,71 @@ def _build_carousel_caption_dsl(
|
|
|
1535
1573
|
for path in narration_locks:
|
|
1536
1574
|
_lock_scene_path(dsl, path)
|
|
1537
1575
|
|
|
1576
|
+
scene_mode = binding.get("sceneMode", "single")
|
|
1577
|
+
if scene_mode not in ("single", "per-item"):
|
|
1578
|
+
raise ValueError("mediaBinding.sceneMode must be single or per-item")
|
|
1579
|
+
if scene_mode == "per-item":
|
|
1580
|
+
if duration_strategy != "fit-images":
|
|
1581
|
+
raise ValueError("mediaBinding.sceneMode=per-item requires durationStrategy=fit-images")
|
|
1582
|
+
holds = _fit_images_hold_frames(custom_payload, duration_model, headline=headline)
|
|
1583
|
+
dsl["assets"] = [a for a in assets if a["source"] != "gen-voice"]
|
|
1584
|
+
dsl["scenes"] = []
|
|
1585
|
+
|
|
1586
|
+
|
|
1587
|
+
scene_defaults = [(path.split("."), value) for path, value in binding.get("sceneDefaults", {}).items()]
|
|
1588
|
+
for index, (item, frames) in enumerate(zip(container[items_parts[-1]], holds)):
|
|
1589
|
+
current = deepcopy(scene)
|
|
1590
|
+
current["id"] = f"scene-{index + 1:02d}"
|
|
1591
|
+
current["purpose"] = "opening" if index == 0 else "point"
|
|
1592
|
+
current["duration"] = frames / 30
|
|
1593
|
+
payload = current["customPayload"]
|
|
1594
|
+
_write_payload(payload, scene_defaults)
|
|
1595
|
+
_write_payload(payload, payload_writes or [])
|
|
1596
|
+
pinned = {**item, "holdSec": frames / 30}
|
|
1597
|
+
hold_schema = (_schema_at(template_config, items_parts).get("items", {}).get("properties", {}).get("holdSec", {}))
|
|
1598
|
+
if not hold_schema.get("minimum", 0) <= frames / 30 <= hold_schema.get("maximum", float("inf")):
|
|
1599
|
+
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
pinned.pop("holdSec", None)
|
|
1603
|
+
_write_payload(payload, [(items_parts, [pinned])])
|
|
1604
|
+
default_hold_schema = _schema_at(template_config, container_parts + ["defaultHoldSec"])
|
|
1605
|
+
if default_hold_schema and default_hold_schema.get("minimum", 0) <= frames / 30 <= default_hold_schema.get("maximum", float("inf")):
|
|
1606
|
+
_write_payload(payload, [(container_parts + ["defaultHoldSec"], frames / 30)])
|
|
1607
|
+
|
|
1608
|
+
|
|
1609
|
+
if _schema_at(template_config, container_parts + ["minSec"]):
|
|
1610
|
+
minimum = _schema_at(template_config, container_parts + ["minSec"]).get("minimum", 0)
|
|
1611
|
+
if frames / 30 >= minimum:
|
|
1612
|
+
_write_payload(payload, [(container_parts + ["minSec"], frames / 30)])
|
|
1613
|
+
else:
|
|
1614
|
+
_get_path(payload, container_parts).pop("minSec", None)
|
|
1615
|
+
if narration_enabled:
|
|
1616
|
+
if "minDurationSec" in payload:
|
|
1617
|
+
payload["minDurationSec"] = frames / 30
|
|
1618
|
+
ref = f"narration-{current['id']}"
|
|
1619
|
+
current["audio"]["narration"].update(
|
|
1620
|
+
assetRef=ref,
|
|
1621
|
+
text=_localize_label(
|
|
1622
|
+
output_language, "narration_opening", label="", topic=_clip(topic, _PLACEHOLDER_TOPIC_CHARS),
|
|
1623
|
+
duration=frames / 30,
|
|
1624
|
+
budget=_budget_suffix(frames / 30, output_language, narration_speed, None),
|
|
1625
|
+
),
|
|
1626
|
+
)
|
|
1627
|
+
dsl["assets"].append({"assetId": ref, "type": "audio", "source": "gen-voice",
|
|
1628
|
+
"status": "planned", "payload": {"voiceId": resolved_voice_id}})
|
|
1629
|
+
dsl["scenes"].append(current)
|
|
1630
|
+
|
|
1631
|
+
|
|
1632
|
+
if container_parts:
|
|
1633
|
+
dsl["meta"]["mergeScenePaths"] = ["customPayload." + ".".join(container_parts)]
|
|
1634
|
+
if _schema_at(template_config, container_parts + ["defaultHoldSec"]):
|
|
1635
|
+
_lock_scene_path(dsl, "customPayload." + ".".join(container_parts + ["defaultHoldSec"]))
|
|
1636
|
+
for parts, _ in scene_defaults:
|
|
1637
|
+
_lock_scene_path(dsl, "customPayload." + ".".join(parts))
|
|
1638
|
+
if _schema_at(template_config, container_parts + ["minSec"]):
|
|
1639
|
+
_lock_scene_path(dsl, "customPayload." + ".".join(container_parts + ["minSec"]))
|
|
1640
|
+
|
|
1538
1641
|
return dsl
|
|
1539
1642
|
|
|
1540
1643
|
|
|
@@ -1918,6 +2021,13 @@ def _apply_scene_payload(
|
|
|
1918
2021
|
raise ValueError("--payload / --motion need --template-id: the allowed keys come from the template.")
|
|
1919
2022
|
|
|
1920
2023
|
writes = _parse_payload_specs(template_config, payload_specs)
|
|
2024
|
+
binding = _resolve_media_binding(template_config)
|
|
2025
|
+
if binding.get("sceneMode") == "per-item":
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
item_path = binding["itemsPath"].split(".")
|
|
2029
|
+
allocated = (item_path, item_path[:-1] + ["minSec"])
|
|
2030
|
+
writes = [(parts, value) for parts, value in writes if parts not in allocated]
|
|
1921
2031
|
if motion:
|
|
1922
2032
|
allowed = ("auto", "static", "kenburns-in", "kenburns-out", "pan-left", "pan-right", "float")
|
|
1923
2033
|
if motion not in allowed:
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "gen-script",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "474",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V24",
|
|
6
6
|
"skillDescription": "Video-script generation skill. Turns a user-supplied topic into a structured Video DSL (JSON) that describes the full video — scene structure, asset requirements, and narrative flow.\n\nUse this skill as soon as the user mentions any of these intents:\n- Write a video script, generate a video script, plan a video, write storyboards\n- Create a short video, plan video content, generate a Video DSL\n- Turn a topic into a video structure / video content plan\n\nEven when the user does not say \"generate the DSL\", use this skill whenever they want to turn a topic into a structured video plan.\n\n⚠️ Stop-and-confirm gate: after this skill returns a DSL, show the full script and wait for the user's explicit confirmation. Never call `prepare_video_assets` in the same turn."
|
|
7
7
|
}
|
|
@@ -112,6 +112,8 @@ This skill is the **first half** of a two-phase user-confirmation flow. After it
|
|
|
112
112
|
|
|
113
113
|
Read the `job_id` from the stdout line `📦 render job jobId: N` and remember it; asset URLs come from the `🔊 TTS audio:` and `🖼 Image assets:` sections. Show images inline (``) and audio as links (`[🔊 listen](http://cdn.../x.mp3)`) so the user can actually review them. The recommended summary format:
|
|
114
114
|
|
|
115
|
+
When `__scene_assets__` is present, the web UI renders the full scene preview from that structured timeline: all visuals, scene times and each scene's audio slice. Give a brief overview/status and request confirmation; do not duplicate the preview or copy URLs into prose. For CLI clients without this UI, render every referenced visual and the corresponding audio per scene from the marker. With older outputs, use the fallback table below, including **all** visuals in each scene rather than one representative image. Scene count is independent of the number of TTS synthesis calls. Visual captions must not automatically become narration.
|
|
116
|
+
|
|
115
117
|
```markdown
|
|
116
118
|
## Render confirmation
|
|
117
119
|
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "prepare-video-assets",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "476",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V8",
|
|
6
6
|
"skillDescription": "Asset-preparation skill: resolves and generates every asset (image / audio / video) referenced by a Video DSL, persists a RenderPlan to the database, and returns a `job_id` for the subsequent `render_video` call.\n\nUse this skill as soon as the user mentions any of these intents:\n- Generate / prepare video assets, resolve assets, render-ready\n- \"Make me a video about X\" (the agent calls gen_script → prepare_video_assets → render_video)\n- Regenerate one asset (image / audio) for a specific scene\n\nNext step: after the user confirms the resolved assets, call `render_video` with the `job_id` returned by this skill.\n\n⚠️ Stop-and-confirm gate: this skill runs only after the user has confirmed the script, and after it returns you must show the resolved assets and wait for the user's explicit confirmation. Never call `render_video` in the same turn."
|
|
7
7
|
}
|
|
@@ -50,6 +50,15 @@ _NATURAL_PAUSE_SEC = 0.45
|
|
|
50
50
|
|
|
51
51
|
_MIN_BOUNDARY_PAUSE_RATIO = 0.75
|
|
52
52
|
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
_MIN_BOUNDARY_PAUSE_SEC = 0.6
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _min_boundary_ms(pause_sec: float) -> float:
|
|
60
|
+
return max(_MIN_BOUNDARY_PAUSE_SEC, pause_sec * _MIN_BOUNDARY_PAUSE_RATIO) * 1000
|
|
61
|
+
|
|
53
62
|
_SILENCE_NOISE_DB = -40
|
|
54
63
|
_SILENCE_MIN_SEC = 0.25
|
|
55
64
|
|
|
@@ -282,7 +291,7 @@ def cuts_from_silences(
|
|
|
282
291
|
return None
|
|
283
292
|
longest = sorted(inner, key=lambda s: s[1] - s[0], reverse=True)[:need]
|
|
284
293
|
longest.sort(key=lambda s: s[0])
|
|
285
|
-
min_len = pause_sec
|
|
294
|
+
min_len = _min_boundary_ms(pause_sec)
|
|
286
295
|
if any(end - start < min_len for start, end in longest):
|
|
287
296
|
return None
|
|
288
297
|
cuts = [(start + end) / 2 for start, end in longest]
|
|
@@ -308,8 +317,7 @@ def _durations_plausible(cuts: list[float], texts: list[str], total_ms: float, p
|
|
|
308
317
|
def _cuts_on_boundary_pauses(cuts: list[float], silences: list[tuple[float, float]], pause_sec: float) -> bool:
|
|
309
318
|
|
|
310
319
|
|
|
311
|
-
|
|
312
|
-
min_len = pause_sec * _MIN_BOUNDARY_PAUSE_RATIO * 1000
|
|
320
|
+
min_len = _min_boundary_ms(pause_sec)
|
|
313
321
|
tol = 100.0
|
|
314
322
|
return all(
|
|
315
323
|
any(end - start >= min_len and start - tol <= c <= end + tol for start, end in silences)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
import unicodedata
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _normalized(text: str) -> str:
|
|
16
|
+
text = re.sub(r"<#\s*\d+(?:\.\d+)?\s*#>", "", text)
|
|
17
|
+
return "".join(c for c in unicodedata.normalize("NFKC", text).casefold() if c.isalnum())
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _number(value) -> bool:
|
|
21
|
+
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def align_narration_lines(
|
|
25
|
+
lines: list[str], subtitles: list[dict], duration_ms: float, speech_span_ms=None,
|
|
26
|
+
) -> tuple[list[dict], str]:
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
normalized = [_normalized(line) for line in lines]
|
|
34
|
+
weights = [max(len(text), 1) for text in normalized]
|
|
35
|
+
start_ms, end_ms = 0.0, float(duration_ms)
|
|
36
|
+
if (isinstance(speech_span_ms, list) and len(speech_span_ms) == 2
|
|
37
|
+
and all(_number(v) for v in speech_span_ms)
|
|
38
|
+
and 0 <= speech_span_ms[0] < speech_span_ms[1] <= duration_ms):
|
|
39
|
+
start_ms, end_ms = map(float, speech_span_ms)
|
|
40
|
+
|
|
41
|
+
anchors = []
|
|
42
|
+
offset = 0
|
|
43
|
+
previous_end = 0.0
|
|
44
|
+
for sub in subtitles:
|
|
45
|
+
if not isinstance(sub, dict) or not isinstance(sub.get("text"), str):
|
|
46
|
+
anchors = []
|
|
47
|
+
break
|
|
48
|
+
text = _normalized(sub["text"])
|
|
49
|
+
if not text:
|
|
50
|
+
continue
|
|
51
|
+
begin, end = sub.get("timeBegin"), sub.get("timeEnd")
|
|
52
|
+
if not (_number(begin) and _number(end) and previous_end <= begin < end <= duration_ms):
|
|
53
|
+
anchors = []
|
|
54
|
+
break
|
|
55
|
+
anchors.append((offset, offset + len(text), float(begin), float(end), text))
|
|
56
|
+
offset += len(text)
|
|
57
|
+
previous_end = end
|
|
58
|
+
|
|
59
|
+
matches = (anchors and all(normalized)
|
|
60
|
+
and "".join(normalized) == "".join(a[4] for a in anchors))
|
|
61
|
+
result = []
|
|
62
|
+
interpolated = False
|
|
63
|
+
if matches:
|
|
64
|
+
offset = 0
|
|
65
|
+
anchor_index = 0
|
|
66
|
+
for line, text in zip(lines, normalized):
|
|
67
|
+
line_end = offset + len(text)
|
|
68
|
+
while anchors[anchor_index][1] <= offset:
|
|
69
|
+
anchor_index += 1
|
|
70
|
+
first = anchors[anchor_index]
|
|
71
|
+
while anchors[anchor_index][1] < line_end:
|
|
72
|
+
anchor_index += 1
|
|
73
|
+
last = anchors[anchor_index]
|
|
74
|
+
begin = first[2] + (first[3] - first[2]) * (offset - first[0]) / (first[1] - first[0])
|
|
75
|
+
end = last[2] + (last[3] - last[2]) * (line_end - last[0]) / (last[1] - last[0])
|
|
76
|
+
interpolated |= offset != first[0] or line_end != last[1]
|
|
77
|
+
result.append({"text": line, "timeBegin": begin, "timeEnd": end})
|
|
78
|
+
offset = line_end
|
|
79
|
+
return result, "tts-interpolated" if interpolated else "tts-boundaries"
|
|
80
|
+
|
|
81
|
+
total = sum(weights)
|
|
82
|
+
offset = 0
|
|
83
|
+
for line, weight in zip(lines, weights):
|
|
84
|
+
begin = start_ms + (end_ms - start_ms) * offset / total
|
|
85
|
+
offset += weight
|
|
86
|
+
end = start_ms + (end_ms - start_ms) * offset / total
|
|
87
|
+
result.append({"text": line, "timeBegin": begin, "timeEnd": end})
|
|
88
|
+
return result, "estimated"
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def scene_assets(plan: dict, narration_texts: dict[str, str] | None = None) -> list[dict]:
|
|
6
|
+
assets = {a.get("assetId"): a for a in plan.get("resolvedAssets", [])}
|
|
7
|
+
fps = (plan.get("renderConfig") or {}).get("fps", 30)
|
|
8
|
+
out = []
|
|
9
|
+
|
|
10
|
+
def refs(value):
|
|
11
|
+
if isinstance(value, dict):
|
|
12
|
+
for key, child in value.items():
|
|
13
|
+
if isinstance(child, str) and key.lower().endswith(("assetref", "assetid")) and child in assets:
|
|
14
|
+
yield child
|
|
15
|
+
else:
|
|
16
|
+
yield from refs(child)
|
|
17
|
+
elif isinstance(value, list):
|
|
18
|
+
for child in value:
|
|
19
|
+
yield from refs(child)
|
|
20
|
+
|
|
21
|
+
for entry in plan.get("timeline", []):
|
|
22
|
+
narration = "".join(s.get("text", "") for s in entry.get("subtitleSegments", []))
|
|
23
|
+
narration = (narration_texts or {}).get(entry.get("sceneId"), narration)
|
|
24
|
+
for ref in dict.fromkeys(refs(entry.get("props") or {})):
|
|
25
|
+
asset = assets[ref]
|
|
26
|
+
if asset.get("type") not in ("image", "video", "audio") or asset.get("status") != "generated" or not asset.get("url"):
|
|
27
|
+
continue
|
|
28
|
+
item = {
|
|
29
|
+
"type": asset["type"], "url": asset["url"], "label": ref,
|
|
30
|
+
"sceneId": entry["sceneId"],
|
|
31
|
+
"sceneStartSec": entry.get("startFrame", 0) / fps,
|
|
32
|
+
"sceneEndSec": entry.get("endFrame", 0) / fps,
|
|
33
|
+
"narrationText": narration,
|
|
34
|
+
}
|
|
35
|
+
if asset.get("duration"):
|
|
36
|
+
|
|
37
|
+
item["durationSec"] = asset["duration"] / (1000 if asset["type"] == "audio" else 1)
|
|
38
|
+
out.append(item)
|
|
39
|
+
return out
|
|
@@ -27,6 +27,7 @@ import argparse
|
|
|
27
27
|
import atexit
|
|
28
28
|
import builtins
|
|
29
29
|
import json
|
|
30
|
+
import math
|
|
30
31
|
import os
|
|
31
32
|
import re
|
|
32
33
|
import signal
|
|
@@ -921,7 +922,7 @@ def build_render_plan(dsl: dict, binding: dict) -> dict:
|
|
|
921
922
|
for idx, scene in enumerate(dsl.get("scenes", [])):
|
|
922
923
|
scene_id = scene.get("id") or scene.get("sceneId") or f"scene-{idx:03d}"
|
|
923
924
|
duration = scene.get("duration", 5)
|
|
924
|
-
duration_frames =
|
|
925
|
+
duration_frames = math.floor(duration * fps + 0.5)
|
|
925
926
|
|
|
926
927
|
scene_binding = binding_map.get(scene_id, {})
|
|
927
928
|
|
|
@@ -1399,6 +1400,7 @@ def _resolve_single_asset(
|
|
|
1399
1400
|
if isinstance(src_payload.get("narrationItems"), list):
|
|
1400
1401
|
asset["narrationLineCount"] = len(src_payload["narrationItems"])
|
|
1401
1402
|
asset["narrationIntroLines"] = int(src_payload.get("narrationIntroLines") or 0)
|
|
1403
|
+
asset["narrationLines"] = list(src_payload["narrationItems"])
|
|
1402
1404
|
LogPrint(f" ✅ {asset['assetId']} generated", file=sys.stderr)
|
|
1403
1405
|
return True
|
|
1404
1406
|
else:
|
|
@@ -1629,6 +1631,7 @@ def resolve_narration_run(
|
|
|
1629
1631
|
if isinstance(payload.get("narrationItems"), list):
|
|
1630
1632
|
asset["narrationLineCount"] = len(payload["narrationItems"])
|
|
1631
1633
|
asset["narrationIntroLines"] = int(payload.get("narrationIntroLines") or 0)
|
|
1634
|
+
asset["narrationLines"] = list(payload["narrationItems"])
|
|
1632
1635
|
log(f"continuous narration: {len(run)} scenes in one TTS call, {total_ms / 1000:.1f}s, cut by {method}")
|
|
1633
1636
|
return True
|
|
1634
1637
|
|
|
@@ -1914,9 +1917,8 @@ def _auto_highlight_map(
|
|
|
1914
1917
|
return
|
|
1915
1918
|
|
|
1916
1919
|
props = entry.get("props") or {}
|
|
1917
|
-
|
|
1918
|
-
if
|
|
1919
|
-
return
|
|
1920
|
+
nested = isinstance(props.get("templateData"), dict)
|
|
1921
|
+
tdata = props["templateData"] if nested else props
|
|
1920
1922
|
if tdata.get("highlightMap"):
|
|
1921
1923
|
return
|
|
1922
1924
|
|
|
@@ -1948,7 +1950,8 @@ def _auto_highlight_map(
|
|
|
1948
1950
|
|
|
1949
1951
|
highlight_map = {str(intro_lines + i): i for i in range(item_count) if item_count > 0}
|
|
1950
1952
|
tdata["highlightMap"] = highlight_map
|
|
1951
|
-
|
|
1953
|
+
if nested:
|
|
1954
|
+
props["templateData"] = tdata
|
|
1952
1955
|
entry["props"] = props
|
|
1953
1956
|
|
|
1954
1957
|
render_plan.setdefault("logs", []).append({
|
|
@@ -1985,6 +1988,49 @@ def _template_default_tail_pad_frames(tpl: dict, fps: int) -> int | None:
|
|
|
1985
1988
|
return int(round(float(value) * fps))
|
|
1986
1989
|
|
|
1987
1990
|
|
|
1991
|
+
def _structured_narration_subtitles(entry: dict, asset: dict, render_plan: dict) -> list[dict]:
|
|
1992
|
+
|
|
1993
|
+
from _narration_alignment import align_narration_lines
|
|
1994
|
+
|
|
1995
|
+
subtitles = asset.get("ttsSubtitles") or []
|
|
1996
|
+
line_count = asset.get("narrationLineCount")
|
|
1997
|
+
if not isinstance(line_count, int) or isinstance(line_count, bool) or line_count <= 0:
|
|
1998
|
+
return subtitles
|
|
1999
|
+
|
|
2000
|
+
|
|
2001
|
+
old_subs = entry.get("subtitleSegments") or []
|
|
2002
|
+
at_sec = entry.get("subtitleAtSec") or []
|
|
2003
|
+
if len(at_sec) == len(old_subs) and any(isinstance(v, (int, float)) for v in at_sec):
|
|
2004
|
+
return []
|
|
2005
|
+
|
|
2006
|
+
lines = asset.get("narrationLines")
|
|
2007
|
+
if not isinstance(lines, list) or len(lines) != line_count:
|
|
2008
|
+
lines = [s.get("text") for s in old_subs] if len(old_subs) == line_count else []
|
|
2009
|
+
if len(lines) != line_count or any(not isinstance(v, str) or not v.strip() for v in lines):
|
|
2010
|
+
method = "authored-lines-unavailable"
|
|
2011
|
+
message = "Authored narration lines unavailable; cannot restore per-line timing"
|
|
2012
|
+
else:
|
|
2013
|
+
|
|
2014
|
+
asset["narrationLines"] = list(lines)
|
|
2015
|
+
subtitles, method = align_narration_lines(
|
|
2016
|
+
lines, subtitles, asset["duration"], asset.get("speechSpanMs"),
|
|
2017
|
+
)
|
|
2018
|
+
message = f"Preserved {line_count} narration lines using {method} timing"
|
|
2019
|
+
|
|
2020
|
+
logs = render_plan.setdefault("logs", [])
|
|
2021
|
+
if not any(l.get("phase") == "narration-align" and l.get("sceneId") == entry.get("sceneId")
|
|
2022
|
+
and l.get("method") == method for l in logs):
|
|
2023
|
+
logs.append({
|
|
2024
|
+
"phase": "narration-align",
|
|
2025
|
+
"sceneId": entry.get("sceneId"),
|
|
2026
|
+
"method": method,
|
|
2027
|
+
"level": "info" if method == "tts-boundaries" else "warn",
|
|
2028
|
+
"message": message,
|
|
2029
|
+
"timestamp": now_iso(),
|
|
2030
|
+
})
|
|
2031
|
+
return subtitles
|
|
2032
|
+
|
|
2033
|
+
|
|
1988
2034
|
def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
1989
2035
|
|
|
1990
2036
|
|
|
@@ -2050,7 +2096,7 @@ def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
|
2050
2096
|
entry["durationFrames"] = new_dur
|
|
2051
2097
|
adjusted += 1
|
|
2052
2098
|
|
|
2053
|
-
tts_subs =
|
|
2099
|
+
tts_subs = _structured_narration_subtitles(entry, narration_asset, render_plan)
|
|
2054
2100
|
if tts_subs:
|
|
2055
2101
|
|
|
2056
2102
|
|
|
@@ -3383,7 +3429,7 @@ def _persist_render_plan(render_plan: dict, args, private_token: str, dsl: dict
|
|
|
3383
3429
|
args.save_job = False
|
|
3384
3430
|
|
|
3385
3431
|
|
|
3386
|
-
def _print_resolve_only_summary(render_plan: dict) -> int:
|
|
3432
|
+
def _print_resolve_only_summary(render_plan: dict, dsl: dict | None = None) -> int:
|
|
3387
3433
|
|
|
3388
3434
|
generated = sum(1 for a in render_plan["resolvedAssets"] if a["status"] == "generated")
|
|
3389
3435
|
failed = sum(1 for a in render_plan["resolvedAssets"] if a["status"] == "failed")
|
|
@@ -3442,6 +3488,16 @@ def _print_resolve_only_summary(render_plan: dict) -> int:
|
|
|
3442
3488
|
print(f" {asset['assetId']:<25} {asset.get('status', '?'):<10} scenes: {scenes_on} {asset.get('url', '')}")
|
|
3443
3489
|
else:
|
|
3444
3490
|
print(f"\n🧑 Digital-human segments: none (this render has no presenter)")
|
|
3491
|
+
from _scene_assets import scene_assets
|
|
3492
|
+
narration_texts = {}
|
|
3493
|
+
for scene in (dsl or {}).get("scenes", []):
|
|
3494
|
+
authored = (scene.get("audio") or {}).get("narration") or {}
|
|
3495
|
+
lines = extract_narration_lines(authored)
|
|
3496
|
+
if lines:
|
|
3497
|
+
narration_texts[scene["id"]] = "\n".join(lines[0])
|
|
3498
|
+
elif isinstance(authored.get("text"), str):
|
|
3499
|
+
narration_texts[scene["id"]] = authored["text"]
|
|
3500
|
+
print("__scene_assets__ " + json.dumps(scene_assets(render_plan, narration_texts), ensure_ascii=False))
|
|
3445
3501
|
|
|
3446
3502
|
return failed
|
|
3447
3503
|
|
|
@@ -3667,7 +3723,7 @@ def main():
|
|
|
3667
3723
|
LogPrint(f"💾 RenderPlan saved: {rp_output}", file=sys.stderr)
|
|
3668
3724
|
|
|
3669
3725
|
if args.resolve_only:
|
|
3670
|
-
if _print_resolve_only_summary(render_plan) > 0:
|
|
3726
|
+
if _print_resolve_only_summary(render_plan, dsl) > 0:
|
|
3671
3727
|
sys.exit(1)
|
|
3672
3728
|
return
|
|
3673
3729
|
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "render-video",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "473",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V42",
|
|
6
6
|
"skillDescription": "Final-render skill: loads a persisted RenderPlan by `job_id` and drives the Remotion engine to produce the final video.\n\nUse this skill as soon as the user mentions any of these intents (after assets are already prepared):\n- Render the video, composite the video, export the video\n- Turn the prepared assets into the final clip\n- Render with Remotion\n\nPrerequisite: assets must already be generated via `prepare_video_assets`. This skill never resolves or regenerates assets — pass it a `job_id` from a previous `prepare_video_assets` call.\n\n⚠️ Stop-and-confirm gate: never call this skill until the user has explicitly confirmed the assets prepared by `prepare_video_assets`. If those assets were prepared in the current turn and the user has not replied since, stop and ask instead of rendering."
|
|
7
7
|
}
|