@remixmate/cli 0.13.5 → 0.13.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/manifest.json +2 -2
- package/package.json +1 -1
- package/skills/gen-script/SKILL.md +4 -0
- package/skills/gen-script/scripts/gen_script.py +114 -4
- package/skills/gen-script/version.json +1 -1
- package/skills/prepare-video-assets/SKILL.md +2 -0
- package/skills/prepare-video-assets/version.json +1 -1
- package/skills/render-video/scripts/_continuous_narration.py +11 -3
- package/skills/render-video/scripts/_narration_alignment.py +128 -0
- package/skills/render-video/scripts/_scene_assets.py +39 -0
- package/skills/render-video/scripts/render_video.py +77 -8
- package/skills/render-video/version.json +1 -1
- package/skills/template-registry/version.json +1 -1
- package/skills/template-registry/video_dsl/runtime/timeline_compiler.py +8 -4
package/dist/manifest.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remixmate/cli",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.7",
|
|
4
4
|
"description": "AI media generation skills for Claude Code / Codex — 12 skills covering image, video, voice, digital human, web screenshot, web recording, script, template registry, rendering, Jianying export, and video deconstruction.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -32,6 +32,8 @@ Turns a user-supplied **topic** into a **Video DSL v1alpha1** JSON describing wh
|
|
|
32
32
|
|
|
33
33
|
- **Video DSL**: a declarative description language for videos — describes the final structure, not the orchestration calls.
|
|
34
34
|
- **AssetRef**: an asset reference declaring which images / audio / video clips are needed; the downstream Render Skill is responsible for producing them.
|
|
35
|
+
|
|
36
|
+
Templates may declare `capabilities.mediaBinding.sceneMode=per-item`: the returned skeleton then contains one scene per input media item, with durations allocated from the complete sequence's existing timing model. Keep those scenes, media references and timing values; fill each scene's narration separately. Captions describe the visuals and are not automatically narration. Continuous TTS can synthesize adjacent scenes together and cut the result back into their separate audio assets; it does not merge the script's scenes.
|
|
35
37
|
- **Scene**: the unit cell of the video timeline.
|
|
36
38
|
- **Image `payload.model`**: must match the `gen-image` skill and the backend `ModelGenImgDTO`. **Only** values from the LiteLLM `model` allowlist below may be used. When editing narration or prompts you **must not** rewrite this field.
|
|
37
39
|
|
|
@@ -508,6 +510,8 @@ python3 <SkillDir>/scripts/gen_script.py \
|
|
|
508
510
|
|
|
509
511
|
Values written by `--background-preset`, `--background-asset`, `--payload`, `--ending-payload`, `--avatar-id` and `--motion` are listed in `meta.lockedScenePaths` (scene-relative paths such as `customPayload.<object>.<field>`, `animationHints.motion`). When the agent later saves its own `dsl_json` in merge mode, ab-agent restores those paths from this skeleton after merging, so a model-written `customPayload` object cannot wipe a card choice. Do not hand-write these keys in `dsl_json`; pass the flag. Templates that author their own scenes (`meta.authoredScenes`) are not merged: there ab-agent writes each locked value that the skeleton holds on every scene (with one value) onto every submitted scene, and adds the skeleton assets those values reference (`asset-background`, `asset-avatar`) when the submitted `assets` lacks them.
|
|
510
512
|
|
|
513
|
+
For image-to-video replicate cards, map **转场效果 / Transition effect** to both `--payload sequence.defaultEffect=<id>` and `--payload sequence.overrideItemEffects=true`. Use the id in parentheses (`fade`, `wipe-left`, `slide-left`, `none`), preserving the chosen preset and all typography. **跟随样片 / Follow example (`follow`)** omits both overrides; `follow` is not a valid effect. Map **转场时长 / Transition duration** to `--payload sequence.transitionMs=<milliseconds>` (e.g. `0.6 秒(600)` → `600`); omit when following the sample or using `none`. These payload keys are locked into every generated scene, so do not re-author them in a saved DSL.
|
|
514
|
+
|
|
511
515
|
## DSL generation principles
|
|
512
516
|
|
|
513
517
|
Apply the following principles when producing the DSL:
|
|
@@ -1129,8 +1129,8 @@ def _sequence_floor_frames(sequence: dict, preset_timing: dict, text: str, fps:
|
|
|
1129
1129
|
return math.floor(min_sec * fps + 0.5)
|
|
1130
1130
|
|
|
1131
1131
|
|
|
1132
|
-
def
|
|
1133
|
-
|
|
1132
|
+
def _fit_images_hold_frames(custom_payload: dict, model: dict | None, fps: int = 30,
|
|
1133
|
+
headline: str = "") -> list[int]:
|
|
1134
1134
|
|
|
1135
1135
|
|
|
1136
1136
|
|
|
@@ -1183,8 +1183,46 @@ def _estimate_fit_images_duration(custom_payload: dict, model: dict | None, fps:
|
|
|
1183
1183
|
if rate is None:
|
|
1184
1184
|
raise ValueError(f"fit-images with a floor requires durationModel.{key}.floorMinVideoRate")
|
|
1185
1185
|
floor_frames = min(floor_frames, sum(math.floor(f / rate) for f in frames))
|
|
1186
|
-
|
|
1187
|
-
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
target = max(natural, floor_frames)
|
|
1190
|
+
original = list(frames)
|
|
1191
|
+
images = [item.get("type") != "video" for item in sequence["items"]]
|
|
1192
|
+
|
|
1193
|
+
def grow(eligible, caps):
|
|
1194
|
+
need = target - sum(frames)
|
|
1195
|
+
room = [max(0, caps[i] - frames[i]) if eligible[i] else 0 for i in range(len(frames))]
|
|
1196
|
+
capacity = sum(room)
|
|
1197
|
+
if need > 0 and capacity > 0:
|
|
1198
|
+
take = min(need, capacity)
|
|
1199
|
+
for i, r in enumerate(room):
|
|
1200
|
+
frames[i] += math.floor(r * take / capacity)
|
|
1201
|
+
|
|
1202
|
+
def uncapped(eligible):
|
|
1203
|
+
need = target - sum(frames)
|
|
1204
|
+
base = sum(f for i, f in enumerate(frames) if eligible[i])
|
|
1205
|
+
if need > 0 and base > 0:
|
|
1206
|
+
for i, f in enumerate(list(frames)):
|
|
1207
|
+
if eligible[i]:
|
|
1208
|
+
frames[i] += math.floor(f * need / base)
|
|
1209
|
+
|
|
1210
|
+
videos = [not image for image in images]
|
|
1211
|
+
grow(images, [math.floor(f * 2.5 + 0.5) for f in original])
|
|
1212
|
+
if any(videos):
|
|
1213
|
+
rate = timing.get("floorMinVideoRate")
|
|
1214
|
+
if not rate:
|
|
1215
|
+
raise ValueError(f"fit-images with a floor requires durationModel.{key}.floorMinVideoRate")
|
|
1216
|
+
grow(videos, [math.floor(f / rate) for f in original])
|
|
1217
|
+
uncapped(images)
|
|
1218
|
+
uncapped(videos)
|
|
1219
|
+
frames[-1] += target - sum(frames)
|
|
1220
|
+
return frames
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def _estimate_fit_images_duration(custom_payload: dict, model: dict | None, fps: int = 30,
|
|
1224
|
+
headline: str = "") -> float:
|
|
1225
|
+
return sum(_fit_images_hold_frames(custom_payload, model, fps, headline)) / fps
|
|
1188
1226
|
|
|
1189
1227
|
|
|
1190
1228
|
def _probe_sequence_video_duration(url: str) -> float | None:
|
|
@@ -1535,6 +1573,71 @@ def _build_carousel_caption_dsl(
|
|
|
1535
1573
|
for path in narration_locks:
|
|
1536
1574
|
_lock_scene_path(dsl, path)
|
|
1537
1575
|
|
|
1576
|
+
scene_mode = binding.get("sceneMode", "single")
|
|
1577
|
+
if scene_mode not in ("single", "per-item"):
|
|
1578
|
+
raise ValueError("mediaBinding.sceneMode must be single or per-item")
|
|
1579
|
+
if scene_mode == "per-item":
|
|
1580
|
+
if duration_strategy != "fit-images":
|
|
1581
|
+
raise ValueError("mediaBinding.sceneMode=per-item requires durationStrategy=fit-images")
|
|
1582
|
+
holds = _fit_images_hold_frames(custom_payload, duration_model, headline=headline)
|
|
1583
|
+
dsl["assets"] = [a for a in assets if a["source"] != "gen-voice"]
|
|
1584
|
+
dsl["scenes"] = []
|
|
1585
|
+
|
|
1586
|
+
|
|
1587
|
+
scene_defaults = [(path.split("."), value) for path, value in binding.get("sceneDefaults", {}).items()]
|
|
1588
|
+
for index, (item, frames) in enumerate(zip(container[items_parts[-1]], holds)):
|
|
1589
|
+
current = deepcopy(scene)
|
|
1590
|
+
current["id"] = f"scene-{index + 1:02d}"
|
|
1591
|
+
current["purpose"] = "opening" if index == 0 else "point"
|
|
1592
|
+
current["duration"] = frames / 30
|
|
1593
|
+
payload = current["customPayload"]
|
|
1594
|
+
_write_payload(payload, scene_defaults)
|
|
1595
|
+
_write_payload(payload, payload_writes or [])
|
|
1596
|
+
pinned = {**item, "holdSec": frames / 30}
|
|
1597
|
+
hold_schema = (_schema_at(template_config, items_parts).get("items", {}).get("properties", {}).get("holdSec", {}))
|
|
1598
|
+
if not hold_schema.get("minimum", 0) <= frames / 30 <= hold_schema.get("maximum", float("inf")):
|
|
1599
|
+
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
pinned.pop("holdSec", None)
|
|
1603
|
+
_write_payload(payload, [(items_parts, [pinned])])
|
|
1604
|
+
default_hold_schema = _schema_at(template_config, container_parts + ["defaultHoldSec"])
|
|
1605
|
+
if default_hold_schema and default_hold_schema.get("minimum", 0) <= frames / 30 <= default_hold_schema.get("maximum", float("inf")):
|
|
1606
|
+
_write_payload(payload, [(container_parts + ["defaultHoldSec"], frames / 30)])
|
|
1607
|
+
|
|
1608
|
+
|
|
1609
|
+
if _schema_at(template_config, container_parts + ["minSec"]):
|
|
1610
|
+
minimum = _schema_at(template_config, container_parts + ["minSec"]).get("minimum", 0)
|
|
1611
|
+
if frames / 30 >= minimum:
|
|
1612
|
+
_write_payload(payload, [(container_parts + ["minSec"], frames / 30)])
|
|
1613
|
+
else:
|
|
1614
|
+
_get_path(payload, container_parts).pop("minSec", None)
|
|
1615
|
+
if narration_enabled:
|
|
1616
|
+
if "minDurationSec" in payload:
|
|
1617
|
+
payload["minDurationSec"] = frames / 30
|
|
1618
|
+
ref = f"narration-{current['id']}"
|
|
1619
|
+
current["audio"]["narration"].update(
|
|
1620
|
+
assetRef=ref,
|
|
1621
|
+
text=_localize_label(
|
|
1622
|
+
output_language, "narration_opening", label="", topic=_clip(topic, _PLACEHOLDER_TOPIC_CHARS),
|
|
1623
|
+
duration=frames / 30,
|
|
1624
|
+
budget=_budget_suffix(frames / 30, output_language, narration_speed, None),
|
|
1625
|
+
),
|
|
1626
|
+
)
|
|
1627
|
+
dsl["assets"].append({"assetId": ref, "type": "audio", "source": "gen-voice",
|
|
1628
|
+
"status": "planned", "payload": {"voiceId": resolved_voice_id}})
|
|
1629
|
+
dsl["scenes"].append(current)
|
|
1630
|
+
|
|
1631
|
+
|
|
1632
|
+
if container_parts:
|
|
1633
|
+
dsl["meta"]["mergeScenePaths"] = ["customPayload." + ".".join(container_parts)]
|
|
1634
|
+
if _schema_at(template_config, container_parts + ["defaultHoldSec"]):
|
|
1635
|
+
_lock_scene_path(dsl, "customPayload." + ".".join(container_parts + ["defaultHoldSec"]))
|
|
1636
|
+
for parts, _ in scene_defaults:
|
|
1637
|
+
_lock_scene_path(dsl, "customPayload." + ".".join(parts))
|
|
1638
|
+
if _schema_at(template_config, container_parts + ["minSec"]):
|
|
1639
|
+
_lock_scene_path(dsl, "customPayload." + ".".join(container_parts + ["minSec"]))
|
|
1640
|
+
|
|
1538
1641
|
return dsl
|
|
1539
1642
|
|
|
1540
1643
|
|
|
@@ -1918,6 +2021,13 @@ def _apply_scene_payload(
|
|
|
1918
2021
|
raise ValueError("--payload / --motion need --template-id: the allowed keys come from the template.")
|
|
1919
2022
|
|
|
1920
2023
|
writes = _parse_payload_specs(template_config, payload_specs)
|
|
2024
|
+
binding = _resolve_media_binding(template_config)
|
|
2025
|
+
if binding.get("sceneMode") == "per-item":
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
item_path = binding["itemsPath"].split(".")
|
|
2029
|
+
allocated = (item_path, item_path[:-1] + ["minSec"])
|
|
2030
|
+
writes = [(parts, value) for parts, value in writes if parts not in allocated]
|
|
1921
2031
|
if motion:
|
|
1922
2032
|
allowed = ("auto", "static", "kenburns-in", "kenburns-out", "pan-left", "pan-right", "float")
|
|
1923
2033
|
if motion not in allowed:
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "gen-script",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "474",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V25",
|
|
6
6
|
"skillDescription": "Video-script generation skill. Turns a user-supplied topic into a structured Video DSL (JSON) that describes the full video — scene structure, asset requirements, and narrative flow.\n\nUse this skill as soon as the user mentions any of these intents:\n- Write a video script, generate a video script, plan a video, write storyboards\n- Create a short video, plan video content, generate a Video DSL\n- Turn a topic into a video structure / video content plan\n\nEven when the user does not say \"generate the DSL\", use this skill whenever they want to turn a topic into a structured video plan.\n\n⚠️ Stop-and-confirm gate: after this skill returns a DSL, show the full script and wait for the user's explicit confirmation. Never call `prepare_video_assets` in the same turn."
|
|
7
7
|
}
|
|
@@ -112,6 +112,8 @@ This skill is the **first half** of a two-phase user-confirmation flow. After it
|
|
|
112
112
|
|
|
113
113
|
Read the `job_id` from the stdout line `📦 render job jobId: N` and remember it; asset URLs come from the `🔊 TTS audio:` and `🖼 Image assets:` sections. Show images inline (``) and audio as links (`[🔊 listen](http://cdn.../x.mp3)`) so the user can actually review them. The recommended summary format:
|
|
114
114
|
|
|
115
|
+
When `__scene_assets__` is present, the web UI renders the full scene preview from that structured timeline: all visuals, scene times and each scene's audio slice. Give a brief overview/status and request confirmation; do not duplicate the preview or copy URLs into prose. For CLI clients without this UI, render every referenced visual and the corresponding audio per scene from the marker. With older outputs, use the fallback table below, including **all** visuals in each scene rather than one representative image. Scene count is independent of the number of TTS synthesis calls. Visual captions must not automatically become narration.
|
|
116
|
+
|
|
115
117
|
```markdown
|
|
116
118
|
## Render confirmation
|
|
117
119
|
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "prepare-video-assets",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "476",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V8",
|
|
6
6
|
"skillDescription": "Asset-preparation skill: resolves and generates every asset (image / audio / video) referenced by a Video DSL, persists a RenderPlan to the database, and returns a `job_id` for the subsequent `render_video` call.\n\nUse this skill as soon as the user mentions any of these intents:\n- Generate / prepare video assets, resolve assets, render-ready\n- \"Make me a video about X\" (the agent calls gen_script → prepare_video_assets → render_video)\n- Regenerate one asset (image / audio) for a specific scene\n\nNext step: after the user confirms the resolved assets, call `render_video` with the `job_id` returned by this skill.\n\n⚠️ Stop-and-confirm gate: this skill runs only after the user has confirmed the script, and after it returns you must show the resolved assets and wait for the user's explicit confirmation. Never call `render_video` in the same turn."
|
|
7
7
|
}
|
|
@@ -50,6 +50,15 @@ _NATURAL_PAUSE_SEC = 0.45
|
|
|
50
50
|
|
|
51
51
|
_MIN_BOUNDARY_PAUSE_RATIO = 0.75
|
|
52
52
|
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
_MIN_BOUNDARY_PAUSE_SEC = 0.6
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _min_boundary_ms(pause_sec: float) -> float:
|
|
60
|
+
return max(_MIN_BOUNDARY_PAUSE_SEC, pause_sec * _MIN_BOUNDARY_PAUSE_RATIO) * 1000
|
|
61
|
+
|
|
53
62
|
_SILENCE_NOISE_DB = -40
|
|
54
63
|
_SILENCE_MIN_SEC = 0.25
|
|
55
64
|
|
|
@@ -282,7 +291,7 @@ def cuts_from_silences(
|
|
|
282
291
|
return None
|
|
283
292
|
longest = sorted(inner, key=lambda s: s[1] - s[0], reverse=True)[:need]
|
|
284
293
|
longest.sort(key=lambda s: s[0])
|
|
285
|
-
min_len = pause_sec
|
|
294
|
+
min_len = _min_boundary_ms(pause_sec)
|
|
286
295
|
if any(end - start < min_len for start, end in longest):
|
|
287
296
|
return None
|
|
288
297
|
cuts = [(start + end) / 2 for start, end in longest]
|
|
@@ -308,8 +317,7 @@ def _durations_plausible(cuts: list[float], texts: list[str], total_ms: float, p
|
|
|
308
317
|
def _cuts_on_boundary_pauses(cuts: list[float], silences: list[tuple[float, float]], pause_sec: float) -> bool:
|
|
309
318
|
|
|
310
319
|
|
|
311
|
-
|
|
312
|
-
min_len = pause_sec * _MIN_BOUNDARY_PAUSE_RATIO * 1000
|
|
320
|
+
min_len = _min_boundary_ms(pause_sec)
|
|
313
321
|
tol = 100.0
|
|
314
322
|
return all(
|
|
315
323
|
any(end - start >= min_len and start - tol <= c <= end + tol for start, end in silences)
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
import unicodedata
|
|
13
|
+
|
|
14
|
+
from video_dsl.runtime.timeline_compiler import assign_sentence_indices, split_narration_sentences
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _normalized(text: str) -> str:
|
|
18
|
+
text = re.sub(r"<#\s*\d+(?:\.\d+)?\s*#>", "", text)
|
|
19
|
+
return "".join(c for c in unicodedata.normalize("NFKC", text).casefold() if c.isalnum())
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _number(value) -> bool:
|
|
23
|
+
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def align_narration_lines(
|
|
27
|
+
lines: list[str], subtitles: list[dict], duration_ms: float, speech_span_ms=None,
|
|
28
|
+
) -> tuple[list[dict], str]:
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
normalized = [_normalized(line) for line in lines]
|
|
36
|
+
weights = [max(len(text), 1) for text in normalized]
|
|
37
|
+
start_ms, end_ms = 0.0, float(duration_ms)
|
|
38
|
+
if (isinstance(speech_span_ms, list) and len(speech_span_ms) == 2
|
|
39
|
+
and all(_number(v) for v in speech_span_ms)
|
|
40
|
+
and 0 <= speech_span_ms[0] < speech_span_ms[1] <= duration_ms):
|
|
41
|
+
start_ms, end_ms = map(float, speech_span_ms)
|
|
42
|
+
|
|
43
|
+
anchors = []
|
|
44
|
+
offset = 0
|
|
45
|
+
previous_end = 0.0
|
|
46
|
+
for sub in subtitles:
|
|
47
|
+
if not isinstance(sub, dict) or not isinstance(sub.get("text"), str):
|
|
48
|
+
anchors = []
|
|
49
|
+
break
|
|
50
|
+
text = _normalized(sub["text"])
|
|
51
|
+
if not text:
|
|
52
|
+
continue
|
|
53
|
+
begin, end = sub.get("timeBegin"), sub.get("timeEnd")
|
|
54
|
+
if not (_number(begin) and _number(end) and previous_end <= begin < end <= duration_ms):
|
|
55
|
+
anchors = []
|
|
56
|
+
break
|
|
57
|
+
anchors.append((offset, offset + len(text), float(begin), float(end), text))
|
|
58
|
+
offset += len(text)
|
|
59
|
+
previous_end = end
|
|
60
|
+
|
|
61
|
+
matches = (anchors and all(normalized)
|
|
62
|
+
and "".join(normalized) == "".join(a[4] for a in anchors))
|
|
63
|
+
result = []
|
|
64
|
+
interpolated = False
|
|
65
|
+
if matches:
|
|
66
|
+
offset = 0
|
|
67
|
+
anchor_index = 0
|
|
68
|
+
for line, text in zip(lines, normalized):
|
|
69
|
+
line_end = offset + len(text)
|
|
70
|
+
while anchors[anchor_index][1] <= offset:
|
|
71
|
+
anchor_index += 1
|
|
72
|
+
first = anchors[anchor_index]
|
|
73
|
+
while anchors[anchor_index][1] < line_end:
|
|
74
|
+
anchor_index += 1
|
|
75
|
+
last = anchors[anchor_index]
|
|
76
|
+
begin = first[2] + (first[3] - first[2]) * (offset - first[0]) / (first[1] - first[0])
|
|
77
|
+
end = last[2] + (last[3] - last[2]) * (line_end - last[0]) / (last[1] - last[0])
|
|
78
|
+
interpolated |= offset != first[0] or line_end != last[1]
|
|
79
|
+
result.append({"text": line, "timeBegin": begin, "timeEnd": end})
|
|
80
|
+
offset = line_end
|
|
81
|
+
return result, "tts-interpolated" if interpolated else "tts-boundaries"
|
|
82
|
+
|
|
83
|
+
total = sum(weights)
|
|
84
|
+
offset = 0
|
|
85
|
+
for line, weight in zip(lines, weights):
|
|
86
|
+
begin = start_ms + (end_ms - start_ms) * offset / total
|
|
87
|
+
offset += weight
|
|
88
|
+
end = start_ms + (end_ms - start_ms) * offset / total
|
|
89
|
+
result.append({"text": line, "timeBegin": begin, "timeEnd": end})
|
|
90
|
+
return result, "estimated"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def restore_narration_sentences(
|
|
94
|
+
text: str, subtitles: list[dict], duration_ms: float, speech_span_ms=None,
|
|
95
|
+
) -> tuple[list[dict], str]:
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
sentences = split_narration_sentences(text)
|
|
103
|
+
if not subtitles:
|
|
104
|
+
if len(sentences) > 1:
|
|
105
|
+
return align_narration_lines(sentences, [], duration_ms, speech_span_ms)
|
|
106
|
+
return [], "tts-unchanged"
|
|
107
|
+
|
|
108
|
+
result = []
|
|
109
|
+
method = "tts-unchanged"
|
|
110
|
+
for sub in subtitles:
|
|
111
|
+
parts = split_narration_sentences(sub.get("text") or "")
|
|
112
|
+
if len(parts) <= 1:
|
|
113
|
+
result.append(sub)
|
|
114
|
+
continue
|
|
115
|
+
split, timing = align_narration_lines(parts, [sub], duration_ms, speech_span_ms)
|
|
116
|
+
result.extend(split)
|
|
117
|
+
if method != "estimated":
|
|
118
|
+
method = "estimated" if timing == "estimated" else "tts-interpolated"
|
|
119
|
+
|
|
120
|
+
if len(sentences) > 1:
|
|
121
|
+
if _normalized(text) != _normalized("".join(s.get("text") or "" for s in subtitles)):
|
|
122
|
+
return result, "tts-text-mismatch"
|
|
123
|
+
indices = assign_sentence_indices([s.get("text") or "" for s in result])
|
|
124
|
+
if len(set(indices)) != len(sentences):
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
return align_narration_lines(sentences, subtitles, duration_ms, speech_span_ms)
|
|
128
|
+
return result, method
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def scene_assets(plan: dict, narration_texts: dict[str, str] | None = None) -> list[dict]:
|
|
6
|
+
assets = {a.get("assetId"): a for a in plan.get("resolvedAssets", [])}
|
|
7
|
+
fps = (plan.get("renderConfig") or {}).get("fps", 30)
|
|
8
|
+
out = []
|
|
9
|
+
|
|
10
|
+
def refs(value):
|
|
11
|
+
if isinstance(value, dict):
|
|
12
|
+
for key, child in value.items():
|
|
13
|
+
if isinstance(child, str) and key.lower().endswith(("assetref", "assetid")) and child in assets:
|
|
14
|
+
yield child
|
|
15
|
+
else:
|
|
16
|
+
yield from refs(child)
|
|
17
|
+
elif isinstance(value, list):
|
|
18
|
+
for child in value:
|
|
19
|
+
yield from refs(child)
|
|
20
|
+
|
|
21
|
+
for entry in plan.get("timeline", []):
|
|
22
|
+
narration = "".join(s.get("text", "") for s in entry.get("subtitleSegments", []))
|
|
23
|
+
narration = (narration_texts or {}).get(entry.get("sceneId"), narration)
|
|
24
|
+
for ref in dict.fromkeys(refs(entry.get("props") or {})):
|
|
25
|
+
asset = assets[ref]
|
|
26
|
+
if asset.get("type") not in ("image", "video", "audio") or asset.get("status") != "generated" or not asset.get("url"):
|
|
27
|
+
continue
|
|
28
|
+
item = {
|
|
29
|
+
"type": asset["type"], "url": asset["url"], "label": ref,
|
|
30
|
+
"sceneId": entry["sceneId"],
|
|
31
|
+
"sceneStartSec": entry.get("startFrame", 0) / fps,
|
|
32
|
+
"sceneEndSec": entry.get("endFrame", 0) / fps,
|
|
33
|
+
"narrationText": narration,
|
|
34
|
+
}
|
|
35
|
+
if asset.get("duration"):
|
|
36
|
+
|
|
37
|
+
item["durationSec"] = asset["duration"] / (1000 if asset["type"] == "audio" else 1)
|
|
38
|
+
out.append(item)
|
|
39
|
+
return out
|
|
@@ -27,6 +27,7 @@ import argparse
|
|
|
27
27
|
import atexit
|
|
28
28
|
import builtins
|
|
29
29
|
import json
|
|
30
|
+
import math
|
|
30
31
|
import os
|
|
31
32
|
import re
|
|
32
33
|
import signal
|
|
@@ -879,6 +880,9 @@ def build_render_plan(dsl: dict, binding: dict) -> dict:
|
|
|
879
880
|
"retryCount": 0,
|
|
880
881
|
"maxRetries": 3,
|
|
881
882
|
}
|
|
883
|
+
narration_text = (asset.get("payload") or {}).get("text")
|
|
884
|
+
if asset.get("type") == "audio" and isinstance(narration_text, str) and narration_text.strip():
|
|
885
|
+
assets_by_id[asset["assetId"]]["narrationText"] = narration_text
|
|
882
886
|
|
|
883
887
|
|
|
884
888
|
|
|
@@ -921,7 +925,7 @@ def build_render_plan(dsl: dict, binding: dict) -> dict:
|
|
|
921
925
|
for idx, scene in enumerate(dsl.get("scenes", [])):
|
|
922
926
|
scene_id = scene.get("id") or scene.get("sceneId") or f"scene-{idx:03d}"
|
|
923
927
|
duration = scene.get("duration", 5)
|
|
924
|
-
duration_frames =
|
|
928
|
+
duration_frames = math.floor(duration * fps + 0.5)
|
|
925
929
|
|
|
926
930
|
scene_binding = binding_map.get(scene_id, {})
|
|
927
931
|
|
|
@@ -1396,9 +1400,12 @@ def _resolve_single_asset(
|
|
|
1396
1400
|
|
|
1397
1401
|
|
|
1398
1402
|
src_payload = asset_with_payload.get("payload", {}) or {}
|
|
1403
|
+
if source == "gen-voice" and isinstance(src_payload.get("text"), str):
|
|
1404
|
+
asset["narrationText"] = src_payload["text"]
|
|
1399
1405
|
if isinstance(src_payload.get("narrationItems"), list):
|
|
1400
1406
|
asset["narrationLineCount"] = len(src_payload["narrationItems"])
|
|
1401
1407
|
asset["narrationIntroLines"] = int(src_payload.get("narrationIntroLines") or 0)
|
|
1408
|
+
asset["narrationLines"] = list(src_payload["narrationItems"])
|
|
1402
1409
|
LogPrint(f" ✅ {asset['assetId']} generated", file=sys.stderr)
|
|
1403
1410
|
return True
|
|
1404
1411
|
else:
|
|
@@ -1626,9 +1633,12 @@ def resolve_narration_run(
|
|
|
1626
1633
|
if per_scene_subs[k]:
|
|
1627
1634
|
asset["ttsSubtitles"] = per_scene_subs[k]
|
|
1628
1635
|
payload = payloads.get(c["assetId"]) or {}
|
|
1636
|
+
if isinstance(payload.get("text"), str):
|
|
1637
|
+
asset["narrationText"] = payload["text"]
|
|
1629
1638
|
if isinstance(payload.get("narrationItems"), list):
|
|
1630
1639
|
asset["narrationLineCount"] = len(payload["narrationItems"])
|
|
1631
1640
|
asset["narrationIntroLines"] = int(payload.get("narrationIntroLines") or 0)
|
|
1641
|
+
asset["narrationLines"] = list(payload["narrationItems"])
|
|
1632
1642
|
log(f"continuous narration: {len(run)} scenes in one TTS call, {total_ms / 1000:.1f}s, cut by {method}")
|
|
1633
1643
|
return True
|
|
1634
1644
|
|
|
@@ -1914,9 +1924,8 @@ def _auto_highlight_map(
|
|
|
1914
1924
|
return
|
|
1915
1925
|
|
|
1916
1926
|
props = entry.get("props") or {}
|
|
1917
|
-
|
|
1918
|
-
if
|
|
1919
|
-
return
|
|
1927
|
+
nested = isinstance(props.get("templateData"), dict)
|
|
1928
|
+
tdata = props["templateData"] if nested else props
|
|
1920
1929
|
if tdata.get("highlightMap"):
|
|
1921
1930
|
return
|
|
1922
1931
|
|
|
@@ -1948,7 +1957,8 @@ def _auto_highlight_map(
|
|
|
1948
1957
|
|
|
1949
1958
|
highlight_map = {str(intro_lines + i): i for i in range(item_count) if item_count > 0}
|
|
1950
1959
|
tdata["highlightMap"] = highlight_map
|
|
1951
|
-
|
|
1960
|
+
if nested:
|
|
1961
|
+
props["templateData"] = tdata
|
|
1952
1962
|
entry["props"] = props
|
|
1953
1963
|
|
|
1954
1964
|
render_plan.setdefault("logs", []).append({
|
|
@@ -1985,6 +1995,55 @@ def _template_default_tail_pad_frames(tpl: dict, fps: int) -> int | None:
|
|
|
1985
1995
|
return int(round(float(value) * fps))
|
|
1986
1996
|
|
|
1987
1997
|
|
|
1998
|
+
def _structured_narration_subtitles(entry: dict, asset: dict, render_plan: dict) -> list[dict]:
|
|
1999
|
+
|
|
2000
|
+
from _narration_alignment import align_narration_lines, restore_narration_sentences
|
|
2001
|
+
|
|
2002
|
+
subtitles = asset.get("ttsSubtitles") or []
|
|
2003
|
+
line_count = asset.get("narrationLineCount")
|
|
2004
|
+
|
|
2005
|
+
old_subs = entry.get("subtitleSegments") or []
|
|
2006
|
+
at_sec = entry.get("subtitleAtSec") or []
|
|
2007
|
+
if len(at_sec) == len(old_subs) and any(isinstance(v, (int, float)) for v in at_sec):
|
|
2008
|
+
return []
|
|
2009
|
+
|
|
2010
|
+
if not isinstance(line_count, int) or isinstance(line_count, bool) or line_count <= 0:
|
|
2011
|
+
text = asset.get("narrationText") or ""
|
|
2012
|
+
subtitles, method = restore_narration_sentences(
|
|
2013
|
+
text, subtitles, asset["duration"], asset.get("speechSpanMs"),
|
|
2014
|
+
)
|
|
2015
|
+
if method == "tts-unchanged":
|
|
2016
|
+
return subtitles
|
|
2017
|
+
message = f"Restored narration sentence boundaries using {method} timing"
|
|
2018
|
+
else:
|
|
2019
|
+
lines = asset.get("narrationLines")
|
|
2020
|
+
if not isinstance(lines, list) or len(lines) != line_count:
|
|
2021
|
+
lines = [s.get("text") for s in old_subs] if len(old_subs) == line_count else []
|
|
2022
|
+
if len(lines) != line_count or any(not isinstance(v, str) or not v.strip() for v in lines):
|
|
2023
|
+
method = "authored-lines-unavailable"
|
|
2024
|
+
message = "Authored narration lines unavailable; cannot restore per-line timing"
|
|
2025
|
+
else:
|
|
2026
|
+
|
|
2027
|
+
asset["narrationLines"] = list(lines)
|
|
2028
|
+
subtitles, method = align_narration_lines(
|
|
2029
|
+
lines, subtitles, asset["duration"], asset.get("speechSpanMs"),
|
|
2030
|
+
)
|
|
2031
|
+
message = f"Preserved {line_count} narration lines using {method} timing"
|
|
2032
|
+
|
|
2033
|
+
logs = render_plan.setdefault("logs", [])
|
|
2034
|
+
if not any(l.get("phase") == "narration-align" and l.get("sceneId") == entry.get("sceneId")
|
|
2035
|
+
and l.get("method") == method for l in logs):
|
|
2036
|
+
logs.append({
|
|
2037
|
+
"phase": "narration-align",
|
|
2038
|
+
"sceneId": entry.get("sceneId"),
|
|
2039
|
+
"method": method,
|
|
2040
|
+
"level": "info" if method == "tts-boundaries" else "warn",
|
|
2041
|
+
"message": message,
|
|
2042
|
+
"timestamp": now_iso(),
|
|
2043
|
+
})
|
|
2044
|
+
return subtitles
|
|
2045
|
+
|
|
2046
|
+
|
|
1988
2047
|
def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
1989
2048
|
|
|
1990
2049
|
|
|
@@ -2050,7 +2109,7 @@ def adjust_timeline_to_audio(render_plan: dict) -> dict:
|
|
|
2050
2109
|
entry["durationFrames"] = new_dur
|
|
2051
2110
|
adjusted += 1
|
|
2052
2111
|
|
|
2053
|
-
tts_subs =
|
|
2112
|
+
tts_subs = _structured_narration_subtitles(entry, narration_asset, render_plan)
|
|
2054
2113
|
if tts_subs:
|
|
2055
2114
|
|
|
2056
2115
|
|
|
@@ -3383,7 +3442,7 @@ def _persist_render_plan(render_plan: dict, args, private_token: str, dsl: dict
|
|
|
3383
3442
|
args.save_job = False
|
|
3384
3443
|
|
|
3385
3444
|
|
|
3386
|
-
def _print_resolve_only_summary(render_plan: dict) -> int:
|
|
3445
|
+
def _print_resolve_only_summary(render_plan: dict, dsl: dict | None = None) -> int:
|
|
3387
3446
|
|
|
3388
3447
|
generated = sum(1 for a in render_plan["resolvedAssets"] if a["status"] == "generated")
|
|
3389
3448
|
failed = sum(1 for a in render_plan["resolvedAssets"] if a["status"] == "failed")
|
|
@@ -3442,6 +3501,16 @@ def _print_resolve_only_summary(render_plan: dict) -> int:
|
|
|
3442
3501
|
print(f" {asset['assetId']:<25} {asset.get('status', '?'):<10} scenes: {scenes_on} {asset.get('url', '')}")
|
|
3443
3502
|
else:
|
|
3444
3503
|
print(f"\n🧑 Digital-human segments: none (this render has no presenter)")
|
|
3504
|
+
from _scene_assets import scene_assets
|
|
3505
|
+
narration_texts = {}
|
|
3506
|
+
for scene in (dsl or {}).get("scenes", []):
|
|
3507
|
+
authored = (scene.get("audio") or {}).get("narration") or {}
|
|
3508
|
+
lines = extract_narration_lines(authored)
|
|
3509
|
+
if lines:
|
|
3510
|
+
narration_texts[scene["id"]] = "\n".join(lines[0])
|
|
3511
|
+
elif isinstance(authored.get("text"), str):
|
|
3512
|
+
narration_texts[scene["id"]] = authored["text"]
|
|
3513
|
+
print("__scene_assets__ " + json.dumps(scene_assets(render_plan, narration_texts), ensure_ascii=False))
|
|
3445
3514
|
|
|
3446
3515
|
return failed
|
|
3447
3516
|
|
|
@@ -3667,7 +3736,7 @@ def main():
|
|
|
3667
3736
|
LogPrint(f"💾 RenderPlan saved: {rp_output}", file=sys.stderr)
|
|
3668
3737
|
|
|
3669
3738
|
if args.resolve_only:
|
|
3670
|
-
if _print_resolve_only_summary(render_plan) > 0:
|
|
3739
|
+
if _print_resolve_only_summary(render_plan, dsl) > 0:
|
|
3671
3740
|
sys.exit(1)
|
|
3672
3741
|
return
|
|
3673
3742
|
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "render-video",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "473",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V43",
|
|
6
6
|
"skillDescription": "Final-render skill: loads a persisted RenderPlan by `job_id` and drives the Remotion engine to produce the final video.\n\nUse this skill as soon as the user mentions any of these intents (after assets are already prepared):\n- Render the video, composite the video, export the video\n- Turn the prepared assets into the final clip\n- Render with Remotion\n\nPrerequisite: assets must already be generated via `prepare_video_assets`. This skill never resolves or regenerates assets — pass it a `job_id` from a previous `prepare_video_assets` call.\n\n⚠️ Stop-and-confirm gate: never call this skill until the user has explicitly confirmed the assets prepared by `prepare_video_assets`. If those assets were prepared in the current turn and the user has not replied since, stop and ask instead of rendering."
|
|
7
7
|
}
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"skillName": "template-registry",
|
|
3
3
|
"repoName": "agent-skill-media-maker",
|
|
4
4
|
"skillId": "475",
|
|
5
|
-
"version": "
|
|
5
|
+
"version": "V31",
|
|
6
6
|
"skillDescription": "Video-template registry skill. Stores every video-template definition and lists the available templates (templateId / name / aspect ratio / style tags).\n\nUse this skill as soon as the user mentions any of these intents:\n- View available templates / list every template\n\nNote: DSL→TemplateBinding is no longer a separate exposed step — once prepare_video_assets receives a template_id it builds the binding internally."
|
|
7
7
|
}
|
|
@@ -213,15 +213,19 @@ def assign_sentence_indices(raw_texts: list[str]) -> list[int]:
|
|
|
213
213
|
return indices
|
|
214
214
|
|
|
215
215
|
|
|
216
|
+
def split_narration_sentences(text: str) -> list[str]:
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
return [part.strip(" \t\r") for part in _PRIMARY_BREAK.split(text.strip()) if part.strip()] if text else []
|
|
221
|
+
|
|
222
|
+
|
|
216
223
|
def count_narration_sentences(text: str) -> int:
|
|
217
224
|
|
|
218
225
|
|
|
219
226
|
|
|
220
227
|
|
|
221
|
-
|
|
222
|
-
return 0
|
|
223
|
-
chunks = [s for s in _PRIMARY_BREAK.split(text.strip()) if s.strip()]
|
|
224
|
-
return len(chunks)
|
|
228
|
+
return len(split_narration_sentences(text))
|
|
225
229
|
|
|
226
230
|
|
|
227
231
|
def _plain_len(text: str) -> int:
|