@remixmate/cli 0.13.4 → 0.13.6

package/skills/render-video/scripts/_continuous_narration.py

@@ -0,0 +1,476 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import unicodedata
+from typing import Optional
+
+MODE_CONTINUOUS = "continuous"
+MODE_PER_SCENE = "perScene"
+
+
+
+DEFAULT_SCENE_PAUSE_SEC = 0.8
+
+
+
+MAX_RUN_CHARS = 1500
+
+
+
+
+ESTIMATE_WINDOW_SEC = 2.0
+_ESTIMATE_WINDOW_RATIO = 0.2
+
+
+_NATURAL_PAUSE_SEC = 0.45
+
+
+_MIN_BOUNDARY_PAUSE_RATIO = 0.75
+
+
+
+
+_MIN_BOUNDARY_PAUSE_SEC = 0.6
+
+
+def _min_boundary_ms(pause_sec: float) -> float:
+    return max(_MIN_BOUNDARY_PAUSE_SEC, pause_sec * _MIN_BOUNDARY_PAUSE_RATIO) * 1000
+
+_SILENCE_NOISE_DB = -40
+_SILENCE_MIN_SEC = 0.25
+
+PAUSE_TAG_RE = re.compile(r"<#\s*(\d+(?:\.\d+)?)\s*#>")
+_LEADING_TAG_RE = re.compile(r"^\s*<#\s*(\d+(?:\.\d+)?)\s*#>\s*")
+_TRAILING_TAG_RE = re.compile(r"\s*<#\s*(\d+(?:\.\d+)?)\s*#>\s*$")
+
+
+
+
+def _pause_value(raw) -> Optional[float]:
+    if isinstance(raw, bool) or not isinstance(raw, (int, float)):
+        return None
+
+    return min(max(float(raw), 0.01), 99.99) if raw > 0 else None
+
+
+def resolve_config(template: dict, override: Optional[dict]) -> dict:
+
+
+
+
+    declared = ((template or {}).get("capabilities") or {}).get("narration") or {}
+    override = override or {}
+    mode = MODE_PER_SCENE
+    for source in (declared, override):
+        if source.get("mode") in (MODE_CONTINUOUS, MODE_PER_SCENE):
+            mode = source["mode"]
+    pause = DEFAULT_SCENE_PAUSE_SEC
+    for source in (declared, override):
+        value = _pause_value(source.get("scenePauseSec"))
+        if value is not None:
+            pause = value
+    return {"mode": mode, "scenePauseSec": pause}
+
+
+
+
+def group_runs(candidates: list[Optional[dict]], max_chars: int = MAX_RUN_CHARS) -> list[list[dict]]:
+
+
+
+
+
+
+    runs: list[list[dict]] = []
+    current: list[dict] = []
+    chars = 0
+
+    def flush() -> None:
+        if len(current) >= 2:
+            runs.append(list(current))
+        current.clear()
+
+    for cand in candidates:
+        if cand is None:
+            flush()
+            chars = 0
+            continue
+        size = len(cand["text"])
+        if current and (
+            cand.get("break")
+            or cand["voiceKey"] != current[-1]["voiceKey"]
+            or chars + size > max_chars
+        ):
+            flush()
+            chars = 0
+        current.append(cand)
+        chars += size
+    flush()
+    return runs
+
+
+
+
+def format_pause(sec: float) -> str:
+    return f"{sec:.2f}".rstrip("0").rstrip(".")
+
+
+def compose_run_text(texts: list[str], pause_sec: float) -> tuple[str, list[tuple[int, int]]]:
+
+
+
+
+    parts: list[str] = []
+    for i, text in enumerate(texts):
+        text = text.strip()
+        if i > 0:
+            text = _LEADING_TAG_RE.sub("", text, count=1)
+        if i < len(texts) - 1:
+            text = _TRAILING_TAG_RE.sub("", text, count=1)
+        parts.append(text)
+
+    pauses: list[float] = []
+    for i in range(len(texts) - 1):
+        candidates = [pause_sec]
+        tail = _TRAILING_TAG_RE.search(texts[i].strip())
+        head = _LEADING_TAG_RE.search(texts[i + 1].strip())
+        for m in (tail, head):
+            if m:
+                candidates.append(_pause_value(float(m.group(1))) or 0)
+        pauses.append(max(candidates))
+
+    out = ""
+    ranges: list[tuple[int, int]] = []
+    for i, part in enumerate(parts):
+        if i > 0:
+            out += f"\n<#{format_pause(pauses[i - 1])}#>\n"
+        start = len(out)
+        out += part
+        ranges.append((start, len(out)))
+    return out, ranges
+
+
+def normalize(text: str) -> str:
+
+    text = PAUSE_TAG_RE.sub("", text or "")
+    return "".join(
+        ch for ch in text
+        if not ch.isspace() and not unicodedata.category(ch).startswith(("P", "S"))
+    )
+
+
+def estimate_durations(texts: list[str], total_ms: float, pause_sec: float) -> list[float]:
+
+
+
+
+    counts = [max(len(normalize(t)), 1) for t in texts]
+    boundaries = len(texts) - 1
+    gap_ms = (pause_sec + _NATURAL_PAUSE_SEC) * 1000
+    speech_ms = total_ms - boundaries * gap_ms
+    if speech_ms <= 0:
+        gap_ms, speech_ms = 0.0, total_ms
+    per_char = speech_ms / sum(counts)
+    durations: list[float] = []
+    for k, count in enumerate(counts):
+        edges = (k > 0) + (k < boundaries)
+        durations.append(count * per_char + edges * gap_ms / 2)
+    return durations
+
+
+
+
+def _sub_mid(subs: list[dict], i: int) -> float:
+    return (float(subs[i]["timeEnd"]) + float(subs[i + 1]["timeBegin"])) / 2
+
+
+def cuts_from_text_offsets(subs: list[dict], ranges: list[tuple[int, int]]) -> Optional[list[float]]:
+
+
+    if len(subs) < 2 or not all(
+        isinstance(s.get("textBegin"), int) and isinstance(s.get("textEnd"), int) for s in subs
+    ):
+        return None
+    cuts: list[float] = []
+    for k in range(1, len(ranges)):
+        prev_end, next_start = ranges[k - 1][1], ranges[k][0]
+        hit = next(
+            (
+                i for i in range(len(subs) - 1)
+                if subs[i]["textEnd"] <= next_start
+                and subs[i + 1]["textBegin"] >= prev_end
+                and subs[i + 1]["textBegin"] >= subs[i]["textEnd"]
+            ),
+            None,
+        )
+        if hit is None:
+            return None
+        cuts.append(_sub_mid(subs, hit))
+    return cuts if _monotonic(cuts) else None
+
+
+def speakable_ranges(run_text: str, ranges: list[tuple[int, int]]) -> list[tuple[int, int]]:
+
+
+
+
+
+
+    keep = [ch not in "\r\n" for ch in run_text]
+    for m in PAUSE_TAG_RE.finditer(run_text):
+        for i in range(m.start(), m.end()):
+            keep[i] = False
+    prefix = [0]
+    for k in keep:
+        prefix.append(prefix[-1] + k)
+    return [(prefix[a], prefix[b]) for a, b in ranges]
+
+
+def cuts_from_text_alignment(subs: list[dict], texts: list[str]) -> Optional[list[float]]:
+
+
+    if len(subs) < 2:
+        return None
+    sub_acc: list[int] = []
+    acc = 0
+    for s in subs:
+        acc += len(normalize(s.get("text") or ""))
+        sub_acc.append(acc)
+    if acc != sum(len(normalize(t)) for t in texts):
+        return None
+    cuts: list[float] = []
+    scene_acc = 0
+    for k in range(len(texts) - 1):
+        scene_acc += len(normalize(texts[k]))
+        try:
+            i = sub_acc.index(scene_acc)
+        except ValueError:
+            return None
+        if i >= len(subs) - 1:
+            return None
+        cuts.append(_sub_mid(subs, i))
+    return cuts if _monotonic(cuts) else None
+
+
+def cuts_from_silences(
+    silences: list[tuple[float, float]],
+    texts: list[str],
+    total_ms: float,
+    pause_sec: float,
+) -> Optional[list[float]]:
+
+    need = len(texts) - 1
+    if need <= 0:
+        return []
+
+    inner = [s for s in silences if 0 < s[0] and s[1] < total_ms]
+    if len(inner) < need:
+        return None
+    longest = sorted(inner, key=lambda s: s[1] - s[0], reverse=True)[:need]
+    longest.sort(key=lambda s: s[0])
+    min_len = _min_boundary_ms(pause_sec)
+    if any(end - start < min_len for start, end in longest):
+        return None
+    cuts = [(start + end) / 2 for start, end in longest]
+    if not _durations_plausible(cuts, texts, total_ms, pause_sec):
+        return None
+    return cuts if _monotonic(cuts) else None
+
+
+def _monotonic(cuts: list[float]) -> bool:
+    return all(b > a for a, b in zip(cuts, cuts[1:])) and all(c > 0 for c in cuts)
+
+
+def _durations_plausible(cuts: list[float], texts: list[str], total_ms: float, pause_sec: float) -> bool:
+
+    edges = [0.0, *cuts, float(total_ms)]
+    for k, predicted in enumerate(estimate_durations(texts, total_ms, pause_sec)):
+        actual = edges[k + 1] - edges[k]
+        if abs(actual - predicted) > max(ESTIMATE_WINDOW_SEC * 1000, predicted * _ESTIMATE_WINDOW_RATIO):
+            return False
+    return True
+
+
+def _cuts_on_boundary_pauses(cuts: list[float], silences: list[tuple[float, float]], pause_sec: float) -> bool:
+
+
+    min_len = _min_boundary_ms(pause_sec)
+    tol = 100.0
+    return all(
+        any(end - start >= min_len and start - tol <= c <= end + tol for start, end in silences)
+        for c in cuts
+    )
+
+
+def find_cuts(
+    texts: list[str],
+    ranges: list[tuple[int, int]],
+    subs: list[dict],
+    silences: Optional[list[tuple[float, float]]],
+    total_ms: float,
+    pause_sec: float,
+    run_text: Optional[str] = None,
+) -> tuple[Optional[list[float]], str]:
+
+
+
+
+    def trusted(cuts: Optional[list[float]]) -> bool:
+        return (
+            cuts is not None
+            and _durations_plausible(cuts, texts, total_ms, pause_sec)
+            and (silences is None or _cuts_on_boundary_pauses(cuts, silences, pause_sec))
+        )
+
+    candidates = [speakable_ranges(run_text, ranges)] if run_text is not None else []
+    for rs in [*candidates, ranges]:
+        cuts = cuts_from_text_offsets(subs, rs)
+        if trusted(cuts):
+            return cuts, "subtitle-offsets"
+    cuts = cuts_from_text_alignment(subs, texts)
+    if trusted(cuts):
+        return cuts, "subtitle-text"
+    if silences is not None:
+        cuts = cuts_from_silences(silences, texts, total_ms, pause_sec)
+        if cuts is not None:
+            return cuts, "silence"
+    return None, "none"
+
+
+
+
+def slice_bounds(cuts: list[float], total_ms: float) -> list[tuple[float, float]]:
+    edges = [0.0, *cuts, float(total_ms)]
+    return list(zip(edges, edges[1:]))
+
+
+def speech_span(bounds: tuple[float, float], silences: list[tuple[float, float]]) -> tuple[int, int]:
+
+    start, end = bounds
+    lo, hi = start, end
+    for s_start, s_end in silences:
+        if s_start <= lo < s_end:
+            lo = min(s_end, end)
+        if s_start < hi <= s_end:
+            hi = max(s_start, lo)
+    return int(round(lo - start)), int(round(hi - start))
+
+
+def split_subtitles(subs: list[dict], bounds: list[tuple[float, float]]) -> list[list[dict]]:
+
+    per_scene: list[list[dict]] = [[] for _ in bounds]
+    for s in subs:
+        mid = (float(s.get("timeBegin", 0)) + float(s.get("timeEnd", 0))) / 2
+        for k, (start, end) in enumerate(bounds):
+            if start <= mid < end or (k == len(bounds) - 1 and mid >= start):
+                length = end - start
+                per_scene[k].append({
+                    "text": s.get("text", ""),
+                    "timeBegin": int(round(min(max(float(s["timeBegin"]) - start, 0), length))),
+                    "timeEnd": int(round(min(max(float(s["timeEnd"]) - start, 0), length))),
+                })
+                break
+    return per_scene
+
+
+
+
+def silencedetect_command(src: str, ffmpeg: str = "ffmpeg") -> list[str]:
+    return [
+        ffmpeg, "-hide_banner", "-nostats", "-i", src,
+        "-af", f"silencedetect=noise={_SILENCE_NOISE_DB}dB:d={_SILENCE_MIN_SEC}",
+        "-f", "null", "-",
+    ]
+
+
+_SILENCE_START_RE = re.compile(r"silence_start:\s*(-?\d+(?:\.\d+)?)")
+_SILENCE_END_RE = re.compile(r"silence_end:\s*(-?\d+(?:\.\d+)?)")
+
+
+def parse_silencedetect(stderr: str, total_ms: float) -> list[tuple[float, float]]:
+
+    silences: list[tuple[float, float]] = []
+    start: Optional[float] = None
+    for line in (stderr or "").splitlines():
+        m = _SILENCE_START_RE.search(line)
+        if m:
+            start = max(float(m.group(1)) * 1000, 0.0)
+            continue
+        m = _SILENCE_END_RE.search(line)
+        if m and start is not None:
+            silences.append((start, float(m.group(1)) * 1000))
+            start = None
+    if start is not None:
+        silences.append((start, float(total_ms)))
+    return silences
+
+
+def detect_silences(src: str, total_ms: float, ffmpeg: str, timeout: int = 120) -> list[tuple[float, float]]:
+    try:
+        result = subprocess.run(
+            silencedetect_command(src, ffmpeg),
+            capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise RuntimeError(f"ffmpeg silencedetect timed out after {timeout}s") from exc
+    if result.returncode != 0:
+        raise RuntimeError(f"ffmpeg silencedetect failed: {(result.stderr or '').strip()[-500:]}")
+    return parse_silencedetect(result.stderr, total_ms)
+
+
+def slice_command(src: str, start_ms: float, end_ms: Optional[float], out_path: str, ffmpeg: str = "ffmpeg") -> list[str]:
+
+    cmd = [ffmpeg, "-y", "-loglevel", "error", "-i", src, "-ss", f"{start_ms / 1000:.3f}"]
+    if end_ms is not None:
+        cmd += ["-to", f"{end_ms / 1000:.3f}"]
+    cmd += ["-c:a", "libmp3lame", "-b:a", "128k", out_path]
+    return cmd
+
+
+def slice_audio(
+    src: str,
+    bounds: list[tuple[float, float]],
+    out_dir: str,
+    names: list[str],
+    ffmpeg: str,
+    timeout: int = 120,
+) -> list[str]:
+
+    os.makedirs(out_dir, exist_ok=True)
+    paths: list[str] = []
+    for k, ((start, end), name) in enumerate(zip(bounds, names)):
+        safe = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in name)
+        out_path = os.path.join(out_dir, f"narration-{safe}.mp3")
+
+        cmd = slice_command(src, start, None if k == len(bounds) - 1 else end, out_path, ffmpeg)
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL)
+        except subprocess.TimeoutExpired as exc:
+            raise RuntimeError(f"ffmpeg slice timed out after {timeout}s") from exc
+        if result.returncode != 0 or not os.path.exists(out_path):
+            raise RuntimeError(f"ffmpeg slice failed: {(result.stderr or '').strip()[:500]}")
+        paths.append(out_path)
+    return paths

package/skills/render-video/scripts/_narration_alignment.py

@@ -0,0 +1,88 @@
+
+
+
+
+
+
+
+from __future__ import annotations
+
+import math
+import re
+import unicodedata
+
+
+def _normalized(text: str) -> str:
+    text = re.sub(r"<#\s*\d+(?:\.\d+)?\s*#>", "", text)
+    return "".join(c for c in unicodedata.normalize("NFKC", text).casefold() if c.isalnum())
+
+
+def _number(value) -> bool:
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
+
+
+def align_narration_lines(
+    lines: list[str], subtitles: list[dict], duration_ms: float, speech_span_ms=None,
+) -> tuple[list[dict], str]:
+
+
+
+
+
+
+    normalized = [_normalized(line) for line in lines]
+    weights = [max(len(text), 1) for text in normalized]
+    start_ms, end_ms = 0.0, float(duration_ms)
+    if (isinstance(speech_span_ms, list) and len(speech_span_ms) == 2
+            and all(_number(v) for v in speech_span_ms)
+            and 0 <= speech_span_ms[0] < speech_span_ms[1] <= duration_ms):
+        start_ms, end_ms = map(float, speech_span_ms)
+
+    anchors = []
+    offset = 0
+    previous_end = 0.0
+    for sub in subtitles:
+        if not isinstance(sub, dict) or not isinstance(sub.get("text"), str):
+            anchors = []
+            break
+        text = _normalized(sub["text"])
+        if not text:
+            continue
+        begin, end = sub.get("timeBegin"), sub.get("timeEnd")
+        if not (_number(begin) and _number(end) and previous_end <= begin < end <= duration_ms):
+            anchors = []
+            break
+        anchors.append((offset, offset + len(text), float(begin), float(end), text))
+        offset += len(text)
+        previous_end = end
+
+    matches = (anchors and all(normalized)
+               and "".join(normalized) == "".join(a[4] for a in anchors))
+    result = []
+    interpolated = False
+    if matches:
+        offset = 0
+        anchor_index = 0
+        for line, text in zip(lines, normalized):
+            line_end = offset + len(text)
+            while anchors[anchor_index][1] <= offset:
+                anchor_index += 1
+            first = anchors[anchor_index]
+            while anchors[anchor_index][1] < line_end:
+                anchor_index += 1
+            last = anchors[anchor_index]
+            begin = first[2] + (first[3] - first[2]) * (offset - first[0]) / (first[1] - first[0])
+            end = last[2] + (last[3] - last[2]) * (line_end - last[0]) / (last[1] - last[0])
+            interpolated |= offset != first[0] or line_end != last[1]
+            result.append({"text": line, "timeBegin": begin, "timeEnd": end})
+            offset = line_end
+        return result, "tts-interpolated" if interpolated else "tts-boundaries"
+
+    total = sum(weights)
+    offset = 0
+    for line, weight in zip(lines, weights):
+        begin = start_ms + (end_ms - start_ms) * offset / total
+        offset += weight
+        end = start_ms + (end_ms - start_ms) * offset / total
+        result.append({"text": line, "timeBegin": begin, "timeEnd": end})
+    return result, "estimated"

package/skills/render-video/scripts/_scene_assets.py

@@ -0,0 +1,39 @@
+
+from __future__ import annotations
+
+
+def scene_assets(plan: dict, narration_texts: dict[str, str] | None = None) -> list[dict]:
+    assets = {a.get("assetId"): a for a in plan.get("resolvedAssets", [])}
+    fps = (plan.get("renderConfig") or {}).get("fps", 30)
+    out = []
+
+    def refs(value):
+        if isinstance(value, dict):
+            for key, child in value.items():
+                if isinstance(child, str) and key.lower().endswith(("assetref", "assetid")) and child in assets:
+                    yield child
+                else:
+                    yield from refs(child)
+        elif isinstance(value, list):
+            for child in value:
+                yield from refs(child)
+
+    for entry in plan.get("timeline", []):
+        narration = "".join(s.get("text", "") for s in entry.get("subtitleSegments", []))
+        narration = (narration_texts or {}).get(entry.get("sceneId"), narration)
+        for ref in dict.fromkeys(refs(entry.get("props") or {})):
+            asset = assets[ref]
+            if asset.get("type") not in ("image", "video", "audio") or asset.get("status") != "generated" or not asset.get("url"):
+                continue
+            item = {
+                "type": asset["type"], "url": asset["url"], "label": ref,
+                "sceneId": entry["sceneId"],
+                "sceneStartSec": entry.get("startFrame", 0) / fps,
+                "sceneEndSec": entry.get("endFrame", 0) / fps,
+                "narrationText": narration,
+            }
+            if asset.get("duration"):
+
+                item["durationSec"] = asset["duration"] / (1000 if asset["type"] == "audio" else 1)
+            out.append(item)
+    return out