@oh-story/dsh 0.1.4 → 0.1.5

package/lib/video-recap/skills/video-assemble/scripts/timeline.py

@@ -0,0 +1,248 @@
+"""Multi-track timeline model for the recap (backend-neutral, stdlib only).
+
+A `Timeline` is a small, serializable representation of the finished recap as a
+set of tracks — exactly like a cut-tool project:
+
+  - one **video** track: the source clip(s), each carrying its own *original
+    audio* with a per-clip volume automation (the ducking: a continuous low bed
+    under narration, held across short inter-sentence gaps, back up only at the
+    lead-in/out and genuine long gaps);
+  - one **narration** audio track: the placed TTS beats;
+  - an optional **bgm** audio track: a looped music bed with its own ducking;
+  - one **subtitle** (text) track: the narration lines.
+  - optional **image** tracks: local photo overlays with normalized center-origin,
+    Y-up transforms for editable JianYing export.
+
+The canonical ducking semantics live in `audio_automation.py`; ffmpeg
+(`assemble.py`) and this timeline model both derive their automation from that
+shared source. This model is emitted as `timeline.json` and consumed by the
+*optional* 剪映 exporter. The model itself knows nothing about ffmpeg or 剪映 —
+times are plain seconds and volumes are plain gains, so any backend can read it.
+"""
+
+import json
+import math
+from copy import deepcopy
+
+from audio_automation import fixed_ducking_keyframes as ducking_keyframes
+from audio_automation import release_ducking_keyframes
+
+SCHEMA_VERSION = 2
+
+
+def _ceil_time(value, digits=4):
+    """Round an interval end outward so serialization can never shorten media."""
+    scale = 10 ** digits
+    return math.ceil((float(value) * scale) - 1e-9) / scale
+
+
+def build_timeline(canvas, duration_s, video_clips, narration_segments,
+                   bgm=None, ducking=None, subtitle_segments=None,
+                   image_segments=None, resource_packages=None,
+                   style_presets=None, extra_tracks=None):
+    """Assemble a Timeline dict from resolved placement data.
+
+    canvas: {"width", "height", "fps"}
+    duration_s: total output length (seconds)
+    video_clips: ordered [{"source_path", "source_start", "source_end",
+                 "timeline_start", "timeline_end"}] (cut mode: one per clip;
+                 full mode: a single clip spanning the whole video).
+    narration_segments: placed beats [{"source_path", "timeline_start",
+                 "timeline_end", "text", "overlaps_speech", "gain"?}].
+    subtitle_segments: optional display-ready text cues [{"text", "timeline_start",
+                 "timeline_end"}]. When present, this is authoritative for the
+                 subtitle/text track; narration segment text remains raw editor metadata.
+    bgm: optional {"source_path", "volume", "ducking_volume"}.
+    ducking: {"idle", "speech", "quiet", "fade", "bridge"?} for the original-audio
+             automation; None disables original ducking (flat original). `bridge` holds
+             the duck across inter-beat gaps shorter than it (defaults to 2*fade).
+    """
+    windows = [(float(s["timeline_start"]), float(s["timeline_end"]))
+               for s in narration_segments
+               if s.get("timeline_end", 0) > s.get("timeline_start", 0)]
+    duck_windows = [
+        (
+            float(s["timeline_start"]),
+            max(float(s["timeline_end"]), float(s.get("source_duck_end", s["timeline_end"]))),
+            float((ducking or {}).get("speech" if s.get("overlaps_speech", True) else "quiet", 1.0)),
+            max(
+                float(s["timeline_end"]),
+                float(s.get("source_duck_end", s["timeline_end"])),
+                float(s.get(
+                    "source_restore_at",
+                    max(float(s["timeline_end"]), float(s.get("source_duck_end", s["timeline_end"])))
+                    + float((ducking or {}).get("fade", 0.0)),
+                )),
+            ),
+        )
+        for s in narration_segments
+        if s.get("timeline_end", 0) > s.get("timeline_start", 0)
+    ]
+
+    # --- video track: each clip carries its original audio + ducking automation
+    video_clip_objs = []
+    for c in video_clips:
+        ts, te = float(c["timeline_start"]), float(c["timeline_end"])
+        audio = {"role": "original", "volume_keyframes": []}
+        if ducking is not None:
+            audio["volume_keyframes"] = release_ducking_keyframes(
+                duck_windows, ducking["idle"], ducking["fade"], ts, te,
+                bridge=ducking.get("bridge"))
+            audio["base_gain"] = round(float(ducking["idle"]), 4)
+        else:
+            audio["base_gain"] = 1.0
+        video_clip = {
+            "source_path": c["source_path"],
+            "source_start": round(float(c["source_start"]), 4),
+            "source_end": round(float(c["source_end"]), 4),
+            "timeline_start": round(ts, 4),
+            "timeline_end": round(te, 4),
+            "audio": audio,
+        }
+        for key in (
+            "chroma", "compound", "flip", "green_background", "lut", "mask",
+            "opacity", "position", "reverse", "reverse_path", "rotation_degrees",
+            "scale", "speed", "transition",
+        ):
+            if key in c:
+                video_clip[key] = deepcopy(c[key])
+        video_clip_objs.append(video_clip)
+
+    tracks = [{"kind": "video", "name": "video", "clips": video_clip_objs}]
+
+    # --- narration track
+    narr_segs = []
+    for s in narration_segments:
+        ts, te = float(s["timeline_start"]), float(s["timeline_end"])
+        if te <= ts:
+            continue
+        narration = {
+            "source_path": s["source_path"],
+            "timeline_start": round(ts, 4),
+            "timeline_end": _ceil_time(te, 4),
+            "gain": round(float(s.get("gain", 1.0)), 4),
+            "text": s.get("text", ""),
+            "overlaps_speech": bool(s.get("overlaps_speech", True)),
+        }
+        for key in ("source_duck_end", "source_restore_at", "source_handoff_status", "source_entry_status"):
+            if key in s:
+                narration[key] = deepcopy(s[key])
+        if "speed" in s:
+            narration["speed"] = float(s["speed"])
+        narr_segs.append(narration)
+    if narr_segs:
+        tracks.append({"kind": "audio", "name": "narration", "role": "narration",
+                       "segments": narr_segs})
+
+    # --- bgm track (optional, looped, ducked under narration)
+    if bgm and bgm.get("source_path"):
+        base = float(bgm.get("volume", 0.18))
+        duck = float(bgm.get("ducking_volume", 0.10))
+        fade = float(bgm.get("fade") or (ducking or {}).get("fade", 0.25))
+        kfs = ducking_keyframes(windows, base, duck, fade, 0.0, duration_s,
+                                bridge=(ducking or {}).get("bridge"))
+        tracks.append({
+            "kind": "audio", "name": "bgm", "role": "bgm", "loop": True,
+            "segments": [{
+                "source_path": bgm["source_path"],
+                "timeline_start": 0.0,
+                "timeline_end": round(float(duration_s), 4),
+                "gain": round(base, 4),
+                "volume_keyframes": kfs,
+            }],
+        })
+
+    # --- subtitle (text) track
+    text_source = subtitle_segments if subtitle_segments is not None else narration_segments
+    text_segs = []
+    for s in text_source or []:
+        if not isinstance(s, dict) or not s.get("text"):
+            continue
+        try:
+            ts = float(s["timeline_start"])
+            te = float(s["timeline_end"])
+        except (KeyError, TypeError, ValueError):
+            continue
+        if te <= ts:
+            continue
+        text_segment = {
+            "text": s.get("text", ""),
+            "timeline_start": round(ts, 4),
+            "timeline_end": round(te, 4),
+        }
+        for key in ("flip", "opacity", "position", "rotation_degrees", "scale", "style", "style_id", "words"):
+            if key in s:
+                text_segment[key] = deepcopy(s[key])
+        text_segs.append(text_segment)
+    if text_segs:
+        tracks.append({"kind": "text", "name": "subtitle", "segments": text_segs})
+
+    # --- local image overlays (optional, timeline schema v2)
+    images = []
+    for segment in image_segments or []:
+        if not isinstance(segment, dict) or not segment.get("source_path"):
+            continue
+        try:
+            ts = float(segment["timeline_start"])
+            te = float(segment["timeline_end"])
+        except (KeyError, TypeError, ValueError):
+            continue
+        if te <= ts:
+            continue
+        scale = segment.get("scale") if isinstance(segment.get("scale"), dict) else {}
+        position = segment.get("position") if isinstance(segment.get("position"), dict) else {}
+        flip = segment.get("flip") if isinstance(segment.get("flip"), dict) else {}
+        image = {
+            "source_path": str(segment["source_path"]),
+            "timeline_start": round(ts, 4),
+            "timeline_end": round(te, 4),
+            "opacity": round(max(0.0, min(1.0, float(segment.get("opacity", 1.0)))), 4),
+            "rotation_degrees": round(float(segment.get("rotation_degrees", 0.0)), 4),
+            "scale": {
+                "x": round(float(scale.get("x", 1.0)), 4),
+                "y": round(float(scale.get("y", 1.0)), 4),
+            },
+            "position": {
+                "x": round(float(position.get("x", 0.0)), 4),
+                "y": round(float(position.get("y", 0.0)), 4),
+            },
+            "flip": {
+                "horizontal": bool(flip.get("horizontal", False)),
+                "vertical": bool(flip.get("vertical", False)),
+            },
+        }
+        for key in ("lut", "mask", "speed", "transition"):
+            if key in segment:
+                image[key] = deepcopy(segment[key])
+        images.append(image)
+    if images:
+        tracks.append({"kind": "image", "name": "image", "segments": images})
+
+    for track in extra_tracks or []:
+        if not isinstance(track, dict):
+            raise TypeError("extra_tracks entries must be objects")
+        tracks.append(deepcopy(track))
+
+    timeline = {
+        "schema_version": SCHEMA_VERSION,
+        "canvas": {"width": int(canvas["width"]), "height": int(canvas["height"]),
+                   "fps": float(canvas.get("fps", 30))},
+        "duration": round(float(duration_s), 4),
+        "tracks": tracks,
+    }
+    if resource_packages:
+        timeline["resource_packages"] = deepcopy(resource_packages)
+    if style_presets:
+        timeline["style_presets"] = deepcopy(style_presets)
+    return timeline
+
+
+def save_timeline(timeline, path):
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(timeline, f, ensure_ascii=False, indent=2)
+    return path
+
+
+def load_timeline(path):
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)

package/lib/video-recap/skills/video-assemble/scripts/timeline_emit.py

@@ -0,0 +1,89 @@
+"""Backend-neutral timeline emission for the video-assemble skill."""
+
+from pathlib import Path
+
+from audio_mix import _seg_place_window
+from lib import CONFIG, log
+from media import _build_video_clips, _probe_canvas
+from source_subtitles import _combined_subtitle_entries
+from timeline import build_timeline, save_timeline
+
+def _timeline_subtitle_segments(tts_segments, work_dir, duration_s):
+    """Display-ready subtitle cues for timeline/export text tracks.
+
+    The narration audio track keeps raw semantic text for editor reference; this
+    payload mirrors SRT/ASS display policy, including terminal-punctuation cleanup
+    and original-dialogue gap subtitles when configured.
+    """
+    return [
+        {
+            "text": entry["text"],
+            "timeline_start": float(entry["start"]),
+            "timeline_end": float(entry["end"]),
+        }
+        for entry in _combined_subtitle_entries(tts_segments, work_dir, duration_s)
+    ]
+
+
+def _emit_timeline(input_video, tts_segments, work_dir, duration_s, has_bgm):
+    """Build and persist the backend-neutral multi-track timeline.json."""
+    canvas = _probe_canvas(input_video)
+    video_clips = _build_video_clips(input_video, work_dir, duration_s)
+    narration_segments = []
+    for seg in tts_segments:
+        if not isinstance(seg, dict):
+            continue
+        s, e = _seg_place_window(seg)
+        if e <= s:
+            continue
+        narration_item = {
+            # JianYing must consume the exact WAV written into narration.wav. In
+            # particular, a tempo-adjusted beat cannot reference its longer pre-fit
+            # source or the editor will trim its final words at timeline_end.
+            "source_path": seg.get("placed_audio_path") or seg.get("audio_path", ""),
+            "timeline_start": s, "timeline_end": e,
+            "text": seg.get("narration", ""),
+            "overlaps_speech": seg.get("overlaps_speech", True),
+            "gain": 1.0,
+        }
+        for key in ("source_duck_end", "source_restore_at", "source_handoff_status", "source_entry_status"):
+            if key in seg:
+                narration_item[key] = seg[key]
+        narration_segments.append(narration_item)
+    fade = CONFIG.get("duck_fade_seconds", 0.3)
+    bgm = None
+    if has_bgm:
+        bgm = {"source_path": CONFIG.get("bgm_path", ""),
+               "volume": CONFIG.get("bgm_volume", 0.18),
+               "ducking_volume": CONFIG.get("bgm_ducking_volume", 0.10),
+               "fade": fade}
+    # carry ducking automation whenever ducking is on at all; even under sidechain
+    # mode the draft gets editable volume keyframes (ffmpeg stays the canonical mix)
+    ducking = None
+    if CONFIG.get("ducking_mode", "fixed") != "none":
+        ducking = {"idle": CONFIG.get("idle_orig_volume", 1.0),
+                   "speech": CONFIG.get("speech_ducking_volume", 0.2),
+                   "quiet": CONFIG.get("zone_ducking_volume", 0.12),
+                   "fade": fade,
+                   "bridge": CONFIG.get("duck_bridge_seconds", 1.5)}
+    subtitle_segments = _timeline_subtitle_segments(tts_segments, work_dir, duration_s)
+    timeline = build_timeline(canvas, duration_s, video_clips,
+                              narration_segments, bgm=bgm, ducking=ducking,
+                              subtitle_segments=subtitle_segments)
+    degraded = [
+        {
+            "source_path": clip.get("source_path"),
+            "reason": clip.get("provenance_reason") or "unknown",
+        }
+        for clip in video_clips
+        if clip.get("provenance_degraded")
+    ]
+    if degraded:
+        timeline["provenance"] = {"degraded": True, "degraded_clips": degraded}
+        log(f"  ⚠️ 时间线 provenance 降级: {degraded[0]['reason']} ({len(degraded)} clip)")
+    else:
+        timeline["provenance"] = {"degraded": False}
+    out = Path(work_dir) / "timeline.json"
+    save_timeline(timeline, out)
+    log(f"时间线模型: {out} ({len(timeline['tracks'])} 轨)")
+    return timeline