@oh-story/dsh 0.1.4 → 0.1.6

package/lib/video-recap/skills/video-understanding/scripts/timeline_fusion.py

@@ -0,0 +1,199 @@
+"""Fuse scenes, ASR, and quiet windows for narration planning."""
+
+import importlib.util
+
+
+from pathlib import Path
+
+from lib import CONFIG
+
+
+from agent_text import _overlap_seconds, _recommended_char_budget
+from narration_lint import _validate_narration_budget
+
+try:
+    from deslop_qc import analyze_deslop_qc
+except ModuleNotFoundError:
+    _deslop_qc_path = Path(__file__).with_name("deslop_qc.py")
+    _deslop_qc_spec = importlib.util.spec_from_file_location(
+        "deslop_qc", _deslop_qc_path
+    )
+    if _deslop_qc_spec is None or _deslop_qc_spec.loader is None:
+        raise
+    _deslop_qc_module = importlib.util.module_from_spec(_deslop_qc_spec)
+    _deslop_qc_spec.loader.exec_module(_deslop_qc_module)
+    analyze_deslop_qc = _deslop_qc_module.analyze_deslop_qc
+
+
+def _align_narration_to_quiet(narration, scenes_analysis, silence_periods):
+    """Recompute overlaps_speech from real quiet windows; keep the agent's timing.
+
+    The dense continuous-bed design places narration ON the pictured beat over a
+    ducked original bed, so we no longer relocate segments into silence gaps. The
+    old shift moved voiceover up to 3s off the moment it was written for and could
+    silently blank a squeezed segment; both fought the design intent. We now only
+    correct the overlaps_speech flag that the ducking stage consumes, leaving the
+    agent's start/end (and text) intact.
+    """
+    if not silence_periods:
+        for n in narration:
+            n["overlaps_speech"] = True
+        return _validate_narration_budget(narration, scenes_analysis)
+
+    quiet_windows = [
+        qp for qp in silence_periods if _silence_window_is_usable_quiet(qp)
+    ]
+    quiet_ratio_min = float(CONFIG.get("quiet_overlap_min_ratio", 0.8) or 0.8)
+
+    # Run budget/dedup FIRST, then set the flag on the final (possibly merged) spans,
+    # so a dedup-merged beat's overlaps_speech reflects its extended timing, not its
+    # original shorter span (which the ducking stage in assemble.py would mis-duck).
+    aligned = _validate_narration_budget(narration, scenes_analysis)
+    for n in aligned:
+        try:
+            seg_start = float(n["start"])
+            seg_end = float(n["end"])
+        except (KeyError, TypeError, ValueError):
+            n["overlaps_speech"] = True
+            continue
+        seg_dur = max(0.0, seg_end - seg_start)
+        quiet_overlap = _quiet_overlap_seconds(seg_start, seg_end, quiet_windows)
+        n["overlaps_speech"] = quiet_overlap < max(0.3, seg_dur * quiet_ratio_min)
+
+    return aligned
+
+
+def _scene_asr_lines(asr_result, scene):
+    lines = []
+    for seg in asr_result or []:
+        try:
+            start = float(seg.get("start", 0))
+            end = float(seg.get("end", start))
+        except (TypeError, ValueError):
+            continue
+        if scene["start"] < end and scene["end"] > start:
+            text = str(seg.get("text", "")).strip()
+            if text:
+                lines.append(f"    [{start:.1f}-{end:.1f}] {text}")
+    return lines
+
+
+def _quiet_windows_for_scene(silence_periods, scene):
+    windows = []
+    for qp in silence_periods or []:
+        if not _silence_window_is_usable_quiet(qp):
+            continue
+        if qp["start"] < scene["end"] and qp["end"] > scene["start"]:
+            start = max(qp["start"], scene["start"])
+            end = min(qp["end"], scene["end"])
+            if end > start:
+                windows.append((start, end))
+    return windows
+
+
+def _silence_window_is_usable_quiet(qp):
+    if not isinstance(qp, dict):
+        return False
+    reason = str(qp.get("has_speech_reason", "")).lower()
+    granularity = str(qp.get("asr_granularity", "")).lower()
+    if reason in {
+        "coarse_asr_overlap_ignored",
+        "asr_overlap_low_confidence_quiet",
+        "coarse_asr_no_overlap",
+    }:
+        return True
+    if granularity == "coarse_grid" and qp.get("has_speech") is True:
+        return True
+    return not qp.get("has_speech", False)
+
+
+def _quiet_overlap_seconds(start, end, quiet_windows):
+    overlap_seconds = 0.0
+    for qw in quiet_windows:
+        try:
+            if isinstance(qw, dict):
+                q_start = float(qw.get("start", 0))
+                q_end = float(qw.get("end", q_start))
+            else:
+                q_start, q_end = qw
+                q_start = float(q_start)
+                q_end = float(q_end)
+        except (TypeError, ValueError):
+            continue
+        overlap_seconds += _overlap_seconds(start, end, q_start, q_end)
+    return overlap_seconds
+
+
+def _build_timeline_fusion(scenes, asr_segments, silence_periods):
+    """Fuse VLM scenes, ASR dialogue and quiet narration slots on one timeline."""
+    fusion = []
+    quiet_windows = [
+        w for w in silence_periods or [] if _silence_window_is_usable_quiet(w)
+    ]
+    for scene in scenes or []:
+        try:
+            start = float(scene.get("start", 0))
+            end = float(scene.get("end", start))
+        except (TypeError, ValueError):
+            continue
+        dialogue_segments = []
+        dialogue_overlap = 0.0
+        for seg in asr_segments or []:
+            if not isinstance(seg, dict):
+                continue
+            try:
+                seg_start = float(seg.get("start", 0))
+                seg_end = float(seg.get("end", seg_start))
+            except (TypeError, ValueError):
+                continue
+            overlap = _overlap_seconds(start, end, seg_start, seg_end)
+            if overlap <= 0:
+                continue
+            text = str(seg.get("text", "")).strip()
+            dialogue_overlap += overlap
+            dialogue_segments.append(
+                {
+                    "start": round(seg_start, 2),
+                    "end": round(seg_end, 2),
+                    "overlap_seconds": round(overlap, 2),
+                    "text": text,
+                }
+            )
+
+        narration_slots = []
+        for window in quiet_windows:
+            try:
+                w_start = float(window.get("start", 0))
+                w_end = float(window.get("end", w_start))
+            except (TypeError, ValueError):
+                continue
+            overlap = _overlap_seconds(start, end, w_start, w_end)
+            if overlap <= 0:
+                continue
+            slot_start = max(start, w_start)
+            slot_end = min(end, w_end)
+            narration_slots.append(
+                {
+                    "start": round(slot_start, 2),
+                    "end": round(slot_end, 2),
+                    "duration": round(slot_end - slot_start, 2),
+                    "char_budget": _recommended_char_budget(slot_start, slot_end),
+                }
+            )
+
+        fusion.append(
+            {
+                "scene_id": scene.get("scene_id"),
+                "time_range": [round(start, 2), round(end, 2)],
+                "visual_description": scene.get("description", ""),
+                "depth_analysis": scene.get("depth_analysis", ""),
+                "frame_facts": scene.get("frame_facts", {}),
+                "dialogue_segments": dialogue_segments,
+                "dialogue_overlap_seconds": round(dialogue_overlap, 2),
+                "narration_slots": narration_slots,
+                "recommended_mode": "quiet-slot"
+                if narration_slots and dialogue_overlap < (end - start) * 0.4
+                else "ducked-bed",
+            }
+        )
+    return fusion

package/lib/video-recap/skills/video-understanding/scripts/understand.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+"""CLI entrypoint for the self-contained video-understanding skill."""
+
+from understanding_runner import main
+
+__all__ = ["main"]
+
+if __name__ == "__main__":
+    main()

package/lib/video-recap/skills/video-understanding/scripts/understanding_brief.py

@@ -0,0 +1,216 @@
+"""Format research context and rebuild briefs from cached analysis."""
+
+import json
+
+from pathlib import Path
+
+from lib import CONFIG, log
+
+
+from detect import detect_speech_boundary_anchors
+
+
+from brief import build_agent_brief, assess_understanding_substrate
+
+
+from understanding_cache import _load_json, _merge_overview_into_scenes
+from understanding_storyboard import (
+    _generate_edited_storyboard,
+    _generate_source_storyboard,
+    _prepend_storyboard_brief_header,
+)
+
+
+def _clip_text(text, limit):
+    value = " ".join(str(text or "").split()).strip()
+    return value[:limit]
+
+
+def _research_context(work_dir):
+    """Fold background_research.json into a compact context string for the VLM prompt.
+
+    The agent does story research first (per references/research-guide.md) and writes
+    work_dir/background_research.json; this surfaces synopsis, named characters,
+    relationships, plot arcs, and cultural notes so scene VLM analysis can name people
+    and read scenes with plot knowledge instead of labelling everyone "黑衣男子".
+    Returns "" when no usable research file is present, so behaviour is unchanged.
+    """
+    path = Path(work_dir) / "background_research.json"
+    if not path.exists():
+        return ""
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return ""
+    if not isinstance(data, dict):
+        return ""
+    parts = []
+    for key in ("synopsis", "episode_context", "worldbuilding"):
+        val = _clip_text(data.get(key), 320)
+        if val:
+            parts.append(val)
+    chars = data.get("characters")
+    if isinstance(chars, dict) and chars:
+        named = "；".join(
+            f"{_clip_text(name, 40)}（{_clip_text(desc, 120)}）"
+            for name, desc in list(chars.items())[:12]
+            if _clip_text(name, 40)
+        )
+        if named:
+            parts.append("主要人物：" + named)
+    details = data.get("character_details")
+    if isinstance(details, dict) and details:
+        detail_lines = []
+        for name, info in list(details.items())[:8]:
+            if not isinstance(info, dict):
+                continue
+            bits = []
+            aliases = info.get("aliases")
+            if isinstance(aliases, list) and aliases:
+                clean_aliases = [_clip_text(alias, 30) for alias in aliases[:3]]
+                clean_aliases = [alias for alias in clean_aliases if alias]
+                if clean_aliases:
+                    bits.append("别名" + "/".join(clean_aliases))
+            role = _clip_text(info.get("role"), 60)
+            if role:
+                bits.append(role)
+            rels = info.get("relationships")
+            if isinstance(rels, list) and rels:
+                clean_rels = [_clip_text(rel, 60) for rel in rels[:3]]
+                bits.extend(rel for rel in clean_rels if rel)
+            clean_name = _clip_text(name, 40)
+            if clean_name and bits:
+                detail_lines.append(f"{clean_name}（{'；'.join(bits)}）")
+        if detail_lines:
+            parts.append("人物关系：" + "；".join(detail_lines))
+    arcs = data.get("plot_arcs")
+    if isinstance(arcs, list) and arcs:
+        arc_lines = []
+        for arc in arcs[:6]:
+            if not isinstance(arc, dict):
+                val = _clip_text(arc, 120)
+                if val:
+                    arc_lines.append(val)
+                continue
+            name = _clip_text(arc.get("name"), 50)
+            desc = _clip_text(arc.get("description"), 120)
+            status = _clip_text(arc.get("status"), 30)
+            if name or desc:
+                tail = f"[{status}]" if status else ""
+                arc_lines.append(f"{name}：{desc}{tail}".strip("："))
+        if arc_lines:
+            parts.append("剧情线：" + "；".join(arc_lines))
+    notes = data.get("cultural_notes")
+    if isinstance(notes, list) and notes:
+        note_lines = []
+        for note in notes[:4]:
+            if not isinstance(note, dict):
+                val = _clip_text(note, 100)
+                if val:
+                    note_lines.append(val)
+                continue
+            item = _clip_text(note.get("item"), 50)
+            expl = _clip_text(note.get("explanation"), 100)
+            if item or expl:
+                note_lines.append(f"{item}：{expl}".strip("："))
+        if note_lines:
+            parts.append("背景注释：" + "；".join(note_lines))
+    return " ".join(parts).strip()[:1200]
+
+
+def _load_understanding_artifacts_for_brief(work_dir):
+    """Load existing analysis artifacts for a brief-only regeneration pass.
+
+    Material-library restores are allowed to reuse expensive analysis artifacts,
+    but cut pass 2 must still rebuild `agent_narration_brief.md` against the
+    rendered OUTPUT timeline. This helper deliberately performs no extraction,
+    ASR, VLM, or external API calls; it only reads already-present JSON.
+    """
+    work_dir = Path(work_dir)
+    scenes = []
+    for name in ("vlm_analysis.json", "scenes.json"):
+        path = work_dir / name
+        if not path.exists():
+            continue
+        try:
+            data = _load_json(path)
+        except (OSError, ValueError, TypeError):
+            continue
+        if isinstance(data, list):
+            scenes = data
+            break
+    asr_result = []
+    if (work_dir / "asr_result.json").exists():
+        try:
+            data = _load_json(work_dir / "asr_result.json")
+        except (OSError, ValueError, TypeError):
+            data = []
+        if isinstance(data, list):
+            asr_result = data
+    silence_periods = []
+    if (work_dir / "silence_periods.json").exists():
+        try:
+            data = _load_json(work_dir / "silence_periods.json")
+        except (OSError, ValueError, TypeError):
+            data = []
+        if isinstance(data, list):
+            silence_periods = data
+    return scenes, asr_result, silence_periods
+
+
+def _write_brief_from_existing_artifacts(video, work_dir, args, video_duration):
+    """Regenerate only the agent brief/timeline-fusion from existing artifacts."""
+    scenes, asr_result, silence_periods = _load_understanding_artifacts_for_brief(
+        work_dir
+    )
+    if not (Path(work_dir) / "speech_boundary_anchors.json").exists():
+        detect_speech_boundary_anchors(work_dir, asr_result)
+    overview_path = Path(work_dir) / "mimo_video_overview.json"
+    if CONFIG.get("mimo_video_overview", False):
+        scenes = _merge_overview_into_scenes(scenes, overview_path)
+
+    source_storyboard = None
+    edited_storyboard = None
+    scenes_json = Path(work_dir) / "scenes.json"
+    if scenes_json.exists():
+        source_storyboard = _generate_source_storyboard(
+            work_dir, Path(video), scenes, scenes_json, force=False
+        )
+    edited_storyboard = _generate_edited_storyboard(work_dir, video, force=False)
+    cut_mode = (Path(work_dir) / "clip_plan_validated.json").exists()
+
+    substrate = assess_understanding_substrate(scenes, asr_result)
+    if substrate["level"] != "rich":
+        banner = "理解素材为空" if substrate["level"] == "empty" else "理解素材偏薄"
+        log(
+            f"⚠️  {banner}：ASR {substrate['asr_chars']} 字 | 场景 {substrate['scene_count']} | "
+            f"带 frame_facts 的场景 {substrate['scenes_with_frame_facts']} | 平均画面描述 {substrate['avg_description_len']} 字"
+        )
+    brief_path = build_agent_brief(
+        scenes,
+        asr_result,
+        silence_periods,
+        video_duration,
+        work_dir,
+        args.style,
+        mimo_overview_enabled=CONFIG.get("mimo_video_overview", False),
+        mimo_overview_video_path=video,
+    )
+    _prepend_storyboard_brief_header(
+        brief_path, source_storyboard, edited_storyboard, cut_mode=cut_mode
+    )
+    log("=" * 50)
+    log(f"brief-only 完成。写作 brief: {brief_path}")
+    print(
+        json.dumps(
+            {
+                "status": "brief_only",
+                "work_dir": str(work_dir),
+                "brief": str(brief_path),
+                "substrate": substrate["level"],
+                "scenes": len(scenes),
+                "asr_segments": len(asr_result),
+            },
+            ensure_ascii=False,
+        )
+    )