@lemoncat7/dsh-partner 2.8.1 → 2.8.2

package/resources/skills/video-recap-production/upstream/skills/video-understanding/scripts/speech_ownership.py

@@ -0,0 +1,143 @@
+"""Load measured source-speech evidence and classify narration ownership."""
+
+import json
+from pathlib import Path
+
+from lib import CONFIG, file_identity
+
+
+def _empty_evidence(mode):
+    return {
+        "anchors": [],
+        "speech_spans": [],
+        "quiet_windows": [],
+        "require_measured": mode == "cut_output",
+    }
+
+
+def _read_json(path):
+    """JSON artifact, or None when the optional file was never written."""
+    return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None
+
+
+def _output_payload_is_current(payload, work_dir):
+    plan_path = Path(work_dir) / "clip_plan_validated.json"
+    return (
+        payload is not None
+        and plan_path.exists()
+        and payload["clip_plan_identity"] == file_identity(plan_path)
+    )
+
+
+def load_source_sentence_evidence(work_dir, mode="full"):
+    """Load sentence boundaries plus speech/quiet spans on the narration clock."""
+    if work_dir is None:
+        return _empty_evidence(mode)
+    work_dir = Path(work_dir)
+    if mode == "full":
+        payload = _read_json(work_dir / "speech_boundary_anchors.json") or {
+            "sentence_anchors": []
+        }
+        speech_spans = [
+            row for row in _read_json(work_dir / "asr_result.json") or [] if row["text"]
+        ]
+        quiet_windows = [
+            row
+            for row in _read_json(work_dir / "silence_periods.json") or []
+            if not row["has_speech"]
+        ]
+    else:
+        payload = _read_json(work_dir / "speech_boundary_anchors_output.json")
+        if mode == "cut_output" and not _output_payload_is_current(payload, work_dir):
+            return _empty_evidence(mode)
+        if payload is None:
+            payload = {"sentence_anchors": [], "speech_spans": [], "quiet_windows": []}
+        speech_spans = payload["speech_spans"]
+        quiet_windows = payload["quiet_windows"]
+    anchors = [
+        anchor
+        for anchor in payload["sentence_anchors"]
+        if anchor["confidence"] in {"high", "medium"}
+    ]
+    return {
+        "anchors": sorted(anchors, key=lambda item: item["time"]),
+        "speech_spans": speech_spans,
+        "quiet_windows": quiet_windows,
+        "require_measured": mode == "cut_output",
+    }
+
+
+def _merged_intervals(start, end, rows):
+    intervals = sorted(
+        (max(start, row["start"]), min(end, row["end"]))
+        for row in rows
+        if row["end"] > start and row["start"] < end
+    )
+    merged = []
+    for left, right in intervals:
+        if right <= left:
+            continue
+        if merged and left <= merged[-1][1]:
+            merged[-1] = (merged[-1][0], max(merged[-1][1], right))
+        else:
+            merged.append((left, right))
+    return merged
+
+
+def _interval_overlap(start, end, rows):
+    return sum(right - left for left, right in _merged_intervals(start, end, rows))
+
+
+def _speech_overlap_excluding_quiet(start, end, speech, quiet):
+    speech_intervals = _merged_intervals(start, end, speech)
+    quiet_intervals = _merged_intervals(start, end, quiet)
+    overlap = sum(right - left for left, right in speech_intervals)
+    for speech_left, speech_right in speech_intervals:
+        overlap -= sum(
+            max(0.0, min(speech_right, quiet_right) - max(speech_left, quiet_left))
+            for quiet_left, quiet_right in quiet_intervals
+        )
+    return max(0.0, overlap)
+
+
+def segment_overlaps_source_speech(seg, evidence):
+    """Classify aggregate mix ownership across the complete narration interval."""
+    start, end = seg["start"], seg["end"]
+    quiet = evidence["quiet_windows"]
+    quiet_min = max(0.3, (end - start) * CONFIG["quiet_overlap_min_ratio"])
+    speech = evidence["speech_spans"]
+    if speech:
+        return _speech_overlap_excluding_quiet(start, end, speech, quiet) > 0.05
+    if quiet and _interval_overlap(start, end, quiet) >= quiet_min:
+        return False
+    if evidence["anchors"] or evidence["require_measured"]:
+        return True
+    return bool(seg.get("overlaps_speech", True))
+
+
+def entry_overlaps_source_speech(start, evidence, *, authored_overlap=True, tolerance=0.05):
+    """Classify the entry instant; later quiet time cannot erase an unsafe start."""
+    if any(
+        row["start"] - tolerance <= start <= row["end"] + tolerance
+        for row in evidence["quiet_windows"]
+    ):
+        return False
+    if any(
+        row["start"] - tolerance <= start < row["end"] - tolerance
+        for row in evidence["speech_spans"]
+    ):
+        return True
+    if evidence["speech_spans"]:
+        return False
+    if evidence["anchors"] or evidence["require_measured"]:
+        return True
+    return bool(authored_overlap)
+
+
+def measure_narration_speech_ownership(narration, work_dir, mode="full"):
+    """Return narration copies with aggregate ownership derived from evidence."""
+    evidence = load_source_sentence_evidence(work_dir, mode=mode)
+    return [
+        {**seg, "overlaps_speech": segment_overlaps_source_speech(seg, evidence)}
+        for seg in narration
+    ]

package/resources/skills/video-recap-production/upstream/skills/video-understanding/scripts/storyboard.py

@@ -0,0 +1,445 @@
+#!/usr/bin/env python3
+"""Storyboard (contact-sheet) generation for video-understanding.
+
+Two ADVISORY artifacts that help the writing agent orient on the timeline by SCANNING
+one image instead of opening dozens of frames:
+
+  - source_storyboard.{jpg,json}  — scene-anchored tiles over the SOURCE timeline.
+  - edited_storyboard.{jpg,json}  — one row per kept clip over the cut OUTPUT timeline,
+                                    each tile dual-labelled (output time / source time).
+
+Both reuse the frames already extracted by understand.py (frames/frame_*.jpg at CONFIG["fps"]).
+Nothing here re-extracts video. Every function returns dict|None and degrades to
+None + log(...) on ANY failure (no frames, ffmpeg missing/non-zero, font probe raises),
+so a storyboard quirk can NEVER block the pipeline (Principle 1: advisory, never blocking).
+
+drawtext "mm:ss" labels are attempted when a usable font is found (labels_burned:true);
+when no font is available (or drawtext errors) the sheet is still produced UNLABELLED
+(labels_burned:false) and the JSON sidecar stays authoritative for all timestamps.
+"""
+import json
+import math
+import shutil
+import subprocess
+from pathlib import Path
+
+from extract import frame_number_for_time, parse_frame_number
+from lib import CONFIG, run_cmd, log, get_video_duration
+
+
+# Candidate font files probed (in order) for burning mm:ss labels. The first that exists
+# AND that drawtext can actually load wins. A probe that RAISES must never abort the sheet.
+_FONT_CANDIDATES = (
+    "/System/Library/Fonts/Supplemental/Arial.ttf",
+    "/System/Library/Fonts/Helvetica.ttc",
+    "/Library/Fonts/Arial.ttf",
+    "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
+    "/usr/share/fonts/dejavu/DejaVuSans.ttf",
+    "/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",
+    "/usr/share/fonts/TTF/DejaVuSans.ttf",
+)
+
+
+def _fmt_mmss(seconds):
+    """Format a timestamp as mm:ss (clamped to >= 0)."""
+    total = int(round(max(0.0, float(seconds))))
+    return f"{total // 60:02d}:{total % 60:02d}"
+
+
+def _ffmpeg_available():
+    return shutil.which("ffmpeg") is not None
+
+
+def _probe_font():
+    """Return a usable font file path, or None. Any exception → None (never raises out).
+
+    Probe order: explicit candidate files first, then `fc-match` if available. The path
+    is only returned when the file exists on disk; we do NOT shell out to drawtext here —
+    a render-time drawtext error is caught separately and also degrades to unlabelled.
+    """
+    try:
+        for candidate in _FONT_CANDIDATES:
+            if Path(candidate).is_file():
+                return candidate
+        fc_match = shutil.which("fc-match")
+        if fc_match:
+            result = subprocess.run(
+                [fc_match, "-f", "%{file}", "sans"],
+                capture_output=True, text=True, timeout=10,
+            )
+            path = (result.stdout or "").strip()
+            if result.returncode == 0 and path and Path(path).is_file():
+                return path
+    except Exception as exc:  # noqa: BLE001 - a font probe must NEVER abort the sheet
+        log(f"storyboard 字体探测异常（降级为不烧时间戳）: {exc}")
+        return None
+    return None
+
+
+def _frame_index(work_dir):
+    """Return (sorted_frame_paths, sorted_numbers) for frames/frame_*.jpg, or ([], [])."""
+    frames_dir = Path(work_dir) / "frames"
+    if not frames_dir.is_dir():
+        return [], []
+    pairs = []
+    for path in frames_dir.glob("frame_*.jpg"):
+        number = parse_frame_number(path)
+        if number is None:
+            continue
+        pairs.append((number, path))
+    pairs.sort(key=lambda item: item[0])
+    numbers = [num for num, _ in pairs]
+    paths = [path for _, path in pairs]
+    return paths, numbers
+
+
+def _nearest_existing_frame(timestamp, fps, paths, numbers):
+    """Map a SOURCE timestamp → the nearest EXISTING frame file, clamped to [first,last].
+
+    Frames are named frame_{n:05d}.jpg; the number↔time mapping is owned by extract.py
+    (frame_00001 is t=0, so n = t*fps + 1). Rounding a timestamp blindly can yield a frame
+    number that was never written (fps boundary / last frame gap), so we resolve to the
+    closest number that actually exists on disk.
+    """
+    if not numbers:
+        return None
+    fps = float(fps)
+    if fps <= 0:
+        return None
+    target = frame_number_for_time(timestamp, fps)
+    # Clamp into the real extracted range so out-of-range timestamps pin to first/last frame.
+    if target <= numbers[0]:
+        return paths[0]
+    if target >= numbers[-1]:
+        return paths[-1]
+    # numbers is sorted; find the closest by absolute distance (ties → earlier frame).
+    best_idx = 0
+    best_dist = None
+    for idx, num in enumerate(numbers):
+        dist = abs(num - target)
+        if best_dist is None or dist < best_dist:
+            best_dist = dist
+            best_idx = idx
+        elif num - target > best_dist:
+            break  # sorted: distance only grows from here
+    return paths[best_idx]
+
+
+def _scene_anchor_timestamps(scenes, max_tiles):
+    """Scene-anchored sample timestamps: each scene midpoint; long scenes also +1/3 & +2/3.
+
+    Returns a list of (scene_id, timestamp). A scene is "long" when adding the thirds gives
+    materially distinct sample points; we treat scenes longer than `long_scene_seconds` as
+    long. Total is capped at max_tiles by evenly subsampling the ordered anchor list, so the
+    sheet stays legible (D2: one bounded contact sheet, not dozens of frame reads).
+    """
+    long_scene_seconds = float(CONFIG["storyboard_long_scene_seconds"])
+    anchors = []
+    for scene_id, scene in enumerate(scenes or []):
+        start = float(scene["start"])
+        end = float(scene["end"])
+        if end <= start:
+            continue
+        mid = (start + end) / 2.0
+        if (end - start) >= long_scene_seconds:
+            third = start + (end - start) / 3.0
+            two_third = start + 2.0 * (end - start) / 3.0
+            points = [third, mid, two_third]
+        else:
+            points = [mid]
+        for ts in points:
+            anchors.append((scene_id, round(ts, 3)))
+    if max_tiles and len(anchors) > max_tiles:
+        # Evenly subsample to the cap, preserving timeline order and scene spread.
+        step = len(anchors) / float(max_tiles)
+        anchors = [anchors[int(i * step)] for i in range(max_tiles)]
+    return anchors
+
+
+def _labelled_frame(frame_path, label, font_path, scratch_dir, out_name):
+    """Burn `label` onto a copy of frame_path via drawtext; return the labelled path or None.
+
+    None signals the caller to fall back to the original frame (and flip labels_burned off).
+    A drawtext failure here is non-fatal: the unlabelled frame still tiles fine.
+    """
+    out_path = scratch_dir / out_name
+    safe_label = label.replace("\\", "\\\\").replace(":", "\\:").replace("'", "’")
+    drawtext = (
+        f"drawtext=fontfile='{font_path}':text='{safe_label}':"
+        "x=8:y=8:fontsize=28:fontcolor=white:"
+        "box=1:boxcolor=black@0.55:boxborderw=6"
+    )
+    cmd = ["ffmpeg", "-y", "-i", str(frame_path), "-vf", drawtext, "-frames:v", "1", str(out_path)]
+    try:
+        result = run_cmd(cmd)
+    except Exception as exc:  # noqa: BLE001 - a label render must never abort the sheet
+        log(f"storyboard drawtext 异常（降级为不烧时间戳）: {exc}")
+        return None
+    if result.returncode != 0 or not out_path.exists():
+        return None
+    return out_path
+
+
+def _tile_pages(frame_paths, columns, out_dir, out_stem, scratch_dir):
+    """Tile frame_paths into one or more contact-sheet pages; return the list of page paths.
+
+    ffmpeg's tile filter lays one grid per page. We page so each sheet holds at most
+    columns*rows tiles where rows is chosen to keep the grid roughly square but capped, then
+    spill into _001.jpg, _002.jpg… Returns [] on any ffmpeg failure (caller degrades to None).
+    """
+    columns = max(1, int(columns))
+    rows_per_page = int(CONFIG["storyboard_rows_per_page"])
+    per_page = columns * rows_per_page
+    pages = []
+    total = len(frame_paths)
+    page_count = max(1, math.ceil(total / per_page))
+    for page_idx in range(page_count):
+        chunk = frame_paths[page_idx * per_page:(page_idx + 1) * per_page]
+        if not chunk:
+            continue
+        cols = min(columns, len(chunk))
+        rows = max(1, math.ceil(len(chunk) / cols))
+        if page_count == 1:
+            page_path = out_dir / f"{out_stem}.jpg"
+        else:
+            page_path = out_dir / f"{out_stem}_{page_idx + 1:03d}.jpg"
+        # Stage the chunk as a contiguous numbered sequence so ffmpeg's image2 demuxer +
+        # tile filter consume EXACTLY these frames (the source frame numbers are sparse).
+        seq_dir = scratch_dir / f"seq_{out_stem}_{page_idx:03d}"
+        seq_dir.mkdir(parents=True, exist_ok=True)
+        for seq_idx, src in enumerate(chunk):
+            shutil.copyfile(src, seq_dir / f"f_{seq_idx:05d}.jpg")
+        cmd = [
+            "ffmpeg", "-y",
+            "-framerate", "1",
+            "-i", str(seq_dir / "f_%05d.jpg"),
+            "-frames:v", "1",
+            "-vf", f"tile={cols}x{rows}",
+            str(page_path),
+        ]
+        result = run_cmd(cmd)
+        if result.returncode != 0 or not page_path.exists():
+            log(f"storyboard tile 失败: {result.stderr[-300:]}")
+            return []
+        pages.append(page_path)
+    return pages
+
+
+def _render_storyboard(work_dir, tiles, out_stem):
+    """Shared render path: optionally burn labels, tile to pages, return (page_paths, labels_burned).
+
+    `tiles` is a list of dicts that ALREADY carry a resolved `frame_file` (absolute path) and a
+    `label` string. Returns (None, _) on hard failure so callers degrade to None.
+    """
+    if not _ffmpeg_available():
+        log("storyboard 跳过：未找到 ffmpeg")
+        return None, False
+    storyboard_dir = Path(work_dir) / "storyboard"
+    storyboard_dir.mkdir(parents=True, exist_ok=True)
+    scratch_dir = storyboard_dir / f".scratch_{out_stem}"
+    if scratch_dir.exists():
+        shutil.rmtree(scratch_dir, ignore_errors=True)
+    scratch_dir.mkdir(parents=True, exist_ok=True)
+
+    font_path = _probe_font()
+    labels_burned = bool(font_path)
+    render_frames = []
+    if labels_burned:
+        for idx, tile in enumerate(tiles):
+            labelled = _labelled_frame(
+                Path(tile["frame_file"]), tile["label"], font_path, scratch_dir,
+                f"lbl_{out_stem}_{idx:05d}.jpg",
+            )
+            if labelled is None:
+                # First failure → abandon labelling entirely so the WHOLE sheet is consistent
+                # (no half-labelled pages). The JSON sidecar still carries every timestamp.
+                labels_burned = False
+                break
+            render_frames.append(labelled)
+    if not labels_burned:
+        render_frames = [Path(tile["frame_file"]) for tile in tiles]
+
+    try:
+        pages = _tile_pages(
+            render_frames, CONFIG["storyboard_columns"],
+            storyboard_dir, out_stem, scratch_dir,
+        )
+    finally:
+        shutil.rmtree(scratch_dir, ignore_errors=True)
+    if not pages:
+        return None, labels_burned
+    return pages, labels_burned
+
+
+def build_source_storyboard(work_dir, video_path, scenes, fps):
+    """Scene-anchored SOURCE-timeline contact sheet. Returns dict|None.
+
+    Samples each scene midpoint (long scenes also +1/3,+2/3), maps every sample to the
+    nearest EXISTING extracted frame (clamped), tiles them, and writes
+    storyboard/source_storyboard.json. Returns None + log on any failure.
+    """
+    try:
+        work_dir = Path(work_dir)
+        paths, numbers = _frame_index(work_dir)
+        if not paths:
+            log("storyboard 跳过 source：frames/ 为空或缺失")
+            return None
+        max_tiles = int(CONFIG["storyboard_max_tiles"])
+        columns = int(CONFIG["storyboard_columns"])
+        anchors = _scene_anchor_timestamps(scenes, max_tiles)
+        if not anchors:
+            log("storyboard 跳过 source：无可用场景锚点")
+            return None
+
+        tiles = []
+        for tile_id, (scene_id, ts) in enumerate(anchors):
+            frame = _nearest_existing_frame(ts, fps, paths, numbers)
+            if frame is None:
+                continue
+            tiles.append({
+                "tile_id": tile_id,
+                "timestamp": round(float(ts), 3),
+                "label": _fmt_mmss(ts),
+                "scene_id": scene_id,
+                "frame_file": str(frame),
+            })
+        if not tiles:
+            log("storyboard 跳过 source：未解析到任何帧")
+            return None
+
+        pages, labels_burned = _render_storyboard(work_dir, tiles, "source_storyboard")
+        if not pages:
+            log("storyboard 跳过 source：拼贴失败")
+            return None
+
+        for tile in tiles:
+            tile["frame_file"] = Path(tile["frame_file"]).name
+        payload = {
+            "schema_version": 1,
+            "timeline": "source",
+            "video_path": str(video_path),
+            "fps": float(fps) if fps else None,
+            "labels_burned": labels_burned,
+            "page_images": [str(p) for p in pages],
+            "sample_policy": {
+                "max_tiles": max_tiles,
+                "columns": columns,
+                "anchors": "scene_midpoint+long_scene_thirds",
+            },
+            "tiles": tiles,
+        }
+        try:
+            payload["duration"] = round(get_video_duration(video_path), 3)
+        except RuntimeError as exc:
+            log(f"storyboard source：无法读取视频时长，sidecar 省略 duration（忽略）: {exc}")
+        json_path = work_dir / "storyboard" / "source_storyboard.json"
+        json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+        log(f"storyboard source: {len(tiles)} tiles → {len(pages)} page(s), labels_burned={labels_burned}")
+        return payload
+    except Exception as exc:  # noqa: BLE001 - advisory: never propagate
+        log(f"storyboard source 失败（忽略）: {exc}")
+        return None
+
+
+def _source_to_output(source_time, clip):
+    """Forward affine source→output map for ONE clip (reimplements cut.py:319 locally).
+
+    output = clip.output_start + (src − clip.source_start), clamped to [output_start, output_end].
+    Read the authoritative numbers from clip_plan_validated.json; do NOT import cut.py.
+    """
+    src = float(source_time)
+    out_start = float(clip["output_start"])
+    out_end = float(clip["output_end"])
+    mapped = out_start + (src - float(clip["source_start"]))
+    return round(max(out_start, min(mapped, out_end)), 3)
+
+
+def build_edited_storyboard(work_dir, source_video_path, clip_plan_validated, fps):
+    """OUTPUT-timeline contact sheet: one row per kept clip over the cut. Returns dict|None.
+
+    For each kept clip, samples source start / mid / (end − 0.5s), maps each to the nearest
+    EXISTING SOURCE frame (SOURCE fps — frames are reused, NO re-extraction), and FRAME-IDENTITY
+    dedupes so a ≤1s clip yields 1-2 tiles (not 3 identical). Each tile is dual-labelled with
+    both `output_timestamp` and `source_timestamp` (+ `source_clip_id`). Writes
+    storyboard/edited_storyboard.json. Returns None + log on any failure.
+    """
+    try:
+        work_dir = Path(work_dir)
+        paths, numbers = _frame_index(work_dir)
+        if not paths:
+            log("storyboard 跳过 edited：frames/ 为空或缺失")
+            return None
+        clips = clip_plan_validated["clips"]
+        if not clips:
+            log("storyboard 跳过 edited：clip_plan_validated 无 clips")
+            return None
+        columns = int(CONFIG["storyboard_columns"])
+        max_tiles = int(CONFIG["storyboard_max_tiles"])
+
+        tiles = []
+        seen_frames = set()  # frame-identity dedupe (NOT luma de-dupe; that stays deferred)
+        tile_id = 0
+        for clip in clips:
+            try:
+                source_start = float(clip["source_start"])
+                source_end = float(clip["source_end"])
+                clip_id = clip.get("clip_id")
+            except (KeyError, TypeError, ValueError):
+                continue
+            if source_end <= source_start:
+                continue
+            mid = (source_start + source_end) / 2.0
+            end_sample = max(source_start, source_end - 0.5)
+            for src_ts in (source_start, mid, end_sample):
+                frame = _nearest_existing_frame(src_ts, fps, paths, numbers)
+                if frame is None:
+                    continue
+                key = (clip_id, str(frame))
+                if key in seen_frames:
+                    continue  # same clip resolving to the same frame → drop the duplicate tile
+                seen_frames.add(key)
+                out_ts = _source_to_output(src_ts, clip)
+                tiles.append({
+                    "tile_id": tile_id,
+                    "output_timestamp": out_ts,
+                    "source_timestamp": round(float(src_ts), 3),
+                    "source_clip_id": clip_id,
+                    "label": f"out {_fmt_mmss(out_ts)} / src {_fmt_mmss(src_ts)}",
+                    "frame_file": str(frame),
+                })
+                tile_id += 1
+        if not tiles:
+            log("storyboard 跳过 edited：未解析到任何帧")
+            return None
+        if len(tiles) > max_tiles:
+            tiles = tiles[:max_tiles]
+
+        pages, labels_burned = _render_storyboard(work_dir, tiles, "edited_storyboard")
+        if not pages:
+            log("storyboard 跳过 edited：拼贴失败")
+            return None
+
+        for tile in tiles:
+            tile["frame_file"] = Path(tile["frame_file"]).name
+        edited_source = Path(work_dir) / "edited_source.mp4"
+        payload = {
+            "schema_version": 1,
+            "timeline": "output",
+            "source_video_path": str(source_video_path),
+            "edited_video_path": str(edited_source) if edited_source.exists() else None,
+            "labels_burned": labels_burned,
+            "page_images": [str(p) for p in pages],
+            "sample_policy": {
+                "max_tiles": max_tiles,
+                "columns": columns,
+                "per_clip": "source_start+mid+(end-0.5s), frame-identity deduped",
+            },
+            "tiles": tiles,
+        }
+        json_path = work_dir / "storyboard" / "edited_storyboard.json"
+        json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+        log(f"storyboard edited: {len(tiles)} tiles → {len(pages)} page(s), labels_burned={labels_burned}")
+        return payload
+    except Exception as exc:  # noqa: BLE001 - advisory: never propagate
+        log(f"storyboard edited 失败（忽略）: {exc}")
+        return None