@oh-story/dsh 0.1.11 → 0.1.13

package/lib/video-recap/skills/video-cut/scripts/sentence_boundaries.py

@@ -5,7 +5,16 @@ import re
 import subprocess
 from pathlib import Path
 
+from frame_grid import edge_frame_snapper, snap_edges_to_frames
 from lib import log
+from sentence_gate import (
+    _GATE_TOLERANCE,
+    _continuous_source_join,
+    _edge_classifier,
+    _edge_joined,
+    _unverified_window,
+    enforce_clip_sentence_boundaries,
+)
 
 
 def _find_source_artifact(work_dir, filename, source_id=None, source_work_dir=None):
@@ -20,11 +29,12 @@ def _find_source_artifact(work_dir, filename, source_id=None, source_work_dir=No
 
 
 def _load_sentence_boundary_windows(work_dir, source_id=None, source_work_dir=None):
-    """Load reliable sentence-end pause windows produced by video-understanding.
+    """Load usable sentence-end pause windows produced by video-understanding.
 
     A sentence anchor's `time` is the acoustic pause end, while `pause_start` is already
-    after the final spoken sample. Any cut within that closed interval preserves the sentence.
-    Low-confidence anchors are deliberately excluded from the hard-safety path.
+    after the final spoken sample. Any cut within that closed interval is word-safe.
+    `boundary_use: unverified` anchors (coarse-ASR estimates) stay usable but keep that
+    label so an edge resting on one is reported as such; `none` anchors are excluded.
     """
     path = _find_source_artifact(
         work_dir, "speech_boundary_anchors.json", source_id, source_work_dir
@@ -32,35 +42,80 @@ def _load_sentence_boundary_windows(work_dir, source_id=None, source_work_dir=No
     if path is None:
         return []
     payload = json.loads(path.read_text(encoding="utf-8"))
-    windows = [
-        {
+    windows = []
+    for anchor in payload["sentence_anchors"]:
+        # Schema-1 anchors (no `boundary_use`) came from the old coarse estimator: high/medium
+        # labels there are usable but unverified.
+        use = anchor.get("boundary_use") or (
+            "unverified" if anchor["confidence"] in {"high", "medium"} else "none"
+        )
+        if use == "none":
+            continue
+        windows.append({
             "start": round(anchor["pause_start"], 3),
             "end": round(anchor["time"], 3),
             "kind": "sentence_anchor",
             "confidence": anchor["confidence"],
-        }
-        for anchor in payload["sentence_anchors"]
-        if anchor["confidence"] in {"high", "medium"}
-    ]
+            "boundary_use": use,
+        })
     return sorted(windows, key=lambda row: (row["start"], row["end"]))
 
 
-def _load_source_speech_spans(work_dir, source_id=None, source_work_dir=None):
-    """Merged ASR speech spans (asr_clean.json wins over asr_result.json).
+# Interjections and common ASR artifacts on screams/music. A window whose text is only these
+# is not dialogue at a clip edge or narration entry; real short lines such as "救我！" still are.
+# Same copy in video-cut, video-script and video-assemble (parity-tested by function).
+_NON_DIALOGUE_TOKENS = frozenset(
+    "啊 嗯 哼 哦 呃 唉 嘿 呦 哈 呀 hi yeah ok okay oh uh ah hmm".split()
+)
+_NON_DIALOGUE_CJK = frozenset("啊嗯哼哦呃唉嘿呦哈呀")
+# Lines cross 15s ASR window edges (a line may run 13.2–15.4 while its window ends at 15.0),
+# so an interjection-only window next to real dialogue keeps this much of its shared edge.
+_INTERJECTION_GUARD_SECONDS = 1.0
 
-    Only used to decide whether an unsafe edge blocks; a missing transcript means unchecked.
+
+def _interjection_only(text):
+    tokens = [token for token in re.split(r"[\W_]+", text.lower()) if token]
+    # Punctuation-only rows ("……", "？") are often ASR for unintelligible speech: keep them.
+    return bool(tokens) and all(
+        token in _NON_DIALOGUE_TOKENS or set(token) <= _NON_DIALOGUE_CJK for token in tokens
+    )
+
+
+def _dialogue_speech_spans(rows):
+    """Merged dialogue spans from timed ASR rows.
+
+    A row holding only interjections ("啊！", "Hi.") is not dialogue, except a
+    `_INTERJECTION_GUARD_SECONDS` guard on an edge it shares with a dialogue row. A row
+    whose text is empty or whitespace (ASR heard no words) is timing-only evidence: it is
+    skipped here and guards nothing. A row with no `text` field is measured timing whose
+    words are unknown and counts as dialogue.
     """
-    rows = []
-    for filename in ("asr_clean.json", "asr_result.json"):
-        path = _find_source_artifact(work_dir, filename, source_id, source_work_dir)
-        if path is not None:
-            payload = json.loads(path.read_text(encoding="utf-8"))
-            rows = payload["segments"] if filename == "asr_clean.json" else payload
-            break
-    spans = sorted(
-        ({"start": row["start"], "end": row["end"]} for row in rows if row["text"].strip()),
+    rows = sorted(
+        (
+            {
+                "start": row["start"],
+                "end": row["end"],
+                "dialogue": not _interjection_only(row.get("text", "")),
+            }
+            for row in rows
+            if "text" not in row or row["text"].strip()
+        ),
         key=lambda row: (row["start"], row["end"]),
     )
+    spans = []
+    for idx, row in enumerate(rows):
+        if row["dialogue"]:
+            spans.append({"start": row["start"], "end": row["end"]})
+            continue
+        before = rows[idx - 1] if idx > 0 else None
+        after = rows[idx + 1] if idx + 1 < len(rows) else None
+        if before and before["dialogue"] and row["start"] - before["end"] <= 0.05:
+            end = min(row["end"], row["start"] + _INTERJECTION_GUARD_SECONDS)
+            spans.append({"start": row["start"], "end": end})
+        if after and after["dialogue"] and after["start"] - row["end"] <= 0.05:
+            start = max(row["start"], row["end"] - _INTERJECTION_GUARD_SECONDS)
+            spans.append({"start": start, "end": row["end"]})
+    spans.sort(key=lambda row: (row["start"], row["end"]))
     merged = []
     for span in spans:
         if merged and span["start"] <= merged[-1]["end"] + 0.05:
@@ -70,98 +125,36 @@ def _load_source_speech_spans(work_dir, source_id=None, source_work_dir=None):
     return merged
 
 
-def _combine_boundary_windows(*groups):
-    unique = {
-        (round(row["start"], 3), round(row["end"], 3)) for group in groups for row in group
-    }
-    return [{"start": start, "end": end} for start, end in sorted(unique)]
-
-
-def _same_source(left, right):
-    # Single-source normalized plans omit source identity; both sides then read as None.
-    return left.get("source_id") == right.get("source_id") and left.get(
-        "source_path"
-    ) == right.get("source_path")
-
-
-def _continuous_source_join(left, right, tolerance=0.05):
-    return (
-        _same_source(left, right)
-        and abs(left["source_end"] - right["source_start"]) <= tolerance
-        and abs(left["output_end"] - right["output_start"]) <= tolerance
-    )
-
-
-def enforce_clip_sentence_boundaries(
-    plan, boundary_windows, speech_spans, video_duration, tolerance=0.05
-):
-    """Block any audible clip edge that falls inside detected source speech.
+def _load_source_speech_spans(work_dir, source_id=None, source_work_dir=None):
+    """Merged ASR dialogue spans (asr_clean.json wins over asr_result.json).
 
-    Safe edges are: source start/end, a reliable sentence/quiet pause, or a truly contiguous
-    same-source join (no media is removed). Missing ASR timing degrades to `unchecked` rather
-    than inventing speech. Once ASR says an edge is speech-owned, failure to snap is blocking.
+    Only used to decide whether an unsafe edge blocks; a missing transcript means unchecked.
     """
-    clips = plan["clips"]
-    checks, new_blockers = [], []
-
-    def inside(rows, ts):
-        return any(row["start"] - tolerance <= ts <= row["end"] + tolerance for row in rows)
-
-    for idx, clip in enumerate(clips):
-        for edge, ts in (("start", clip["source_start"]), ("end", clip["source_end"])):
-            contiguous = (
-                edge == "start"
-                and idx > 0
-                and _continuous_source_join(clips[idx - 1], clip, tolerance)
-            ) or (
-                edge == "end"
-                and idx + 1 < len(clips)
-                and _continuous_source_join(clip, clips[idx + 1], tolerance)
-            )
-            if edge == "start" and ts <= tolerance:
-                status, reason = "safe", "source_start"
-            elif edge == "end" and ts >= video_duration - tolerance:
-                status, reason = "safe", "source_end"
-            elif contiguous:
-                status, reason = "safe", "continuous_source_join"
-            elif inside(boundary_windows, ts):
-                status, reason = "safe", "sentence_or_quiet_boundary"
-            elif not speech_spans:
-                status, reason = "unchecked", "speech_timing_unavailable"
-            elif not inside(speech_spans, ts):
-                status, reason = "safe", "outside_detected_speech"
-            else:
-                status, reason = "blocking", "inside_detected_speech"
-            check = {
-                "clip_id": clip["clip_id"],
-                "source_id": clip.get("source_id"),
-                "edge": edge,
-                "time": round(ts, 3),
-                "status": status,
-                "reason": reason,
-            }
-            checks.append(check)
-            if status == "blocking":
-                new_blockers.append(
-                    {
-                        "code": "unsafe_clip_sentence_boundary",
-                        **check,
-                        "message": "剪辑边界仍落在原声讲话区间内，必须移动到句末锚点，不能截断原声句子。",
-                    }
-                )
+    rows = []
+    for filename in ("asr_clean.json", "asr_result.json"):
+        path = _find_source_artifact(work_dir, filename, source_id, source_work_dir)
+        if path is not None:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+            rows = payload["segments"] if filename == "asr_clean.json" else payload
+            break
+    return _dialogue_speech_spans(row for row in rows if row["text"].strip())
 
-    qc = plan.setdefault("qc", {})
-    qc.setdefault("boundary_status", {})["sentence_checks"] = checks
-    existing = [
-        row
-        for row in qc.get("blocking", [])
-        if row["code"] != "unsafe_clip_sentence_boundary"
-    ]
-    if existing or new_blockers:
-        qc["blocking"] = existing + new_blockers
-    else:
-        qc.pop("blocking", None)
-    return plan
+
+def _combine_boundary_windows(*groups):
+    """Dedupe windows by span; a span backed by any verified/quiet row counts as verified."""
+    unique = {}
+    for group in groups:
+        for row in group:
+            key = (round(row["start"], 3), round(row["end"], 3))
+            if key not in unique or _unverified_window(unique[key]):
+                unique[key] = row
+    combined = []
+    for (start, end), row in sorted(unique.items()):
+        item = {"start": start, "end": end, "kind": row.get("kind", "quiet_window")}
+        if _unverified_window(row):
+            item["boundary_use"] = "unverified"
+        combined.append(item)
+    return combined
 
 
 def _recompute_clip_timeline(clips):
@@ -420,7 +413,12 @@ def _detect_shot_changes(video, win_start, win_end, threshold, lead=0.25):
     return sorted(changes)
 
 
-def snap_clips_off_shot_changes(plan, video, margin, threshold, min_keep=0.5):
+# How far past each edge's own search window the shot-change scan also looks, so a hard cut
+# within a frame on the other side of the edge is known to the frame-grid pass.
+_FRAME_CUT_SCAN_PAD = 0.1
+
+
+def snap_clips_off_shot_changes(plan, video, margin, threshold, min_keep=0.5, found_cuts=None):
     """Nudge each clip's boundaries clear of the ORIGINAL footage's hard cuts to avoid 闪烁.
 
     A clip whose source_start sits just before a shot-change opens on a brief sliver of the old
@@ -430,9 +428,14 @@ def snap_clips_off_shot_changes(plan, video, margin, threshold, min_keep=0.5):
       - move source_end   BACK   onto a shot-change in [end-margin, end)       → clean close
     Boundaries already on a cut, or with no nearby cut, are left untouched. Snaps that would shrink
     a clip below `min_keep` are skipped. Recomputes the output timeline cursor-based.
+
+    `found_cuts` (a set) collects every cut the scan saw, including those up to
+    `_FRAME_CUT_SCAN_PAD` on the far side of each edge, for the frame-grid pass's tie-break.
     """
     if margin <= 0:
         return plan
+    pad = _FRAME_CUT_SCAN_PAD if found_cuts is not None else 0.0
+    found_cuts = set() if found_cuts is None else found_cuts
     clips = [dict(c) for c in plan["clips"]]
     n_start = n_end = 0
     events = []
@@ -449,11 +452,9 @@ def snap_clips_off_shot_changes(plan, video, margin, threshold, min_keep=0.5):
             "end_action": "kept",
         }
         # Opening: a shot-change just AFTER source_start leaves an old-shot sliver before it.
-        start_changes = [
-            c
-            for c in _detect_shot_changes(video, s, min(e, s + margin), threshold)
-            if c > s + 1e-3
-        ]
+        seen = _detect_shot_changes(video, max(0.0, s - pad), min(e, s + margin), threshold)
+        found_cuts.update(seen)
+        start_changes = [c for c in seen if c > s + 1e-3]
         if start_changes:
             cand = max(start_changes)  # open after the last rapid cut in the window
             if cand < e - min_keep:
@@ -463,11 +464,9 @@ def snap_clips_off_shot_changes(plan, video, margin, threshold, min_keep=0.5):
             else:
                 event["start_unsnapped_reason"] = "collapse"
         # Closing: a shot-change just BEFORE source_end leaves a next-shot sliver after it.
-        end_changes = [
-            c
-            for c in _detect_shot_changes(video, max(new_s, e - margin), e, threshold)
-            if c < e - 1e-3
-        ]
+        seen = _detect_shot_changes(video, max(new_s, e - margin), e + pad, threshold)
+        found_cuts.update(seen)
+        end_changes = [c for c in seen if c < e - 1e-3]
         if end_changes:
             cand = min(end_changes)  # close before the first rapid cut in the window
             if cand > new_s + min_keep:
@@ -489,15 +488,57 @@ def snap_clips_off_shot_changes(plan, video, margin, threshold, min_keep=0.5):
     return _plan_with_snapped_clips(plan, clips, "shot_snaps", events)
 
 
+def _revert_unsafe_shot_snaps(plan, before, classify):
+    """Undo a shot-change move that left an edge in speech where the plan's own edge was safe.
+
+    Sound outranks picture cleanup. Line snapping can pull a moved edge back out of speech
+    only next to a pause window, so an end the shot-change pass pulled back up to
+    `scene_margin` into a sentence (the plan ended it in an ASR gap that is music, not
+    silence) would block, and the nearest safe edge the gate suggests would be pulled back
+    again on the next run. Such an edge returns to `before` (the clips before the shot-change
+    pass) and its shot_snaps event reads `reverted_unsafe`, unless that would overlap
+    another clip's source range.
+    """
+    events = plan.get("qc", {}).get("boundary_status", {}).get("shot_snaps", [])
+    if len(events) != len(before):
+        return plan
+    clips = [dict(c) for c in plan["clips"]]
+    events = [dict(e) for e in events]
+    reverted = 0
+    for idx, (clip, old, event) in enumerate(zip(clips, before, events)):
+        for edge, key, moved in (("start", "source_start", "moved_forward"),
+                                 ("end", "source_end", "moved_back")):
+            if event[f"{edge}_action"] != moved:
+                continue
+            status, reason = classify(edge, clip[key], _edge_joined(clips, idx, edge))
+            if status != "blocking" or classify(
+                    edge, old[key], _edge_joined(before, idx, edge))[0] != "safe":
+                continue
+            span = {**clip, key: old[key]}
+            if not plan["allow_overlap"] and _candidate_overlaps(
+                    clips, idx, span["source_start"], span["source_end"]):
+                continue
+            clip[key] = old[key]
+            event[f"{edge}_action"] = "reverted_unsafe"
+            event[f"rejected_{edge}"] = event.pop(f"new_{edge}")
+            event[f"{edge}_revert_reason"] = reason
+            reverted += 1
+    if not reverted:
+        return plan
+    log(f"切镜头避让会截断原声，已撤回 {reverted} 处边界")
+    return _plan_with_snapped_clips(plan, clips, "shot_snaps", events)
+
+
 def _load_silence_for_source(work_dir, source_id, source_work_dir=None):
     """Read a source's silence_periods.json (project layout when source_id is None); [] when absent."""
     path = _find_source_artifact(work_dir, "silence_periods.json", source_id, source_work_dir)
     return json.loads(path.read_text(encoding="utf-8")) if path else []
 
 
-def snap_multi_source_clips(
+def snap_source_clips(
     plan,
-    sources,
+    video,
+    duration,
     work_dir,
     *,
     line_max_extend,
@@ -507,13 +548,68 @@ def snap_multi_source_clips(
     start_max_trim,
     do_line_snap=True,
     do_scene_snap=True,
+    source_id=None,
+    source_work_dir=None,
+    frame_grid=None,
 ):
-    """Per-source line/shot snapping for a multi-source validated plan.
+    """Snap every clip of ONE source: shot changes, quiet starts/ends, frame grid, then the gate.
+
+    Visual cleanup goes first. Sentence/quiet snapping is the sound authority because a
+    clean picture is never allowed to reintroduce a mid-sentence audio cut (a shot-change
+    move line snapping cannot repair is reverted when the plan's own edge was gate-safe);
+    the frame-grid pass only moves an edge by under a frame and prefers the side that keeps
+    it safe, then the side that does not straddle a hard cut the shot-change scan saw, and
+    the gate judges the final, frame-aligned edges. source_id None reads the
+    project-level understanding artifacts (single-source layout). frame_grid is a
+    frame_grid.source_frame_grids() entry; None leaves edges off the grid.
+    """
+    before_shot_snap = [dict(c) for c in plan["clips"]]
+    scene_cuts = set()
+    if do_scene_snap:
+        plan = snap_clips_off_shot_changes(
+            plan, video, margin=scene_margin, threshold=scene_threshold, found_cuts=scene_cuts
+        )
+    boundaries = _combine_boundary_windows(
+        _load_silence_for_source(work_dir, source_id, source_work_dir),
+        _load_sentence_boundary_windows(work_dir, source_id, source_work_dir),
+    )
+    if do_line_snap:
+        plan = snap_clip_starts_to_lines(
+            plan, boundaries, duration, start_max_prepend, max_trim=start_max_trim
+        )
+        plan = snap_clip_ends_to_lines(plan, boundaries, duration, line_max_extend)
+    speech_spans = _load_source_speech_spans(work_dir, source_id, source_work_dir)
+    video_start = frame_grid["origin"] if frame_grid is not None else 0.0
+    classify = _edge_classifier(boundaries, speech_spans, duration, _GATE_TOLERANCE,
+                                video_start)
+    if do_scene_snap:
+        plan = _revert_unsafe_shot_snaps(plan, before_shot_snap, classify)
+    if frame_grid is None:
+        return enforce_clip_sentence_boundaries(plan, boundaries, speech_spans, duration)
+
+    frame_grid = {**frame_grid, "scene_cuts": sorted(scene_cuts)}
+    clips = plan["clips"]
+    joined = [idx > 0 and _continuous_source_join(clips[idx - 1], clip)
+              for idx, clip in enumerate(clips)]
+    clips, events = snap_edges_to_frames(
+        clips, frame_grid, boundaries, classify, duration,
+        allow_overlap=plan["allow_overlap"], joined_to_previous=joined,
+    )
+    plan = _plan_with_snapped_clips(plan, clips, "frame_snaps", events)
+    return enforce_clip_sentence_boundaries(
+        plan, boundaries, speech_spans, duration, video_start=video_start,
+        frame_snap=edge_frame_snapper(frame_grid, boundaries, classify, duration),
+    )
+
 
-    Each clip is snapped using ITS OWN source's silence windows / shot changes and duration
-    (a clip in source B never constrains a clip in source A), then the global OUTPUT timeline
-    is recomputed once in plan order. Missing silence data leaves a boundary unchanged.
-    Mirrors the single-source snap_clip_ends_to_lines + snap_clips_off_shot_changes.
+def snap_multi_source_clips(plan, sources, work_dir, frame_grids=None, **snap_options):
+    """Per-source snap_source_clips for a multi-source validated plan.
+
+    Each clip is snapped using ITS OWN source's silence windows / shot changes, frame grid
+    and duration (a clip in source B never constrains a clip in source A), then the global
+    OUTPUT timeline is recomputed once in plan order. Missing silence data leaves a boundary
+    unchanged. `frame_grids` maps source_id to its frame grid; `snap_options` are
+    snap_source_clips' keyword options.
     """
     clips = plan["clips"]
     allow_overlap = plan["allow_overlap"]
@@ -524,34 +620,21 @@ def snap_multi_source_clips(
         "start_snaps": [],
         "end_snaps": [],
         "shot_snaps": [],
+        "frame_snaps": [],
         "sentence_checks": [],
     }
     blocking_accum = []
     for sid, group in groups.items():
         source = sources[sid]
-        duration = source["duration"]
-        mini = {"clips": [dict(c) for c in group], "allow_overlap": allow_overlap}
-        # Visual cleanup goes first. Sentence/quiet snapping is the final authority because
-        # a clean picture is never allowed to reintroduce a mid-sentence audio cut.
-        if do_scene_snap:
-            mini = snap_clips_off_shot_changes(
-                mini, source["source_path"], margin=scene_margin, threshold=scene_threshold
-            )
-        source_work_dir = source.get("source_work_dir")
-        boundaries = _combine_boundary_windows(
-            _load_silence_for_source(work_dir, sid, source_work_dir),
-            _load_sentence_boundary_windows(work_dir, sid, source_work_dir),
-        )
-        if do_line_snap:
-            mini = snap_clip_starts_to_lines(
-                mini, boundaries, duration, start_max_prepend, max_trim=start_max_trim
-            )
-            mini = snap_clip_ends_to_lines(mini, boundaries, duration, line_max_extend)
-        mini = enforce_clip_sentence_boundaries(
-            mini,
-            boundaries,
-            _load_source_speech_spans(work_dir, sid, source_work_dir),
-            duration,
+        mini = snap_source_clips(
+            {"clips": [dict(c) for c in group], "allow_overlap": allow_overlap},
+            source["source_path"],
+            source["duration"],
+            work_dir,
+            source_id=sid,
+            source_work_dir=source.get("source_work_dir"),
+            frame_grid=(frame_grids or {}).get(sid),
+            **snap_options,
         )
         mini_boundary = mini["qc"]["boundary_status"]
         for key, events in boundary_accum.items():