@oh-story/dsh 0.1.11 → 0.1.13

package/lib/video-recap/skills/video-cut/scripts/frame_grid.py

@@ -0,0 +1,249 @@
+"""Frame-grid facts and edge snapping so edited_source.mp4 is constant frame rate.
+
+A clip edge between two source frames leaves the concat filter a segment whose audio is a
+fraction of a frame longer than its video, so the next segment starts off the frame grid and
+the encoded file loses one frame slot per join (an 80 ms hold at 25 fps). Every clip therefore
+starts on its source's frame grid and lasts a whole number of output frames.
+"""
+
+import math
+from fractions import Fraction
+
+# The NTSC rates ffprobe reports as N/1001; canvas fps buckets store them rounded.
+_NTSC_RATES = {
+    23.976: Fraction(24000, 1001),
+    29.97: Fraction(30000, 1001),
+    47.952: Fraction(48000, 1001),
+    59.94: Fraction(60000, 1001),
+}
+# An edge "inside" a pause window is judged without the gate's 50 ms tolerance: a half-frame
+# move must not pull an edge that sits on a pause boundary back over the spoken word.
+_INSIDE_EPSILON = 0.001
+
+
+def parse_frame_rate(text):
+    """An ffprobe 'N/D' rate as a Fraction; None when unknown or implausible.
+
+    Rates above 120 fps are container timebases of variable-rate sources, not a frame grid.
+    """
+    num, _, den = str(text or "").partition("/")
+    try:
+        rate = Fraction(num) / Fraction(den or 1)
+    except (ValueError, ZeroDivisionError):
+        return None
+    return rate if 0 < rate <= 120 else None
+
+
+def canvas_frame_rate(fps):
+    """Exact rate for a canvas fps bucket (29.97 is 30000/1001, not 2997/100)."""
+    return _NTSC_RATES.get(round(float(fps), 3)) or Fraction(float(fps)).limit_denominator(1001)
+
+
+def output_frame_rate(rows, canvas_fps):
+    """The frame rate edited_source.mp4 is rendered at.
+
+    One source keeps its own exact rate; several sources are resampled to the canvas bucket.
+    """
+    if len(rows) == 1:
+        rate = parse_frame_rate(rows[0].get("frame_rate"))
+        if rate:
+            return rate
+    return canvas_frame_rate(canvas_fps)
+
+
+def frame_count(duration, rate):
+    """Whole output frames a clip of `duration` seconds renders to (at least one)."""
+    return max(1, int(round(duration * rate)))
+
+
+def source_frame_grids(geometry_qc):
+    """{source_path: grid} from the selected output geometry, for snap_edges_to_frames."""
+    out_rate = Fraction(geometry_qc["frame_rate"])
+    return {
+        row["path"]: {
+            "source_rate": parse_frame_rate(row.get("frame_rate")),
+            "origin": float(row.get("video_start_offset", 0.0)),
+            "output_rate": out_rate,
+        }
+        for row in geometry_qc["sources"]
+    }
+
+
+# Gate statuses from sentence_boundaries' edge classifier, safest first.
+_GATE_ORDER = {"safe": 0, "unchecked": 1, "blocking": 2}
+
+
+def _rank(edge, ts, windows, classify):
+    """(gate status, 0 strictly inside a sentence/quiet window else 1).
+
+    `classify(edge, ts)` is the sentence-boundary gate itself, so a sub-frame move never
+    turns an edge the gate passes into one it blocks.
+    """
+    strict = any(w["start"] - _INSIDE_EPSILON <= ts <= w["end"] + _INSIDE_EPSILON
+                 for w in windows)
+    return _GATE_ORDER[classify(edge, ts)[0]], 0 if strict else 1
+
+
+def _drops_kept_edge(edge, original, ts, keep_ranges):
+    """Whether moving a clip edge from `original` to `ts` cuts into a required range.
+
+    Required-evidence coverage is exact, so a start that moves past a node's start (or an
+    end that moves before a node's end) the clip used to cover would lose that node.
+    """
+    if edge == "start":
+        return any(original <= start < ts for start, _ in keep_ranges)
+    return any(ts < end <= original for _, end in keep_ranges)
+
+
+# How far a detected shot change may sit from a frame boundary and still count as on it
+# (ffmpeg pts times and clip edges are both rounded to the millisecond).
+_CUT_EPSILON = 0.002
+
+
+def _straddles_cut(edge, ts, candidates, scene_cuts):
+    """Whether choosing `ts` keeps a sliver of the other shot across a source hard cut.
+
+    A start before a cut that lies up to the later candidate opens on the old shot (a flash
+    frame before the cut); an end after a cut that lies from the earlier candidate on closes on
+    the next shot. The candidate at or past the cut (start) or before it (end) does not.
+    """
+    if edge == "start":
+        last = max(candidates)
+        return any(ts + _CUT_EPSILON < cut <= last + _CUT_EPSILON for cut in scene_cuts)
+    first = min(candidates)
+    return any(first - _CUT_EPSILON <= cut < ts - _CUT_EPSILON for cut in scene_cuts)
+
+
+def _pick(edge, original, candidates, windows, classify, keep_ranges, scene_cuts=()):
+    """The candidate the gate likes best (then one strictly inside a pause window), then one
+    that keeps required ranges, then one that does not straddle a detected source hard cut
+    (no flash frame), then the nearest, then the earlier (a deterministic tie-break)."""
+    return min(
+        candidates,
+        key=lambda t: (
+            *_rank(edge, t, windows, classify),
+            _drops_kept_edge(edge, original, t, keep_ranges),
+            _straddles_cut(edge, t, candidates, scene_cuts),
+            abs(t - original),
+            t,
+        ),
+    )
+
+
+def _overlaps(clips, idx, start, end):
+    return any(
+        start < other["source_end"] and end > other["source_start"]
+        for j, other in enumerate(clips)
+        if j != idx
+    )
+
+
+def snap_edges_to_frames(clips, grid, windows, classify, source_duration, *,
+                         allow_overlap, joined_to_previous):
+    """Snap every clip of ONE source onto the frame grid; returns (clips, events).
+
+    The start moves to the nearest source frame boundary and the end to the nearest whole
+    number of output frames after it, each choosing between the two neighbouring candidates
+    the one the sentence-boundary gate `classify(edge, ts) -> (status, reason)` rates safest
+    (safe, then unchecked, then blocking), preferring one strictly inside a pause window.
+    A start that continues the previous clip's source range exactly follows that clip's
+    snapped end, so a lossless join stays lossless. Between equally safe candidates the one
+    that still covers every `grid["keep_ranges"]` (required-evidence nodes) edge it covered
+    wins, then the one that does not straddle a `grid["scene_cuts"]` source hard cut (a start
+    at or after the cut, an end at or before it, so no frame of the other shot flashes).
+    Without a usable source rate the start is left alone, but the length is still a whole
+    number of output frames.
+    """
+    clips = [dict(c) for c in clips]
+    rate, origin, out_rate = grid["source_rate"], grid["origin"], grid["output_rate"]
+    keep_ranges = grid.get("keep_ranges", ())
+    scene_cuts = grid.get("scene_cuts", ())
+    events = []
+    for idx, clip in enumerate(clips):
+        start, end = clip["source_start"], clip["source_end"]
+        if joined_to_previous[idx] and idx > 0:
+            new_start, start_reason = clips[idx - 1]["source_end"], "continuous_source_join"
+        elif rate is None:
+            new_start, start_reason = start, "source_frame_rate_unknown"
+        else:
+            pos = (start - origin) * rate
+            ks = {max(0, math.floor(pos)), max(0, math.ceil(pos))}
+            options = [round(origin + float(k / rate), 3) for k in sorted(ks)]
+            safe = [t for t in options if t >= start or allow_overlap
+                    or not _overlaps(clips, idx, t, end)]
+            new_start = _pick("start", start, safe or [max(options)], windows, classify,
+                              keep_ranges, scene_cuts)
+            start_reason = "frame_grid"
+        span = (end - new_start) * out_rate
+        counts = sorted({max(1, math.floor(span)), max(1, math.ceil(span))})
+        options = [
+            (n, round(new_start + float(n / out_rate), 3)) for n in counts
+            if new_start + float(n / out_rate) <= source_duration + _INSIDE_EPSILON
+        ] or [(counts[0], round(new_start + float(counts[0] / out_rate), 3))]
+        safe = [(n, t) for n, t in options if t <= end or allow_overlap
+                or not _overlaps(clips, idx, new_start, t)] or options[:1]
+        new_end = _pick("end", end, [t for _, t in safe], windows, classify, keep_ranges,
+                        scene_cuts)
+        frames = next(n for n, t in safe if t == new_end)
+        clip["source_start"], clip["source_end"] = new_start, new_end
+        events.append({
+            "clip_id": clip["clip_id"],
+            "source_id": clip.get("source_id"),
+            "original_start": round(start, 3),
+            "original_end": round(end, 3),
+            "new_start": new_start,
+            "new_end": new_end,
+            "start_reason": start_reason,
+            "frame_count": frames,
+        })
+    return clips, events
+
+
+def edge_frame_snapper(grid, windows, classify, source_duration):
+    """frame_snap(edge, clip, ts): where snap_edges_to_frames puts `clip`'s edge moved to ts.
+
+    The gate uses it to check a suggested edge time after frame alignment, not before.
+    """
+    def frame_snap(edge, clip, when):
+        key = "source_start" if edge == "start" else "source_end"
+        moved = {**clip, key: when}
+        if moved["source_end"] <= moved["source_start"]:
+            return when
+        snapped, _ = snap_edges_to_frames([moved], grid, windows, classify, source_duration,
+                                          allow_overlap=True, joined_to_previous=[False])
+        return snapped[0][key]
+
+    return frame_snap
+
+
+def record_frame_grid(plan, geometry_qc):
+    """Stamp each clip's frame_count and qc.frame_grid; the render draws exactly these frames.
+
+    The output timeline (output_start/output_end, total_duration) is rewritten from the
+    cumulative frame count, so it matches the render instead of drifting by the per-clip
+    millisecond rounding (a 29.97 fps frame is 33.367 ms).
+    """
+    out_rate = Fraction(geometry_qc["frame_rate"])
+    total = 0
+    for clip in plan["clips"]:
+        clip["frame_count"] = frame_count(clip["duration"], out_rate)
+        clip["output_start"] = round(float(total / out_rate), 3)
+        total += clip["frame_count"]
+        clip["output_end"] = round(float(total / out_rate), 3)
+    plan["total_duration"] = round(float(total / out_rate), 3)
+    plan.setdefault("qc", {})["frame_grid"] = {
+        "output_frame_rate": geometry_qc["frame_rate"],
+        "frame_count": total,
+        "duration": round(float(total / out_rate), 3),
+        "sources": [
+            {
+                "source_id": row["source_id"],
+                "path": row["path"],
+                "frame_rate": row.get("frame_rate"),
+                "video_start_offset": row.get("video_start_offset", 0.0),
+                "start_snapped": parse_frame_rate(row.get("frame_rate")) is not None,
+            }
+            for row in geometry_qc["sources"]
+        ],
+    }
+    return plan

package/lib/video-recap/skills/video-cut/scripts/lib.py

@@ -46,9 +46,6 @@ CONFIG = {
     "clip_start_snap_max_prepend": env_float("CLIP_START_SNAP_MAX_PREPEND", 1.8, min_val=0.0),
     "clip_start_snap_max_trim": env_float("CLIP_START_SNAP_MAX_TRIM", 0.35, min_val=0.0),
     "clip_join_audio_fade_ms": env_float("CLIP_JOIN_AUDIO_FADE_MS", 30.0, min_val=0.0),
-    # video-cut is the only skill that implements clip padding, so it is the only skill that
-    # may declare the knob. cut_cli reads this as the default for --clip-padding.
-    "clip_padding": env_float("CLIP_PADDING", 0.0, min_val=0.0),
     # Keep clip boundaries off the ORIGINAL footage's hard cuts: a clip that opens/closes a few
     # tenths of a second from a source shot-change shows a brief sliver of the adjacent shot that
     # then hard-cuts again — a visible 闪烁/flicker at the edit point. Snap source_start forward

package/lib/video-recap/skills/video-cut/scripts/media_geometry.py

@@ -2,6 +2,7 @@
 
 import json
 
+from frame_grid import canvas_frame_rate, output_frame_rate, parse_frame_rate
 from lib import run_cmd
 
 
@@ -57,7 +58,38 @@ def _fps_from_rate(rate):
     return float(num) / float(den) if float(den) > 0 else 0.0
 
 
-def _geometry_from_stream(stream):
+def _frame_rate_text(stream):
+    """The stream's real frame grid as an 'N/D' string.
+
+    Interlaced streams often report the field rate as r_frame_rate (50/1 for 25 frames a
+    second); when r_frame_rate is exactly twice avg_frame_rate the average is the grid.
+    A variable-rate phone clip can report r_frame_rate well above its real average (60/1
+    for ~29.6 frames a second); rendering that at 60 fps CFR would double every frame, so
+    the common rate nearest the average is used instead, as it is when r_frame_rate is
+    unusable (`0/0`) but the average is not.
+    """
+    r_rate = stream.get("r_frame_rate", "0/0")
+    avg = stream.get("avg_frame_rate", "0/0")
+    avg_fps = _fps_from_rate(avg)
+    if 0 < avg_fps <= 120:
+        ratio = _fps_from_rate(r_rate) / avg_fps
+        if abs(ratio - 2) < 0.01:
+            return avg
+        if ratio > 1.5 or parse_frame_rate(r_rate) is None:
+            rate = canvas_frame_rate(_fps_bucket(avg_fps))
+            return f"{rate.numerator}/{rate.denominator}"
+    return r_rate
+
+
+def _video_start_offset(stream, format_start):
+    """Seconds from the file start (ffmpeg's input zero) to the first video frame."""
+    try:
+        return round(max(0.0, float(stream.get("start_time")) - float(format_start or 0.0)), 6)
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def _geometry_from_stream(stream, format_start=0.0):
     coded_width, coded_height = stream["width"], stream["height"]
     parsed_sar = _parse_ratio(stream.get("sample_aspect_ratio"))
     dar = _parse_ratio(stream.get("display_aspect_ratio"))
@@ -80,9 +112,9 @@ def _geometry_from_stream(stream):
         display_width, display_height = display_height, display_width
 
     width, height = _clamp_even_geometry(round(display_width), round(display_height))
-    fps = _fps_from_rate(stream["r_frame_rate"]) or _fps_from_rate(
-        stream.get("avg_frame_rate", "0/0")
-    )
+    frame_rate = _frame_rate_text(stream)
+    # The canvas fps bucket follows the same frame grid (not an interlaced field rate).
+    fps = _fps_from_rate(frame_rate) or _fps_from_rate(stream.get("avg_frame_rate", "0/0"))
     if not 0 < fps <= 120:
         fps = 30.0
     facts = {
@@ -91,6 +123,9 @@ def _geometry_from_stream(stream):
         "width": width,
         "height": height,
         "fps": round(fps, 3),
+        # Exact rate and first-frame time: the grid video-cut snaps clip edges onto.
+        "frame_rate": frame_rate,
+        "video_start_offset": _video_start_offset(stream, format_start),
         "sample_aspect_ratio": stream.get("sample_aspect_ratio", "1:1"),
         "sample_aspect_ratio_float": round(sar, 6),
         "display_aspect_ratio": stream.get("display_aspect_ratio"),
@@ -118,7 +153,8 @@ def _probe_video_geometry(video_path):
         "-select_streams",
         "v:0",
         "-show_entries",
-        "stream=width,height,r_frame_rate,avg_frame_rate,sample_aspect_ratio,display_aspect_ratio:stream_tags=rotate:stream_side_data=rotation",
+        "stream=width,height,r_frame_rate,avg_frame_rate,start_time,sample_aspect_ratio,"
+        "display_aspect_ratio:stream_tags=rotate:stream_side_data=rotation:format=start_time",
         "-of",
         "json",
         str(video_path),
@@ -126,10 +162,11 @@ def _probe_video_geometry(video_path):
     result = run_cmd(cmd)
     if result.returncode != 0:
         raise RuntimeError(f"ffprobe 无法读取视频几何信息: {video_path}: {result.stderr.strip()}")
-    streams = json.loads(result.stdout).get("streams", [])
+    payload = json.loads(result.stdout)
+    streams = payload.get("streams", [])
     if not streams:
         raise RuntimeError(f"没有视频流: {video_path}")
-    return _geometry_from_stream(streams[0])
+    return _geometry_from_stream(streams[0], payload.get("format", {}).get("start_time"))
 
 
 def _orientation(width, height):
@@ -179,6 +216,8 @@ def _select_output_geometry(source_paths, clips):
                 "display_height": facts["display_height"],
                 "area": width * height,
                 "fps": fps,
+                "frame_rate": facts["frame_rate"],
+                "video_start_offset": facts["video_start_offset"],
                 "fps_bucket": _fps_bucket(fps),
                 "orientation": _orientation(width, height),
                 "rotation": facts["rotation"],
@@ -220,6 +259,8 @@ def _select_output_geometry(source_paths, clips):
         "width": width,
         "height": height,
         "fps": round(fps, 3),
+        # The exact rate edited_source.mp4 is rendered at (one source keeps its own rate).
+        "frame_rate": str(output_frame_rate(rows, fps)),
         "reason": "weighted_orientation_area_fps",
         "source_id": selected["source_id"],
         "source_path": selected["path"],
@@ -236,3 +277,128 @@ def _select_output_geometry(source_paths, clips):
         "sources": rows,
     }
     return width, height, round(fps, 3), reason
+
+
+# Delivered-picture format. `_probe_video_format`, `_output_color_tags`,
+# `_color_tag_filter` and `_color_tag_args` are intentional function-level copies shared
+# with the final render (tests/orchestrator/test_render_format_parity.py keeps them identical).
+_COLOR_FIELDS = (
+    ("color_space", "colorspace"),
+    ("color_primaries", "color_primaries"),
+    ("color_transfer", "color_trc"),
+)
+_UNTAGGED_COLOR = {None, "", "unknown", "unspecified", "reserved", "N/A"}
+# ffprobe's colour space for an RGB picture (PNG, libx264rgb, FFV1 RGB). ffmpeg rejects
+# `-colorspace gbr`, and the delivered picture is YUV anyway, so it is never written.
+_RGB_COLOR_SPACE = "gbr"
+# Pixel-format name prefixes of ffmpeg's RGB family (packed, planar `gbr*`, paletted, Bayer).
+# Some RGB sources report no colour space at all (QuickTime RLE `argb`, raw `bgr24`, GIF
+# `pal8`), so the pixel format decides too; tests/assemble/test_render_delivery.py checks
+# the prefixes against every format a real ffprobe lists.
+_RGB_PIX_FMT_PREFIXES = (
+    "rgb", "bgr", "gbr", "argb", "abgr", "0rgb", "0bgr", "x2rgb", "x2bgr", "pal8", "bayer_",
+)
+# Names ffprobe prints that setparams and the -colorspace/-color_primaries/-color_trc
+# output options accept and libx264 writes back (tests/assemble/test_render_delivery.py
+# runs each through a real ffmpeg); anything else is not written explicitly.
+_KNOWN_COLOR_VALUES = {
+    "colorspace": {
+        "bt709", "fcc", "bt470bg", "smpte170m", "smpte240m", "ycgco", "bt2020nc", "bt2020c",
+        "smpte2085", "chroma-derived-nc", "chroma-derived-c", "ictcp",
+    },
+    "color_primaries": {
+        "bt709", "bt470m", "bt470bg", "smpte170m", "smpte240m", "film", "bt2020",
+        "smpte428", "smpte431", "smpte432",
+    },
+    "color_trc": {
+        "bt709", "bt470m", "bt470bg", "smpte170m", "smpte240m", "linear", "log100",
+        "log316", "iec61966-2-4", "bt1361e", "iec61966-2-1", "bt2020-10", "bt2020-12",
+        "smpte2084", "smpte428", "arib-std-b67",
+    },
+}
+# setparams only knows the BT.470 transfers by their ffprobe names, while ffmpeg 8's
+# -color_trc output option only knows them as gamma22/gamma28 (ffmpeg 9 takes both).
+_COLOR_OPTION_SPELLING = {"color_trc": {"bt470m": "gamma22", "bt470bg": "gamma28"}}
+
+
+def _probe_video_format(video_path):
+    """Codec, pixel format, size and colour tags of the first video stream ({} if unreadable)."""
+    result = run_cmd([
+        "ffprobe", "-v", "error", "-select_streams", "v:0",
+        "-show_entries",
+        "stream=codec_name,pix_fmt,width,height,color_space,color_primaries,color_transfer,color_range",
+        "-of", "json", str(video_path),
+    ])
+    if result.returncode != 0:
+        return {}
+    try:
+        streams = json.loads(result.stdout).get("streams") or []
+    except (ValueError, AttributeError):
+        return {}
+    return streams[0] if streams and isinstance(streams[0], dict) else {}
+
+
+def _output_color_tags(stream):
+    """Colour tags to write on the delivered picture, as {ffmpeg option: value}.
+
+    An untagged or BT.709 source is labelled BT.709, which is how players already decode
+    untagged web video. Any other declared colour space passes through as declared. The
+    range follows the source: full range stays `pc`, everything else is limited `tv`.
+    For a YUV source nothing here converts pixels; it only fixes the labels.
+
+    An RGB source (colour space `gbr`, or an RGB pixel format with or without a colour
+    space) has no YUV matrix to keep: it is converted to BT.709 limited range, and
+    `from_rgb` tells `_color_tag_filter` to do that conversion explicitly instead of
+    letting ffmpeg pick a matrix.
+    """
+    declared = {option: stream.get(key) for key, option in _COLOR_FIELDS}
+    from_rgb = (
+        declared["colorspace"] == _RGB_COLOR_SPACE
+        or str(stream.get("pix_fmt") or "").startswith(_RGB_PIX_FMT_PREFIXES)
+    )
+    if from_rgb:
+        declared["colorspace"] = "bt709"
+    if all(value in _UNTAGGED_COLOR or value == "bt709" for value in declared.values()):
+        tags = {option: "bt709" for _, option in _COLOR_FIELDS}
+    else:
+        tags = {
+            option: value for option, value in declared.items()
+            if value in _KNOWN_COLOR_VALUES[option]
+        }
+    if from_rgb:
+        tags["color_range"] = "tv"
+        tags["from_rgb"] = True
+    else:
+        tags["color_range"] = "pc" if stream.get("color_range") == "pc" else "tv"
+    return tags
+
+
+def _color_tag_filter(tags):
+    """setparams filter that stamps the tags on every frame before the encoder.
+
+    Output options alone are not enough when re-encoding: ffmpeg 8/9 let the (untagged)
+    frame properties win for primaries and transfer, so the bitstream would stay untagged.
+    An RGB frame is first converted to BT.709 yuv420p explicitly: stamping a YUV colour
+    space on an RGB frame makes ffmpeg's own later conversion use the BT.601 matrix
+    while the file says BT.709.
+    """
+    parts = [f"{option}={tags[option]}" for _, option in _COLOR_FIELDS if option in tags]
+    parts.append(f"range={tags['color_range']}")
+    setparams = "setparams=" + ":".join(parts)
+    if tags.get("from_rgb"):
+        return (
+            f"scale=out_color_matrix=bt709:out_range={tags['color_range']},"
+            f"format=yuv420p,{setparams}"
+        )
+    return setparams
+
+
+def _color_tag_args(tags):
+    """Output options that write the tags into the container (and encoder) metadata."""
+    args = []
+    for _, option in _COLOR_FIELDS:
+        if option in tags:
+            value = _COLOR_OPTION_SPELLING.get(option, {}).get(tags[option], tags[option])
+            args += [f"-{option}", value]
+    return args + ["-color_range", tags["color_range"]]
+