ffmpeg-skill 0.7.1 → 0.8.0

package/README.md

@@ -124,6 +124,22 @@ More examples: [examples/README.md](examples/README.md). To see everything run e
 
 All scripts: Python 3.9+, standard library only, `--help`, non-zero exit + stderr message on failure.
 
+## Measured, not assumed
+
+`tests/corpus.py` downloads public real-device footage (GoPro, DJI, iPhone incl. Dolby Vision, Android screen recordings, HDR10, 24p, Tears of Steel) and runs the toolchain on it; `tests/bench_sync.py`, `bench_silence.py` and `bench_scenes.py` score the algorithms against known ground truth.
+
+| What | Result (0.8.0, local ffmpeg 6.1) |
+|---|---|
+| Real-device corpus, 10 files | 92 verify steps, all pass after fixes |
+| sync.py, ±30 s offsets, gain/noise/EQ, real dialogue+music | 120 s windows (the documented rule): 40/40 within 10 ms, max 1.1 ms. 60 s stress windows: 95 % within 10 ms, 4 of 5 misses flagged by confidence |
+| silence.py, 20 cases, known gaps | 0 missed gaps, ≤ 1 ms leftover silence |
+| scenes.py, 53 hard cuts between single takes (GoPro/DJI/iPhone/…) | precision 0.95, recall 1.00, F1 0.97 at the default threshold |
+
+```bash
+python3 tests/corpus.py --fetch --verify     # ~1.4 GB download, then verify (slow on 4K)
+python3 tests/bench_sync.py --cases 100
+```
+
 ## MCP
 
 ```json

package/SKILL.md

@@ -167,8 +167,12 @@ scenes.py INPUT [--threshold 10] [--min-scene 1] [--highlights N [--target SECON
 Lists scenes with audio energy, the loudest moments, and (with
 `--highlights`) proposes N ranges that add up to `--target` seconds, biased to
 the loudest window of each scene. Review the sheet + JSON, adjust the EDL, then
-`cut.py --segments`. It is a proposal engine, not a judgement of content:
-tell the user what it picked and why (energy, scene length).
+`cut.py --segments`. Cut detection is a one-frame spike test (benchmark on
+hard cuts between real single takes: precision 0.95, recall 1.00 at the default
+threshold; raise `--threshold` to 12 for 0.98 precision at 0.94 recall).
+Dissolves and very slow fades are not cuts and will be missed. Highlights are
+a proposal engine, not a judgement of content: tell the user what it picked
+and why (energy, scene length).
 
 ### check.py — pre-delivery compliance
 ```
@@ -300,7 +304,12 @@ video with the second file's audio aligned (video stream copied).
 the clock difference in ppm, and resamples the second file so a 60-minute
 take stays in sync (typical consumer devices drift 20-500 ppm = up to 1.8 s/h).
 Use it whenever the recording is longer than ~10 minutes. Check `confidence`
-(0–1); below ~0.3 the match is doubtful — use a window with a clear event.
+(0–1, normalised correlation with a runner-up penalty); below 0.3 the match is
+doubtful. Benchmark on real dialogue/music (±30 s offsets, gain, noise, EQ):
+with the default 120 s window 40/40 within 10 ms (max 1.1 ms); with a 60 s
+window 95 %, misses flagged below 0.3. Keep `--analyze-seconds` at least 4×
+`--max-offset` (default 120 s vs 30 s): lags with under 35 % overlap are
+ignored, so an offset larger than ~60 % of the window cannot be found.
 
 ### color.py — HDR to SDR, LUTs, colour tags, Dolby Vision
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ffmpeg-skill",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Agent Skill that lets coding agents (Claude Code, Cursor, Codex) do professional video editing with local FFmpeg: MCP server, batch processing, declarative project rendering, brand kits, motion-graphics templates, HTML delivery reports, scene detection, delivery checks, cut, silence removal, transitions, multicam, captions, sync with drift correction, HDR to SDR, LUTs, audio clean-up and ducking, loudness, platform exports. No API keys, no cloud, no dependencies.",
   "keywords": ["ffmpeg", "video", "agent-skill", "claude-code", "cursor", "codex", "skill", "video-editing"],
   "license": "MIT",

package/scripts/loudness.py

@@ -34,7 +34,9 @@ def measure(path: str, I: float, tp: float, lra: float) -> dict:
     data = json.loads(m.group(0))
     for k in ("input_i", "input_tp", "input_lra", "input_thresh", "target_offset"):
         if data.get(k) in (None, "-inf", "inf", "nan"):
-            die(f"loudnorm returned unusable value for {k}: {data.get(k)} (silent input?)")
+            data["silent"] = True
+            return data
+    data["silent"] = False
     return data
 
 
@@ -57,6 +59,12 @@ def main() -> int:
         die("input has no audio stream")
 
     stats = measure(args.input, args.lufs, args.tp, args.lra)
+    if stats.get("silent"):
+        info("audio is silent (integrated loudness -inf); nothing to normalise")
+        if args.measure_only:
+            print(json.dumps({"silent": True, "input_i": "-inf"}, indent=2))
+            return 0
+        die("input audio is silent; loudness normalisation is meaningless (use audio.py --replace to add a track)")
     info(f"measured: {float(stats['input_i']):.1f} LUFS, TP {float(stats['input_tp']):.1f} dBTP, LRA {float(stats['input_lra']):.1f} LU")
     if args.measure_only:
         print(json.dumps({k: stats[k] for k in ("input_i", "input_tp", "input_lra", "input_thresh", "target_offset")}, indent=2))
@@ -79,7 +87,8 @@ def main() -> int:
     run(cmd)
 
     after = measure(output, args.lufs, args.tp, args.lra)
-    info(f"result:   {float(after['input_i']):.1f} LUFS, TP {float(after['input_tp']):.1f} dBTP (target {args.lufs} LUFS)")
+    if not after.get("silent"):
+        info(f"result:   {float(after['input_i']):.1f} LUFS, TP {float(after['input_tp']):.1f} dBTP (target {args.lufs} LUFS)")
     emit(output)
     return 0
 

package/scripts/scenes.py

@@ -23,17 +23,49 @@ from typing import Dict, List, Tuple
 
 from _common import add_common, apply_common, die, emit, ffmpeg_base, info, print_json, probe, require_tool, run
 
-SCENE_RE = re.compile(r"lavfi\.scd\.time=([0-9.]+)")
+SCORE_RE = re.compile(r"frame:(\d+)\s+pts:\d+\s+pts_time:([0-9.]+)")
 
 
-def detect_scenes(path: str, threshold: float, min_len: float, duration: float) -> List[float]:
+def detect_scenes(path: str, threshold: float, min_len: float, duration: float, ratio: float = 3.0) -> List[float]:
+    """Scene cuts = frames whose scdet score is above `threshold` AND stands out from its
+    neighbourhood (score > ratio x median of the surrounding +-12 frames). Sustained motion,
+    flashes and fast pans raise the score on many consecutive frames and are rejected;
+    a real cut is a one-frame spike. On real footage this roughly doubles precision at
+    equal recall compared with the raw scdet threshold."""
     ffmpeg = require_tool("ffmpeg")
     proc = subprocess.run([ffmpeg, "-hide_banner", "-nostdin", "-i", path, "-an", "-vf",
-                           f"scale=320:-2,scdet=threshold={threshold}:sc_pass=1,metadata=print:file=-", "-f", "null", "-"],
+                           "scale=320:-2,scdet=threshold=0:sc_pass=1,metadata=print:file=-", "-f", "null", "-"],
                           stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-    times = [float(t) for t in SCENE_RE.findall(proc.stdout)]
+    times: List[float] = []
+    scores: List[float] = []
+    cur_t = None
+    for line in proc.stdout.splitlines():
+        m = SCORE_RE.match(line)
+        if m:
+            cur_t = float(m.group(2))
+            continue
+        if line.startswith("lavfi.scd.score=") and cur_t is not None:
+            try:
+                times.append(cur_t)
+                scores.append(float(line.split("=", 1)[1]))
+            except ValueError:
+                pass
     cuts = [0.0]
-    for t in times:
+    if not scores:
+        return cuts
+    w = 12
+    for i, sc in enumerate(scores):
+        if sc < threshold:
+            continue
+        lo, hi = max(0, i - w), min(len(scores), i + w + 1)
+        neigh = sorted(scores[lo:i] + scores[i + 1:hi])
+        med = neigh[len(neigh) // 2] if neigh else 0.0
+        if sc < ratio * max(med, 0.5):
+            continue
+        # keep only the local maximum inside +-2 frames
+        if any(scores[j] > sc for j in range(max(0, i - 2), min(len(scores), i + 3)) if j != i):
+            continue
+        t = times[i]
         if t - cuts[-1] >= min_len:
             cuts.append(t)
     if duration - cuts[-1] < min_len and len(cuts) > 1:
@@ -60,7 +92,8 @@ def audio_envelope(path: str, step_s: float) -> List[float]:
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument("input")
-    ap.add_argument("--threshold", type=float, default=10.0, help="scdet threshold 0-100 (default 10; lower = more cuts)")
+    ap.add_argument("--threshold", type=float, default=8.0, help="minimum scdet score for a cut, 0-100 (default 8)")
+    ap.add_argument("--ratio", type=float, default=3.0, help="a cut must exceed this multiple of the neighbouring frames' median score (default 3; lower = more cuts)")
     ap.add_argument("--min-scene", type=float, default=1.0, help="ignore cuts closer than this in seconds (default 1)")
     ap.add_argument("--highlights", type=int, default=0, help="number of highlight ranges to propose")
     ap.add_argument("--target", type=float, help="with --highlights: total seconds the picks should add up to (trims long scenes)")
@@ -75,7 +108,7 @@ def main() -> int:
     if not meta.get("video"):
         die("input has no video stream")
     dur = meta.get("duration") or 0.0
-    cuts = detect_scenes(args.input, args.threshold, args.min_scene, dur)
+    cuts = detect_scenes(args.input, args.threshold, args.min_scene, dur, args.ratio)
     bounds = cuts + [dur]
     step_s = 0.5
     env = audio_envelope(args.input, step_s) if meta.get("audio") else []

package/scripts/sync.py

@@ -92,6 +92,14 @@ def ifft(a: List[complex]) -> List[complex]:
 
 
 def cross_correlate(ref: List[float], other: List[float], max_lag: int):
+    """Normalised cross-correlation over the overlapping region only.
+
+    The raw FFT correlation sum grows with the overlap length, so with a 60 s window a
+    correct 28 s offset (32 s overlap) loses to a wrong 2 s offset (58 s overlap) on
+    music-like material. Dividing each lag by the energy of the overlapping parts
+    (prefix sums, O(1) per lag) makes lags comparable and turns the peak value into a
+    real similarity score in 0..1 that doubles as the confidence.
+    """
     n = 1
     while n < len(ref) + len(other):
         n <<= 1
@@ -99,15 +107,50 @@ def cross_correlate(ref: List[float], other: List[float], max_lag: int):
     fb = fft([complex(x) for x in other] + [0j] * (n - len(other)))
     prod = [x * y.conjugate() for x, y in zip(fa, fb)]
     corr = ifft(prod)
-    # corr[k] = sum ref[i+k]*other[i]  -> lag k means 'other' is delayed by k relative to ref? see below
-    best_lag, best_val = 0, -float("inf")
+    # prefix sums of squares for overlap energy
+    def prefix(v: List[float]) -> List[float]:
+        out = [0.0]
+        acc = 0.0
+        for x in v:
+            acc += x * x
+            out.append(acc)
+        return out
+    pr, po = prefix(ref), prefix(other)
+    lr, lo = len(ref), len(other)
     max_lag = min(max_lag, n // 2 - 1)
+    best_lag, best_val, second = 0, -float("inf"), -float("inf")
+    # ignore lags with less than 35 % overlap: with the documented rule (analysis window >= 4x the
+    # largest expected offset) true offsets always keep >= 75 % overlap, while short-overlap lags are
+    # where coincidental matches on quasi-periodic material (music, tone beds) live
+    min_overlap = max(10, int(0.35 * min(lr, lo)))
+    scores = []
     for lag in range(-max_lag, max_lag + 1):
-        val = corr[lag % n].real
+        # corr[lag] = sum_i ref[i] * other[i - lag]  -> ref index range and other index range overlap:
+        r0, r1 = max(0, lag), min(lr, lo + lag)
+        if r1 - r0 < min_overlap:
+            continue
+        e_ref = pr[r1] - pr[r0]
+        e_oth = po[r1 - lag] - po[r0 - lag]
+        denom = math.sqrt(e_ref * e_oth)
+        if denom <= 0:
+            continue
+        val = corr[lag % n].real / denom
+        # mild preference for longer overlaps: a perfect match over 55 % of the window must not tie
+        # with a perfect match over 100 % (quasi-periodic material). Exponent 0.5: with the window rule (>= 4x offset) a true match keeps >= 75 % overlap (x0.87) while a coincidental 55 % match drops to x0.74; keeps large true
+        # offsets (28 s in 60 s = 53 % overlap -> x0.94) competitive while still breaking exact ties.
+        val *= ((r1 - r0) / min(lr, lo)) ** 0.5
+        scores.append((val, lag))
         if val > best_val:
+            second = best_val
             best_val, best_lag = val, lag
-    energy = math.sqrt(sum(x * x for x in ref) * sum(x * x for x in other)) or 1.0
-    return best_lag, best_val / energy
+        elif val > second and abs(lag - best_lag) > 5:
+            second = val
+    # confidence: peak similarity, penalised when a distant runner-up is nearly as good
+    conf = max(0.0, min(1.0, best_val))
+    if second > -float("inf") and best_val > 0:
+        margin = (best_val - second) / best_val
+        conf *= min(1.0, 0.5 + margin)
+    return best_lag, conf
 
 
 def refine(ref_s: List[float], oth_s: List[float], coarse_offset: float, fine_step: int, window_s: float) -> float:

package/scripts/verify.py

@@ -118,7 +118,7 @@ def main() -> int:
                 plan.append(("overlay text", ["overlay.py", cut, "--text", "verify", "--position", "top-left", "-o", f"{stem}_ovl.mp4"] + fast))
                 plan.append(("look sheet", ["look.py", cut, "-o", f"{stem}_sheet.png"]))
                 if (meta.get("video") or {}).get("hdr"):
-                    plan.append(("color to-sdr", ["color.py", str(f), "--to-sdr", "-o", f"{stem}_sdr.mp4"] + fast))
+                    plan.append(("color to-sdr", ["color.py", f"{stem}_acc.mp4", "--to-sdr", "-o", f"{stem}_sdr.mp4"] + fast))
                     plan.append(("hdr preserved", ["__check_hdr__", f"{stem}_acc.mp4"]))
                 plan.append(("probe analyze", ["probe.py", cut, "--analyze"]))
             plan.append(("export x", ["export.py", cut, "--preset", "x", "-o", f"{stem}_x.mp4"]))