ffmpeg-skill 1.16.1 → 1.17.1

package/scripts/scenes.py

@@ -20,17 +20,38 @@ Examples:
 import argparse
 import math
 import sys
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 # `detect_scenes` moved into _common/probe.py in 1.16.0 (see silence.py); the body is unchanged.
-from _common import detect_scenes, STATE, add_common, apply_common, default_font_file, die, emit, escape_filter_path, ffmpeg_base, info, print_json, probe, run, decode_pcm_mono, rms_envelope
+from _common import (detect_scenes, STATE, add_common, apply_common, beat_grid, default_font_file, die, emit,
+                     escape_filter_path, ffmpeg_base, info, print_json, probe, run, decode_pcm_mono,
+                     rms_envelope, BEAT_MIN_CONFIDENCE)
 
 
 
-def audio_envelope(path: str, step_s: float) -> List[float]:
-    """RMS level per step_s window at 8 kHz, absolute (a loud scene scores higher); [] when the
-    audio cannot be decoded (the cut scoring then runs on the picture alone)."""
-    return rms_envelope(decode_pcm_mono(path, 8000, check=False), int(8000 * step_s))
+def audio_envelope(path: str, step_s: float, *, rate: int = 8000,
+                   samples: "Optional[List[float]]" = None) -> List[float]:
+    """RMS level per step_s window, absolute (a loud scene scores higher); [] when the audio
+    cannot be decoded (the cut scoring then runs on the picture alone).
+
+    `samples` reuses PCM a caller already decoded rather than decoding the same file twice --
+    --beats needs a 10 ms envelope and the scene scoring a 0.5 s one, and 0.5 s is an integer
+    multiple of 10 ms, so both come from one pass.
+    """
+    if samples is None:
+        samples = decode_pcm_mono(path, rate, check=False)
+    return rms_envelope(samples, int(rate * step_s))
+
+
+def parse_beat_range(text: str) -> "tuple":
+    """`--beat-range 60-200` as (lo, hi) BPM."""
+    try:
+        lo, hi = (float(p) for p in str(text).replace(" ", "").split("-", 1))
+    except ValueError:
+        die(f"--beat-range: expected LO-HI in BPM (e.g. 60-200), got {text!r}", kind="input")
+    if not (0 < lo < hi):
+        die(f"--beat-range {text}: LO must be above 0 and below HI", kind="input")
+    return (lo, hi)
 
 
 def main() -> int:
@@ -45,6 +66,16 @@ def main() -> int:
     ap.add_argument("--target", type=float, help="with --highlights: total seconds the picks should add up to (trims long scenes)")
     ap.add_argument("--max-scene", type=float, default=15.0, help="cap a highlight range at this many seconds (default 15)")
     ap.add_argument("--edl", help="write highlight ranges as START-END lines (cut.py --segments format)")
+    ap.add_argument("--beats", action="store_true",
+                    help="measure the music's beat grid (tempo, beat times, confidence) and report it; "
+                         "a measurement, not a proposal -- no cut is made and no beat is invented")
+    ap.add_argument("--beat-step", type=float, default=0.01,
+                    help="envelope resolution in seconds for the onset pass with --beats (default 0.01)")
+    ap.add_argument("--beat-range", default="60-200",
+                    help="tempo search range in BPM for --beats (default 60-200)")
+    ap.add_argument("--min-confidence", type=float, default=BEAT_MIN_CONFIDENCE,
+                    help="with --beats: below this confidence the grid is still reported, marked "
+                         f"usable: false (default {BEAT_MIN_CONFIDENCE})")
     ap.add_argument("--sheet", help="write a contact sheet PNG with the first frame of every scene")
     ap.add_argument("--no-timecode", action="store_true", help="--sheet without the burnt-in timecode stamp (a way out if drawtext itself is unusable, see doctor)")
     add_common(ap)
@@ -54,11 +85,26 @@ def main() -> int:
     meta = probe(args.input)
     if not meta.get("video"):
         die("input has no video stream")
+    # Only parsed when it is going to be used: --beat-range is a --beats flag, and a run that
+    # never asked for beats should not be able to die on one.
+    beat_range = parse_beat_range(args.beat_range) if args.beats else (60.0, 200.0)
+    if args.beats:
+        if not meta.get("audio"):
+            die("--beats needs an audio stream; this file has none", kind="input")
+        if args.beat_step <= 0:
+            die("--beat-step must be greater than 0", kind="input")
     dur = meta.get("duration") or 0.0
     cuts = detect_scenes(args.input, args.threshold, args.min_scene, dur, args.ratio)
     bounds = cuts + [dur]
     step_s = 0.5
-    env = audio_envelope(args.input, step_s) if meta.get("audio") else []
+    # With --beats the file is decoded once, at the finer rate, and both envelopes come from that
+    # one pass: the 0.5 s scene blocks are an exact multiple of the 10 ms onset blocks.
+    beat_rate = 22050
+    fine_samples = decode_pcm_mono(args.input, beat_rate, check=False) if (args.beats and meta.get("audio")) else None
+    if fine_samples is not None:
+        env = audio_envelope(args.input, step_s, rate=beat_rate, samples=fine_samples)
+    else:
+        env = audio_envelope(args.input, step_s) if meta.get("audio") else []
 
     scenes = []
     for i in range(len(bounds) - 1):
@@ -82,6 +128,34 @@ def main() -> int:
     result: Dict = {"file": args.input, "duration": round(dur, 3), "scene_count": len(scenes), "scenes": scenes, "audio_peaks": peaks}
     info(f"{len(scenes)} scenes, {len(peaks)} audio peaks over {dur:.1f}s")
 
+    if args.beats:
+        # A beat grid is a measurement of the music's periodicity, not a statement about where a
+        # cut belongs. scenes.py reports what it measured, including a low confidence: reporting a
+        # weak measurement is honest, and only a tool that CHANGES a file refuses to act on one.
+        fine = rms_envelope(fine_samples or [], max(1, int(round(beat_rate * args.beat_step))))
+        grid = beat_grid(fine, args.beat_step, bpm_range=beat_range,
+                         min_confidence=args.min_confidence, duration=dur)
+        result["beats"] = grid["beats"]
+        result["beat_grid"] = {
+            # The regular grid AND the subset a measured onset supports. A tool that MOVES
+            # something (cut.py --snap beats) may only use the subset; scenes.py reports both,
+            # because here the regular grid is the measurement being made.
+            "supported_beats": grid["supported_beats"],
+            "tempo_bpm": grid["tempo_bpm"], "interval": grid["interval"],
+            "confidence": grid["confidence"], "phase": grid["phase"],
+            "onsets": len(grid["onsets"]), "supported": grid["supported"],
+            "unsupported": grid["unsupported"], "method": grid["method"],
+            "step_s": grid["step_s"], "range_bpm": grid["range_bpm"], "usable": grid["usable"],
+        }
+        if grid["tempo_bpm"] is None:
+            info(f"--beats: no steady pulse in this audio (confidence {grid['confidence']:.2f}) -- "
+                 "speech, ambience or rubato has no tempo to measure")
+        else:
+            info(f"--beats: {grid['tempo_bpm']:.1f} BPM, {len(grid['beats'])} beats, confidence "
+                 f"{grid['confidence']:.2f} ({grid['supported']} of {len(grid['beats'])} grid points "
+                 f"have a measured onset)"
+                 + ("" if grid["usable"] else f" -- below --min-confidence {args.min_confidence}, usable: false"))
+
     if args.highlights:
         if args.rank_by == "duration":
             rank_key = lambda sc: (-sc["duration"], sc["start"])

package/scripts/silence.py

@@ -12,29 +12,168 @@ Examples:
   python3 silence.py talk.mp4 --edl keep.txt                  # also save the kept ranges (START-END per line, cut.py --segments format)
 """
 import argparse
+import json
 import os
 import sys
 from typing import List, Tuple
 
 # `detect` moved into _common/probe.py in 1.16.0 so metadata.py --auto-chapters can measure the
 # same silences without importing this tool; the body is unchanged and the name still lives here.
+from _common import (filler_spans, FILLER_WORDS, FILLER_AMBIGUOUS, FILLER_DISCOURSE_MARKERS,
+                     FILLER_PAD, transcribe_words, read_text_or_die)
 from _common import detect_silences as detect, STATE, video_args, add_common, apply_common, audio_codec_for, cfr_args, default_output, die, emit, ffmpeg_base, info, is_audio_output, print_json, probe, run, X264_PRESETS, measured_level_dbfs, fmt_secs
 
 
 
+def merge_spans(spans: List[Tuple[float, float]]) -> List[Tuple[float, float]]:
+    """`spans` sorted and coalesced: any pair that touches or overlaps becomes one.
+
+    keep_ranges() walks a single cursor forward, so it needs a removal list in which no span
+    starts before the previous one ended. Silences and filler spans are each merged only among
+    themselves -- and a mumbled "um" is very often quiet enough to sit INSIDE a detected silence --
+    so the union has to be taken before the two lists are handed over as one.
+    """
+    out: List[Tuple[float, float]] = []
+    for s, e in sorted(spans):
+        if out and s <= out[-1][1]:
+            out[-1] = (out[-1][0], max(out[-1][1], e))
+        else:
+            out.append((s, e))
+    return out
+
+
 def keep_ranges(silences: List[Tuple[float, float]], duration: float, margin: float, min_keep: float) -> List[Tuple[float, float]]:
     keeps: List[Tuple[float, float]] = []
     cursor = 0.0
-    for s, e in silences:
+    for s, e in sorted(silences):
         s_adj = max(cursor, s + margin)
         if s_adj - cursor >= min_keep:
             keeps.append((cursor, s_adj))
-        cursor = min(duration, e - margin) if e != float("inf") else duration
+        # max(cursor, ...) so the walk is monotone. A span nested inside the previous one used to
+        # rewind the cursor and hand back the very stretch that had just been removed: with a
+        # filler word inside a detected silence, adding --filler made the tool remove LESS.
+        # merge_spans() above is the caller-side fix; this keeps the function safe on its own.
+        cursor = max(cursor, min(duration, e - margin) if e != float("inf") else duration)
     if duration - cursor >= min_keep:
         keeps.append((cursor, duration))
     return keeps
 
 
+def _word_list(text: "str") -> "list":
+    return [w.strip() for w in str(text or "").replace(",", "\n").splitlines() if w.strip()]
+
+
+def resolve_filler(args, meta):
+    """(the `filler` result block, the spans to remove) for --filler. Refuses before any encode.
+
+    Never without measured word timings: no heuristic fallback, no guess from the filename. The
+    three refusals below are the whole safety story for this flag.
+    """
+    if not args.words and not args.transcribe:
+        die("--filler needs word timings: pass --words transcript.json (a whisper JSON with word "
+            "timestamps) or --transcribe. There is no way to find a filler word without them -- "
+            "cutting the short quiet blips instead would remove real speech.", kind="input")
+    source, engine, raw = None, None, None
+    if args.words:
+        try:
+            raw = json.loads(read_text_or_die(args.words, "--words"))
+        except ValueError as exc:
+            die(f"--words {args.words}: not readable JSON ({exc})", kind="input")
+        source = f"whisper-json:{args.words}"
+        words, had_segments = _words_from_transcript(raw)
+        if not words:
+            if had_segments:
+                die(f"the transcript in {args.words} has segment timings but no word timings; "
+                    "--filler removes words, and cutting on segment boundaries would remove whole "
+                    "sentences. Re-run whisper with word timestamps (whisper.cpp "
+                    "--output-json-full / faster-whisper word_timestamps=True), or use --list to "
+                    "see the pauses instead.", kind="input")
+            die(f"no word timings in {args.words}: --filler needs "
+                '{"words": [{"word": ..., "start": ..., "end": ...}]} (or the same inside '
+                '"segments").', kind="input")
+    else:
+        # No pre-check for an installed engine here: transcribe_words() probes for one and
+        # raises the same die_no_engine() refusal when there is none. Two places deciding "is
+        # whisper here" is two places to disagree.
+        # The engine is driven with ITS word-timestamp option (whisper.cpp --output-json-full,
+        # faster-whisper word_timestamps=True, openai-whisper --word_timestamps True). An SRT
+        # cannot answer this question: a cue has a start and an end, a word does not.
+        words, engine = transcribe_words(args.input, args.filler_lang if args.filler_lang != "auto" else None)
+        source = f"whisper:{engine}" if engine else "whisper"
+        if not words:
+            die(f"{engine or 'the local engine'} ran but produced no word-level timings, so there "
+                "is nothing for --filler to cut on. Some builds do not support word timestamps. "
+                "Re-run that engine yourself with them (whisper.cpp --output-json-full / "
+                "faster-whisper word_timestamps=True / openai-whisper --word_timestamps True) and "
+                "pass the result with --words.", kind="input")
+
+    lang = args.filler_lang
+    if lang == "auto":
+        lang = str((raw or {}).get("language") or "").lower()[:2] if isinstance(raw, dict) else ""
+        lang = lang if lang in FILLER_WORDS else "en"
+    listname = "builtin"
+    if args.filler_words:
+        wordlist = set(_word_list(read_text_or_die(args.filler_words, "--filler-words")))
+        listname = args.filler_words
+    else:
+        if lang not in FILLER_WORDS:
+            die(f"--filler-lang {lang}: no built-in filler list for that language. The languages "
+                f"with one are {', '.join(sorted(FILLER_WORDS))}; pass --filler-words FILE with "
+                "your own list for anything else.", kind="input")
+        wordlist = set(FILLER_WORDS[lang])
+    wordlist |= set(_word_list(args.filler_extra))
+    wordlist -= set(_word_list(args.filler_keep))
+
+    spans = filler_spans(words, wordlist, pad=args.filler_pad)
+    removed_words = sorted({s["word"] for s in spans})
+    warnings = []
+    ambiguous = sorted(set(FILLER_AMBIGUOUS.get(lang, ())) & set(
+        t for s in spans for t in s["word"].split()))
+    if ambiguous:
+        warnings.append(
+            f"removed {', '.join(ambiguous)} -- in {lang} these are as often ordinary words as "
+            f"fillers. Keep one with --filler-keep {ambiguous[0]} and re-run if a sentence lost "
+            "its meaning.")
+    if FILLER_DISCOURSE_MARKERS.get(lang):
+        extra_markers = sorted(set(_word_list(args.filler_extra))
+                               & set(FILLER_DISCOURSE_MARKERS[lang]))
+        if extra_markers:
+            warnings.append(f"--filler-extra {', '.join(extra_markers)}: a discourse marker, not a "
+                            "disfluency -- this will cut real sentences.")
+    block = {
+        "lang": lang, "source": source, "engine": engine,
+        "words": sorted(wordlist), "removed": [dict(s) for s in spans],
+        "removed_count": len(spans),
+        "removed_seconds": round(sum(s["end"] - s["start"] for s in spans), 3),
+        "word_timings": len(words), "list": listname, "removed_words": removed_words,
+        "warnings": warnings,
+    }
+    return block, [(s["start"], s["end"]) for s in spans]
+
+
+def _words_from_transcript(data):
+    """([{word,start,end}], whether the document had segments at all) from a whisper JSON."""
+    raw, had_segments = [], False
+    if isinstance(data, dict):
+        raw = list(data.get("words") or [])
+        segments = data.get("segments") or []
+        had_segments = bool(segments)
+        for seg in segments:
+            raw.extend((seg or {}).get("words") or [])
+    elif isinstance(data, list):
+        raw = list(data)
+    out = []
+    for w in raw:
+        if not isinstance(w, dict):
+            continue
+        try:
+            out.append({"word": str(w.get("word") or w.get("text") or ""),
+                        "start": float(w["start"]), "end": float(w["end"])})
+        except (KeyError, TypeError, ValueError):
+            continue
+    return out, had_segments
+
+
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument("input")
@@ -45,27 +184,90 @@ def main() -> int:
     ap.add_argument("--min-keep", type=float, default=0.2, help="drop kept pieces shorter than this (default 0.2)")
     ap.add_argument("--list", action="store_true", help="only print silences and the kept ranges")
     ap.add_argument("--edl", help="write the kept ranges to this file, one START-END per line")
+    fil = ap.add_argument_group("filler words (1.17)")
+    fil.add_argument("--filler", action="store_true",
+                     help="also remove filler words. Needs measured word timings: pass --words or "
+                          "--transcribe. There is no heuristic fallback -- a word is cut only where "
+                          "a speech engine timed it.")
+    fil.add_argument("--filler-lang", choices=["auto"] + sorted(FILLER_WORDS), default="auto",
+                     help="which built-in list to use (default auto: the transcript's language field)")
+    fil.add_argument("--filler-words", metavar="FILE",
+                     help="one word per line; replaces the built-in list for this run")
+    fil.add_argument("--filler-extra", metavar="W[,W...]",
+                     help="add words to the list. 'like', 'tipo' and 'cio\u00e8' live here rather than in "
+                          "the defaults: they are discourse markers, not disfluencies, and cutting "
+                          "them cuts real sentences.")
+    fil.add_argument("--filler-keep", metavar="W[,W...]",
+                     help="remove words from the built-in list (e.g. --filler-keep \u306a\u3093\u304b)")
+    fil.add_argument("--filler-pad", type=float, default=FILLER_PAD,
+                     help=f"seconds trimmed either side of a filler word (default {FILLER_PAD})")
+    fil.add_argument("--words", metavar="FILE",
+                     help="a whisper JSON with word-level timings, for --filler")
+    fil.add_argument("--transcribe", action="store_true",
+                     help="produce the word timings with a local whisper (never required; the same "
+                          "bridge caption.py uses)")
+    fil.add_argument("--filler-list", action="store_true",
+                     help="report what --filler would remove and write nothing")
+    fil.add_argument("--max-cuts", type=int, default=400,
+                     help="refuse above this many removal ranges: the filter graph grows with them "
+                          "(default 400)")
     ap.add_argument("--crf", type=int, default=18)
     ap.add_argument("--preset", default="medium", choices=X264_PRESETS)
     add_common(ap)
     args = ap.parse_args()
     apply_common(args)
 
+    if args.filler_pad < 0:
+        die("--filler-pad cannot be negative: a negative pad turns each word's span inside out "
+            "(end before start) and the span is then silently dropped, so nothing is removed",
+            kind="input")
+    if args.max_cuts < 1:
+        die("--max-cuts must be at least 1", kind="input")
+    if args.filler_list and not args.filler:
+        die("--filler-list reports what --filler would remove: pass --filler as well", kind="input")
     meta = probe(args.input)
     if not meta.get("audio"):
         die("input has no audio stream to analyse")
     duration = meta.get("duration") or 0.0
     silences = detect(args.input, args.threshold, args.min_silence)
-    keeps = keep_ranges(silences, duration, args.margin, args.min_keep)
+    filler_info, filler_ranges = resolve_filler(args, meta) if args.filler else (None, [])
+    # One sorted, merged removal list through the graph the tool already has: filler removal IS
+    # time-range removal, so it reuses keep_ranges() and the same aselect/concat chain.
+    removals = merge_spans(list(silences) + list(filler_ranges))
+    keeps = keep_ranges(removals, duration, args.margin, args.min_keep)
     kept = sum(e - s for s, e in keeps)
     removed = max(0.0, duration - kept)
+    # `removed_seconds` keeps the meaning it has had since this tool existed: the seconds of
+    # SILENCE this run removes. It must not quietly start counting filler time as well, because a
+    # caller that has been reading it since 1.0 asked how much dead air went. The silence-only
+    # figure is the one the same run would have reported without --filler, so it is computed from
+    # the silences alone; everything removed is `removed_seconds_total`.
+    silence_only = removed
+    if args.filler:
+        silence_keeps = keep_ranges(merge_spans(list(silences)), duration, args.margin, args.min_keep)
+        silence_only = max(0.0, duration - sum(e - s for s, e in silence_keeps))
     summary = {
         "silences": [[round(s, 3), None if e == float("inf") else round(e, 3)] for s, e in silences],
         "keep": [[round(s, 3), round(e, 3)] for s, e in keeps],
         "input_duration": round(duration, 3),
         "kept_duration": round(kept, 3),
-        "removed_seconds": round(removed, 3),
+        "removed_seconds": round(silence_only, 3),
     }
+    if filler_info is not None:
+        # removed_seconds above is the silence-only figure, unchanged in meaning; the filler share
+        # is reported inside `filler`, and removed_seconds_total is the additive sibling that
+        # covers everything this run took out.
+        summary["filler"] = filler_info
+        summary["removed_seconds_total"] = round(removed, 3)
+        info(f"--filler: {filler_info['removed_count']} filler word(s), "
+             f"{filler_info['removed_seconds']:.2f}s, from {filler_info['word_timings']} word timings "
+             f"({filler_info['lang']}, list {filler_info['list']})")
+        for warning in filler_info.get("warnings") or []:
+            info("warning: " + warning)
+    if len(keeps) > args.max_cuts:
+        die(f"{len(keeps)} keep ranges is above --max-cuts {args.max_cuts}: the filter graph grows "
+            "with every range and a graph this size is slow and fragile. Raise --max-cuts if you "
+            "mean it, or use --min-silence/--filler-pad to merge the short ones.", kind="input")
     info(f"{len(silences)} silences, keeping {len(keeps)} ranges: {kept:.2f}s of {duration:.2f}s (removing {removed:.2f}s)")
     if not silences and not (STATE.dry_run and not os.path.exists(args.input)):
         # Nothing under the threshold is a valid result, not a failure -- but an agent that only
@@ -88,7 +290,7 @@ def main() -> int:
                     fh.write(f"{s:.3f}-{e:.3f}\n")
         info(f"wrote {args.edl}")
 
-    if args.list:
+    if args.list or args.filler_list:
         if args.json:
             emit(None, **summary)
         else:

package/scripts/verify.py

@@ -81,7 +81,7 @@ def step(name: str, argv: List[str], timeout: float) -> Dict:
             STATE.json = was_json
         return {"step": name, "ok": ok, "seconds": round(time.time() - t0, 1), "error": err}
     try:
-        proc = subprocess.run([sys.executable, str(HERE / argv[0])] + argv[1:], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=timeout)
+        proc = subprocess.run([sys.executable, str(HERE / argv[0])] + argv[1:], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace", timeout=timeout)
         ok = proc.returncode == 0
         err = "" if ok else (proc.stderr.strip().splitlines() or ["?"])[-1][:200]
     except subprocess.TimeoutExpired:

package/scripts/waveform.py

@@ -274,8 +274,7 @@ def _child(script_name: str, argv: "list") -> None:
     if STATE.dry_run:
         cmd.append("--dry-run")
     info("-> " + " ".join(os.path.basename(c) if c.endswith(".py") else str(c) for c in cmd[1:]))
-    proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
-                          encoding="utf-8", errors="replace")
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace")
     if proc.returncode != 0:
         die(f"{script_name} failed:\n{(proc.stderr or proc.stdout).strip()[-800:]}", kind="ffmpeg")