ffmpeg-skill 2.6.0 → 2.7.0

package/scripts/_contract.py

@@ -54,6 +54,8 @@ ROLES = {
 #   filter:<name>               `ffmpeg -filters`
 #   bsf:<name>                  `ffmpeg -bsfs`
 #   external:whisper            a local whisper engine (whisper.cpp / faster-whisper / openai-whisper)
+#   external:parakeet           a local Parakeet engine (parakeet-mlx, or parakeet-cli with a .gguf model);
+#                               English. Either external engine satisfies --transcribe.
 # "optional" entries name the flag or condition under which the capability is needed.
 FF = ["ffmpeg", "ffprobe"]
 X264 = "encoder:libx264"
@@ -145,7 +147,7 @@ TOOL_META: Dict[str, Dict[str, Any]] = {
                       required=FF + [X264], optional=[],
                       video_required=True, audio_only=False, visual=True, verify=["probe", "look"], produces_artifact=True, idempotency="content_equivalent", deterministic=True),
     "caption": dict(role="execution", inputs=["video asset", "SRT/ASS file or timed text (--text)"], outputs=["video artifact with burnt-in captions (--mode burn)", "video artifact with one or several language-tagged soft subtitle streams (--mode mux, --srt file:lang repeated)", "generated .srt / .ass sidecar"],
-                    required=FF + [X264, AAC, "filter:subtitles"], optional=[{"capability": "filter:ass", "when": "--animate / --karaoke"}, HDR_X265, {"capability": "external:whisper", "when": "--transcribe"},
+                    required=FF + [X264, AAC, "filter:subtitles"], optional=[{"capability": "filter:ass", "when": "--animate / --karaoke"}, HDR_X265, {"capability": "external:whisper", "when": "--transcribe"}, {"capability": "external:parakeet", "when": "--transcribe (English; either engine family satisfies it)"},
                                             {"capability": "encoder:mov_text", "when": "--mode mux with a .mp4/.m4v/.mov output"}, {"capability": "encoder:webvtt", "when": "--mode mux with a .webm output"}, {"capability": "encoder:srt", "when": "--mode mux with a .mkv output"}],
                     video_required=True, audio_only=False, visual=True, verify=["probe", "look"], produces_artifact=True, idempotency="content_equivalent", deterministic=True),
     "overlay": dict(role="execution", inputs=["video asset", "image (--image / --logo), text (--text), or a second video (--video) to composite"], outputs=["video artifact with the overlay composited"],
@@ -172,7 +174,8 @@ TOOL_META: Dict[str, Dict[str, Any]] = {
                      video_required=False, audio_only=True, visual=False, verify=["probe", "check"], produces_artifact=True, idempotency="content_equivalent", deterministic=True),
     "silence": dict(role="analysis_and_execution", inputs=["video or audio asset"], outputs=["silence list JSON (--list)", "artifact with silences removed", "EDL text (--edl)"],
                     required=FF + ["filter:silencedetect"], optional=[{"capability": X264, "when": "removing silences from a video"}, HDR_X265, {"capability": AAC, "when": "removing silences from a video"},
-                                                                    {"capability": "external:whisper", "when": "--filler --transcribe"}] + AUDIO_OUT,
+                                                                    {"capability": "external:whisper", "when": "--filler --transcribe"},
+                                                                    {"capability": "external:parakeet", "when": "--filler --transcribe (English; either engine family satisfies it)"}] + AUDIO_OUT,
                     video_required=False, audio_only=True, visual=False, verify=["probe"], produces_artifact=True, idempotency="content_equivalent", deterministic=True),
     "join": dict(role="execution", inputs=["two or more video assets, or two or more audio-only assets"], outputs=["concatenated video artifact", "concatenated audio artifact (audio-only inputs, audio output extension)"],
                  required=FF + ["filter:xfade", "filter:acrossfade"],
@@ -409,7 +412,8 @@ def output_schema(name: str, meta: Dict[str, Any], hw: bool = False) -> Dict[str
                  "notes": {"type": "array", "items": {"type": "string"}, "description": "present when no --platform was named: youtube was assumed and judgement rows are WARN"},
                  "checks": {"type": "array", "items": {"type": "object", "properties": {"check": {"type": "string", "description": "row name; --content adds black, frozen and silence; audio FAILs when the track is present but silent (peak <= -50 dBFS)"}, "status": {"enum": ["PASS", "WARN", "FAIL"]}, "value": {}, "expected": {}, "fix": {"type": "string"}, "kind": {"enum": ["format", "judgement"]}}}}}
     elif name == "caption":
-        extra = {"caption": {"type": "object", "description": "cue layout: shifted / wrapped / rebalanced / split / extended / dropped counts, plus wrap ('phrase' or 'measured'), phrase_breaks (1.16), broken_inside_word -- atoms hard-sliced at the column edge because they did not fit alone even at the size floor (1.18.4) -- and overlong -- now residual: a single character alone wider than the column. Burn mode: cues_burned -- non-blank cues that overlap [0, duration] and are drawn -- and cues_outside -- non-blank cues wholly outside it (2.2.6); a burn with no visible cue is refused, kind input"},
+        extra = {"transcription": {"type": "object", "description": "with --transcribe: {engine, model, language, routing (why auto chose it), detected_language when auto detected it}; when auto ran the English-only Parakeet model on English assumed, notes says so"},
+                 "caption": {"type": "object", "description": "cue layout: shifted / wrapped / rebalanced / split / extended / dropped counts, plus wrap ('phrase' or 'measured'), phrase_breaks (1.16), broken_inside_word -- atoms hard-sliced at the column edge because they did not fit alone even at the size floor (1.18.4) -- and overlong -- now residual: a single character alone wider than the column. Burn mode: cues_burned -- non-blank cues that overlap [0, duration] and are drawn -- and cues_outside -- non-blank cues wholly outside it (2.2.6); a burn with no visible cue is refused, kind input"},
                  "tracks": {"type": "array", "description": "--mode mux: one entry per subtitle stream in the output ({index, file, language, title, codec, default, cues, kept_from_input}); a stream the input already carried has file null and kept_from_input true (1.16)"},
                  "subtitle_tracks": {"type": "integer", "description": "--mode mux: how many subtitle streams the output carries"},
                  "emoji": {"type": "object", "description": "how the emoji in the text were drawn (mode, overlays, missing)"},
@@ -430,7 +434,9 @@ def output_schema(name: str, meta: Dict[str, Any], hw: bool = False) -> Dict[str
                  "speech": {"type": "array", "description": "--speech (1.18): [{time, speech_music_ratio}]"}}
     elif name == "silence":
         extra = {"silences": {"type": "array"}, "keep": {"type": "array"}, "input_duration": {"type": "number"}, "kept_duration": {"type": "number"}, "removed_seconds": {"type": "number"},
-                 "speech_aware": {"type": "object", "description": "--speech-aware (1.18): {min_silence, floor, breaths_kept, breaths_kept_seconds, breaths}"}}
+                 "speech_aware": {"type": "object", "description": "--speech-aware (1.18): {min_silence, floor, breaths_kept, breaths_kept_seconds, breaths}"},
+                 "filler": {"type": "object", "description": "--filler: {lang, source (whisper-json:FILE | whisper:ENGINE | parakeet:ENGINE), engine, transcription (with --transcribe: {engine, model, language, routing}), removed, removed_count, ...}"},
+                 "notes": {"type": "array", "items": {"type": "string"}, "description": "--filler --transcribe under --engine auto: present when the English-only Parakeet model ran on English assumed (no --filler-lang, no language detected, no Whisper engine installed or none that succeeded); the transcript is wrong if the speech is not English"}}
     elif name == "sync":
         extra = {"reference": {"type": "string"}, "second": {"type": "string"}, "offset_seconds": {"type": "number"}, "confidence": {"type": "number"}, "meaning": {"type": "string"}, "drift": {"type": "object"},
                  "sources": {"type": "array", "description": "1.18: [{path, offset_s, confidence, drift_ppm}], one per SOURCE; the only per-source shape once more than one SOURCE is given"}}
@@ -672,6 +678,13 @@ def _whisper_available() -> bool:
     return importlib.util.find_spec("faster_whisper") is not None or importlib.util.find_spec("whisper") is not None
 
 
+def _parakeet_available() -> bool:
+    """parakeet-mlx on PATH, or parakeet-cli with a model it can find (PARAKEET_CPP_MODEL or a
+    .gguf in ~/.cache/parakeet.cpp) -- the same test the speech bridge applies before running it."""
+    from _common.asr import _parakeet_available as runtime_check  # the bridge's own test, not a copy of it
+    return any(runtime_check(e, shutil) for e in ("parakeet-mlx", "parakeet.cpp"))
+
+
 def _hw_default() -> Dict[str, Any]:
     """--hw's machine facts, normalised (never the raw environment value): whether VideoToolbox
     constant-quality encoding can run here (the tools' own check: Apple Silicon hardware, also
@@ -870,6 +883,8 @@ def doctor() -> Dict[str, Any]:
             state[cap] = _from("bsfs", cap[4:])
         elif cap == "external:whisper":
             state[cap] = "available" if _whisper_available() else "missing"
+        elif cap == "external:parakeet":
+            state[cap] = "available" if _parakeet_available() else "missing"
         else:
             state[cap] = "missing"
     available = sorted(c for c, st in state.items() if st == "available")
@@ -997,7 +1012,15 @@ def _capability_fix_hint(cap: str) -> str:
         return (f"no installed font covers {LANGUAGE_NAMES.get(script, script)} text on this machine; "
                 f"{FONT_INSTALL_HINT} (doctor --json .fonts.scripts lists every script)")
     if cap == "external:whisper":
-        return "install a local whisper (whisper-cli, whisper-cpp, faster-whisper or openai-whisper) for --transcribe"
+        return ("install a local whisper (whisper-cli, whisper-cpp, faster-whisper or openai-whisper) for --transcribe; "
+                "faster-whisper and openai-whisper download their model on first use, and every engine transcribes "
+                "on this machine (the audio is never uploaded)")
+    if cap == "external:parakeet":
+        from _common.asr import PARAKEET_MLX_DEFAULT_MODEL
+        return ("for English --transcribe, install parakeet-mlx (uv tool install parakeet-mlx; its first run downloads "
+                f"{PARAKEET_MLX_DEFAULT_MODEL} from Hugging Face, then it transcribes on this machine and the audio is "
+                "never uploaded) or parakeet-cli with a tdt-0.6b-v2 .gguf in ~/.cache/parakeet.cpp; a local whisper "
+                "covers every language")
     return f"'{cap}' is not available; see docs/contract.md"
 
 
@@ -1362,7 +1385,7 @@ def build(detect: bool = True) -> Dict[str, Any]:
     version = skill_version()
     tools = [tool_spec(n, version) for n in public_tools()]
     wanted = required_capabilities()
-    caps: Dict[str, Any] = {"required": wanted["required"], "optional": wanted["optional"], "naming": "ffmpeg | ffprobe | encoder:<name> | filter:<name> | bsf:<name> | external:whisper"}
+    caps: Dict[str, Any] = {"required": wanted["required"], "optional": wanted["optional"], "naming": "ffmpeg | ffprobe | encoder:<name> | filter:<name> | bsf:<name> | external:whisper | external:parakeet"}
     if detect:
         d = doctor()
         caps.update({"available": d["available"], "missing": d["missing"], "missing_optional": d["missing_optional"],
@@ -1389,8 +1412,21 @@ def build(detect: bool = True) -> Dict[str, Any]:
         "execution": {
             "shell": False,
             "arbitrary_executables": False,
-            "subprocess": "argv list only: [python3, scripts/<tool>.py, ...] and [ffmpeg|ffprobe, ...] resolved from PATH",
+            "subprocess": "argv list only: [python3, scripts/<tool>.py, ...] and [ffmpeg|ffprobe, ...] resolved from PATH; "
+                          "with --transcribe, a local speech engine's CLI (whisper.cpp's whisper-cli / whisper-cpp / main, "
+                          "openai-whisper's whisper, parakeet-mlx, parakeet-cli) resolved from PATH",
             "network": False,
+            # `network` is about the skill's own code. A speech engine may fetch its model weights the
+            # first time it runs; faster-whisper is a library, so its fetch is made from the skill's own
+            # process. A caller that sandboxes the network must know that before a first --transcribe.
+            "model_downloads": {
+                "when": "first use of a speech engine (--transcribe) whose model is not cached yet; media is never uploaded",
+                "in_process": {"faster-whisper": "Hugging Face (Systran/faster-whisper-<model>)"},
+                "child_process": {"parakeet-mlx": "Hugging Face (mlx-community/parakeet-*)",
+                                  "openai-whisper": "OpenAI (its model checkpoint)"},
+                "none": ["whisper.cpp", "parakeet.cpp"],
+                "offline": "HF_HUB_OFFLINE=1 keeps faster-whisper and parakeet-mlx on the cached copy",
+            },
             "input_mutation": False,
         },
         "invocation": {

package/scripts/caption.py

@@ -49,6 +49,8 @@ from _common import emoji_filter_chain, EMOJI_ASSET_HINT, emoji_asset_for, emoji
 # The ASR bridge and the SRT reader/writer live in _common.asr since 1.17 (silence.py --filler
 # shares them). They stay caption.py's public names -- every caller and test that reached for
 # caption.parse_srt / caption.transcribe / caption.whisper_word_timings before 1.17 still does.
+from _common import asr as _asr
+from _common.asr import ENGINE_CHOICES
 from _common import (ASR_INSTALL_HINT, die_no_engine, parse_srt, transcribe, whisper_word_timings,
                      write_srt)
 from _common import (SAFE_WIDTH_FRACTION, ORPHAN_MIN_EM, WRAP_MODES, wrap_text, wrap_variants, best_break,
@@ -901,7 +903,7 @@ def main() -> int:
                           "them; .mkv is the honest multi-track container and stores the code you give verbatim")
     src.add_argument("--ass", help="ASS file to burn (styles inside the file are used)")
     src.add_argument("--text", help="plain text cue file to convert into SRT (see format above)")
-    src.add_argument("--transcribe", action="store_true", help="generate the SRT from the audio with a local speech-to-text engine if one is installed (whisper-cli / whisper / faster-whisper); never required")
+    src.add_argument("--transcribe", action="store_true", help="generate the SRT from the audio with a local speech-to-text engine if one is installed (parakeet-mlx / parakeet.cpp / whisper-cli / whisper / faster-whisper); never required")
     src.add_argument("--language", "--lang", help="language code (e.g. en, ja, zh, ko): the language for --transcribe (default auto), "
                                                   "the tag on the subtitle stream with --mode mux, and the hint that says whether Han-only "
                                                   "text is Chinese, Japanese or Korean when a font is picked by script")
@@ -914,7 +916,12 @@ def main() -> int:
                           "(default: none, so no player burns in a language the viewer did not ask for)")
     src.add_argument("--offset", default="0", help="shift every cue by TIME (seconds, mm:ss, hh:mm:ss.ms or "
                                                     "hh:mm:ss:ff; a leading - shifts earlier); works for --text, --srt and --ass")
-    src.add_argument("--model", default="base", help="whisper model name/path for --transcribe (default base)")
+    src.add_argument("--model", default="base",
+                     help="speech model for --transcribe: a whisper model name/path (default base), or a Parakeet "
+                          "model (an mlx-community/parakeet-* repo or a .gguf) for the Parakeet engines")
+    src.add_argument("--engine", choices=ENGINE_CHOICES, default=None,
+                     help="speech engine for --transcribe (default: $FFMPEG_SKILL_ASR_ENGINE, else auto = Parakeet "
+                          "for English speech when installed, else whisper.cpp / faster-whisper / openai-whisper)")
     src.add_argument("--write-srt", help="where to save the generated SRT (default: <text>.srt)")
     src.add_argument("--auto-seconds", type=float, default=3.0, help="duration for cues without timing (default 3)")
     src.add_argument("--gap", type=float, default=0.0, help="gap after auto-timed cues in seconds")
@@ -1241,8 +1248,13 @@ def main() -> int:
             cues = []
             info(f"[dry-run] would transcribe {args.input} and write {srt_path}")
         else:
-            cues = transcribe(args.input, srt_path, args.language, args.model, args.audio_stream)
-            args._word_timings = whisper_word_timings(srt_path)
+            # the call's own result, not module state: engine, routing and words belong to this run
+            asr_run = _asr.transcribe_result(args.input, srt_path, args.language, args.model, args.audio_stream,
+                                             engine=args.engine)
+            cues = asr_run.cues
+            args._asr = dict(asr_run.facts)
+            args._asr_notes = list(asr_run.notes)
+            args._word_timings = whisper_word_timings(srt_path) or sorted((w["start"], w["end"], w["word"]) for w in asr_run.words)
             cues, changed = lay_out(cues)
             if changed:
                 write_srt(cues, srt_path)
@@ -1279,7 +1291,8 @@ def main() -> int:
     # WRITING waits for a real run (the rule every side file in this tool follows), which is why
     # the cues are kept in hand below for the font sample and the ASS generator.
     planned_cues: Optional[List[Tuple[float, float, str]]] = None
-    side_notes: List[str] = []
+    # an English-only transcript made on an assumption says so first (asr.assumed_english_note)
+    side_notes: List[str] = list(getattr(args, "_asr_notes", None) or [])
     ass_sample_path = args.ass
     if args.srt and not (args.text or args.transcribe) and os.path.exists(srt_path or ""):
         adjusted, changed = lay_out(parse_srt(srt_path))
@@ -1416,7 +1429,8 @@ def main() -> int:
         result = probe(output, role="output")
         info(f"wrote {output} ({fmt_secs(result.get('duration'))}, mux, {len(added)} "
              f"subtitle track(s) added, codec {codec})")
-        emit(output, tracks=tracks, subtitle_tracks=total, **({"notes": notes} if notes else {}))
+        emit(output, tracks=tracks, subtitle_tracks=total, **({"notes": notes} if notes else {}),
+             **({"transcription": args._asr} if getattr(args, "_asr", None) else {}))
         return 0
 
     # A font that covers the text, before anything is rendered: non-Latin cues in a Latin-only
@@ -1582,6 +1596,8 @@ def main() -> int:
         info("caption text unchanged: the cues were burned exactly as given (line breaks, timing "
              "and type size only)")
     extra["caption"] = dict(caption_stats)
+    if getattr(args, "_asr", None):
+        extra["transcription"] = args._asr
     if emoji_plan:
         notes = list(extra.get("notes") or [])
         if emoji_plan["mode"] == "mono":

package/scripts/silence.py

@@ -20,7 +20,8 @@ from typing import List, Tuple
 # `detect` moved into _common/probe.py in 1.16.0 so metadata.py --auto-chapters can measure the
 # same silences without importing this tool; the body is unchanged and the name still lives here.
 from _common import (filler_spans, FILLER_WORDS, FILLER_AMBIGUOUS, FILLER_DISCOURSE_MARKERS,
-                     FILLER_PAD, transcribe_words, read_text_or_die)
+                     FILLER_PAD, read_text_or_die)
+from _common import asr as _asr
 from _common import detect_silences as detect, STATE, video_args, add_common, apply_common, audio_codec_for, cfr_args, default_output, die, emit, ffmpeg_base, info, is_audio_output, print_json, probe, run, X264_PRESETS, measured_level_dbfs, fmt_secs
 
 
@@ -99,7 +100,7 @@ def resolve_filler(args, meta):
         die("--filler needs word timings: pass --words transcript.json (a whisper JSON with word "
             "timestamps) or --transcribe. There is no way to find a filler word without them -- "
             "cutting the short quiet blips instead would remove real speech.", kind="input")
-    source, engine, raw = None, None, None
+    source, engine, raw, transcription = None, None, None, {}
     if args.words:
         try:
             raw = json.loads(read_text_or_die(args.words, "--words"))
@@ -118,14 +119,20 @@ def resolve_filler(args, meta):
                 '{"words": [{"word": ..., "start": ..., "end": ...}]} (or the same inside '
                 '"segments").', kind="input")
     else:
-        # No pre-check for an installed engine here: transcribe_words() probes for one and
+        # No pre-check for an installed engine here: transcribe_words_result() probes for one and
         # raises the same die_no_engine() refusal when there is none. Two places deciding "is
         # whisper here" is two places to disagree.
         # The engine is driven with ITS word-timestamp option (whisper.cpp --output-json-full,
         # faster-whisper word_timestamps=True, openai-whisper --word_timestamps True). An SRT
         # cannot answer this question: a cue has a start and an end, a word does not.
-        words, engine = transcribe_words(args.input, args.filler_lang if args.filler_lang != "auto" else None)
-        source = f"whisper:{engine}" if engine else "whisper"
+        # The call's own result, not module state: engine, routing and notes belong to this run.
+        asr_run = _asr.transcribe_words_result(args.input, args.filler_lang if args.filler_lang != "auto" else None,
+                                               engine=args.engine)
+        words, engine = asr_run.words, asr_run.engine
+        transcription = dict(asr_run.facts)
+        args._asr_notes = list(asr_run.notes)
+        family = "parakeet" if engine in _asr.PARAKEET_ENGINES else "whisper"
+        source = f"{family}:{engine}" if engine else "whisper"
         if not words:
             die(f"{engine or 'the local engine'} ran but produced no word-level timings, so there "
                 "is nothing for --filler to cut on. Some builds do not support word timestamps. "
@@ -168,6 +175,7 @@ def resolve_filler(args, meta):
                             "disfluency -- this will cut real sentences.")
     block = {
         "lang": lang, "source": source, "engine": engine,
+        **({"transcription": transcription} if engine and transcription else {}),
         "words": sorted(wordlist), "removed": [dict(s) for s in spans],
         "removed_count": len(spans),
         "removed_seconds": round(sum(s["end"] - s["start"] for s in spans), 3),
@@ -235,8 +243,11 @@ def main() -> int:
     fil.add_argument("--words", metavar="FILE",
                      help="a whisper JSON with word-level timings, for --filler")
     fil.add_argument("--transcribe", action="store_true",
-                     help="produce the word timings with a local whisper (never required; the same "
+                     help="produce the word timings with a local speech engine, Parakeet or whisper (never required; the same "
                           "bridge caption.py uses)")
+    fil.add_argument("--engine", choices=_asr.ENGINE_CHOICES, default=None,
+                     help="speech engine for --transcribe (default: $FFMPEG_SKILL_ASR_ENGINE, else auto = Parakeet "
+                          "for English speech when installed, else whisper)")
     fil.add_argument("--filler-list", action="store_true",
                      help="report what --filler would remove and write nothing")
     fil.add_argument("--max-cuts", type=int, default=400,
@@ -311,6 +322,10 @@ def main() -> int:
              f"({filler_info['lang']}, list {filler_info['list']})")
         for warning in filler_info.get("warnings") or []:
             info("warning: " + warning)
+    if getattr(args, "_asr_notes", None):
+        # an English-only transcript made on an assumption (asr.assumed_english_note): the
+        # bridge already logged the warning; the result document carries it at the top level
+        summary["notes"] = list(args._asr_notes)
     if len(keeps) > args.max_cuts:
         die(f"{len(keeps)} keep ranges is above --max-cuts {args.max_cuts}: the filter graph grows "
             "with every range and a graph this size is slow and fragile. Raise --max-cuts if you "