ffmpeg-skill 2.8.7 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/docs/contract.md +7 -3
- package/package.json +1 -1
- package/references/scripts.md +23 -4
- package/scripts/_common/asr.py +41 -11
- package/scripts/_common/timeline.py +1 -1
- package/scripts/caption.py +4 -3
- package/scripts/render.py +47 -4
- package/scripts/silence.py +4 -1
package/README.md
CHANGED
|
@@ -82,7 +82,7 @@ An agent that "knows FFmpeg" still guesses: it assumes a frame rate, picks a cod
|
|
|
82
82
|
- **Structured tools, not shell strings.** Each operation is a script with typed arguments. Nothing runs through a shell; no filter graph is accepted from the caller.
|
|
83
83
|
- **A contract the agent can read.** `contract --json` states, for every tool, what it takes, what it writes, which FFmpeg components it needs and how the result is verified. The MCP surface is derived from it.
|
|
84
84
|
- **Verification after execution.** The result is probed, checked against the destination's spec and, when the picture changed, looked at as a contact sheet.
|
|
85
|
-
- **Local first.** No cloud, no API keys, no Python dependencies. Optional local transcription is used when a speech engine (Parakeet or a whisper) is installed, never required — by `caption.py --transcribe` and, since 1.17, by `silence.py --filler --transcribe`; both take a transcript you already have instead, and both refuse with the install lines rather than guessing. The engine runs on your machine and the audio is never uploaded; parakeet-mlx, faster-whisper and openai-whisper download their model weights the first time they run (parakeet-mlx's default from Hugging Face).
|
|
85
|
+
- **Local first.** No cloud, no API keys, no Python dependencies. Optional local transcription is used when a speech engine (Parakeet or a whisper) is installed, never required — by `caption.py --transcribe` and, since 1.17, by `silence.py --filler --transcribe`; both take a transcript you already have instead, and both refuse with the install lines rather than guessing. Both take `--engine` and `--model`; `FFMPEG_SKILL_ASR_ENGINE` and `FFMPEG_SKILL_ASR_MODEL` set this machine's defaults (e.g. `large-v3-turbo` where that whisper.cpp model is installed; without either, auto and `base`). The engine runs on your machine and the audio is never uploaded; parakeet-mlx, faster-whisper and openai-whisper download their model weights the first time they run (parakeet-mlx's default from Hugging Face).
|
|
86
86
|
|
|
87
87
|
## Quick start
|
|
88
88
|
|
|
@@ -257,7 +257,7 @@ These are the rules the skill file gives the agent and the code enforces.
|
|
|
257
257
|
|
|
258
258
|
| Tool | What it does |
|
|
259
259
|
|---|---|
|
|
260
|
-
| `render.py` | Render a whole edit from a declarative `project.json` (clips, transitions, captions, overlays including the social sticker/hook/meme graphics, music and stem levels, loudness, export, chapter markers, check); `--init`, `--dry-run`, `--stop-after`, `--cache DIR`/`--from STAGE` (reuse the stages that did not change); the captions block takes the fit-size policy (`fit_size`, `min_size`, `fit_size_scope`) and the run reports the caption stage's counts as `caption`; `--template NAME INPUT` renders a shipped delivery template (`--template all` writes the whole social pack plus its table); `--export-timeline edit.fcpxml\|.edl\|.otio` (2.1) writes the cut as an editor timeline for Final Cut, DaVinci Resolve or Premiere instead of rendering it — clips, speed, dissolves, the music bed and chapter markers carry over, and `not_exported` names what stays behind |
|
|
260
|
+
| `render.py` | Render a whole edit from a declarative `project.json` (clips, transitions, captions, overlays including the social sticker/hook/meme graphics and a picture-in-picture or chroma-keyed `video` layer sized for the final frame, music and stem levels, loudness, export, chapter markers, check); `--init`, `--dry-run`, `--stop-after`, `--cache DIR`/`--from STAGE` (reuse the stages that did not change); the captions block takes the fit-size policy (`fit_size`, `min_size`, `fit_size_scope`) and the run reports the caption stage's counts as `caption`; `--template NAME INPUT` renders a shipped delivery template (`--template all` writes the whole social pack plus its table); `--export-timeline edit.fcpxml\|.edl\|.otio` (2.1) writes the cut as an editor timeline for Final Cut, DaVinci Resolve or Premiere instead of rendering it — clips, speed, dissolves, the music bed and chapter markers carry over, and `not_exported` names what stays behind |
|
|
261
261
|
| `batch.py` | Apply a step recipe or a project to a folder with a content-hash cache; `--watch`; `--jobs N` processes several files at once under one shared `--timeout` |
|
|
262
262
|
| `multicam.py` | Align any number of cameras and recorders by audio (with drift correction) and cut between them from a switch list; `--switch energy` auto-cuts to whichever camera is loudest/talking instead of a hand-built list, with `--min-shot` (minimum shot length) and `--edl` (write a plain START-END cut list, not a CMX EDL; for Premiere/Resolve use `--write-project` then `render.py --export-timeline`) |
|
|
263
263
|
| `verify.py` | Run the toolchain on real device files and report PASS / FAIL per step |
|
package/docs/contract.md
CHANGED
|
@@ -21,7 +21,7 @@ The contract is derived from the code that runs, not maintained beside it:
|
|
|
21
21
|
| Field | Meaning | Changes when |
|
|
22
22
|
|---|---|---|
|
|
23
23
|
| `contract_version` | shape of this document (`1.0`) | a key is renamed, removed or changes meaning |
|
|
24
|
-
| `skill.version` | the npm / package.json version (`2.
|
|
24
|
+
| `skill.version` | the npm / package.json version (`2.10.0`) | any release |
|
|
25
25
|
|
|
26
26
|
A release that adds a tool or a flag keeps `contract_version`; a breaking change to the
|
|
27
27
|
ToolSpec shape bumps it. Consumers pin on `contract_version` and read `skill.version`
|
|
@@ -119,7 +119,7 @@ JSON would have been rewritten for that. The flat keys, typed per tool in `outpu
|
|
|
119
119
|
"contract_version": "1.0",
|
|
120
120
|
"deprecated": [],
|
|
121
121
|
"removed": [{"what": "...", "since": "1.10.0", "replacement": "...", "removed_in": "2.0.0", "where": "cli | json | mcp | behaviour"}],
|
|
122
|
-
"skill": {"id": "ffmpeg-skill", "version": "2.
|
|
122
|
+
"skill": {"id": "ffmpeg-skill", "version": "2.10.0", "execution_mode": "local", "kind": "execution",
|
|
123
123
|
"entrypoints": {"cli": "...", "mcp": "...", "contract": "...", "doctor": "..."},
|
|
124
124
|
"not_provided": ["AI reasoning", "decisions", "production plans", "project IR", "approvals", "network access", "transcription engine"]},
|
|
125
125
|
"requirements": {"python": ">=3.9 (standard library only)", "ffmpeg": ">=5.0", "ffprobe": ">=5.0"},
|
|
@@ -378,7 +378,11 @@ engine present. Whisper is never a dependency of this skill.
|
|
|
378
378
|
`external:parakeet` is the same kind of optional capability for the same two flags:
|
|
379
379
|
parakeet-mlx, or parakeet-cli with a `.gguf` model it can find. Either engine family
|
|
380
380
|
satisfies `--transcribe`; `--engine auto` (the default, or `FFMPEG_SKILL_ASR_ENGINE`) runs
|
|
381
|
-
Parakeet for English speech and Whisper for every other language.
|
|
381
|
+
Parakeet for English speech and Whisper for every other language. `--model` (both tools) is the
|
|
382
|
+
model; without it `FFMPEG_SKILL_ASR_MODEL` is this machine's default, else `base`. An explicit
|
|
383
|
+
`--model` goes to whichever engine runs; a Parakeet model in the variable goes to the Parakeet
|
|
384
|
+
engines only, and the Whisper engines keep `base`. `doctor` reports neither variable (it prints no
|
|
385
|
+
environment values); the result's `transcription.model` names the model that ran. When no `--language` is given
|
|
382
386
|
and no detector can name it, a Whisper engine that can run takes it (whisper.cpp is passed
|
|
383
387
|
`-l auto`: its own default is English). Parakeet then runs only on English *assumed* -- when no
|
|
384
388
|
Whisper engine is installed, or every one failed, or `--model` names an English-only Parakeet
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ffmpeg-skill",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.10.0",
|
|
4
4
|
"description": "Agent Skill that gives coding agents (Claude Code, Cursor, Codex) a local video editor: 42 FFmpeg tools with a machine-readable contract, contract-derived MCP server, FFmpeg capability detection, probe-first / verify-last workflow. Cut, join, silence removal, fit, captions and karaoke, overlays, motion graphics, HDR to SDR, LUTs, audio clean-up and typed dynamics, sync with drift correction, multicam, loudness, delivery checks, project rendering, batch. No API keys, no cloud, no dependencies.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ffmpeg",
|
package/references/scripts.md
CHANGED
|
@@ -558,7 +558,7 @@ silence.py INPUT [--threshold -35] [--min-silence 0.6] [--margin 0.15] [--min-ke
|
|
|
558
558
|
silence.py INPUT --filler --words transcript.json [--filler-lang auto|en|ja|es|de|fr|pt|it]
|
|
559
559
|
[--filler-words FILE] [--filler-extra W,W] [--filler-keep W,W] [--filler-pad 0.02]
|
|
560
560
|
[--transcribe] [--engine auto|parakeet-mlx|parakeet.cpp|whisper.cpp|faster-whisper|openai-whisper]
|
|
561
|
-
[--filler-list] [--max-cuts 400]
|
|
561
|
+
[--model MODEL] [--filler-list] [--max-cuts 400]
|
|
562
562
|
```
|
|
563
563
|
Runs `silencedetect`, keeps `--margin` seconds of air around speech, drops
|
|
564
564
|
gaps shorter than `--min-silence`, and re-encodes once with `select`/`aselect`
|
|
@@ -735,6 +735,23 @@ stem wins, and a stems level with no file to apply it to (`effects` without
|
|
|
735
735
|
`"audio": {"voice": "light"|"medium"|"strong"}` picks the voice strength
|
|
736
736
|
(`true` is `medium`).
|
|
737
737
|
|
|
738
|
+
An `overlays[]` entry draws one of `"logo": true`, `"image"`, `"text"` or
|
|
739
|
+
`"video"` (#326: a picture-in-picture or green-screen layer, `overlay.py
|
|
740
|
+
--video`), with `position`, `scale`, `opacity`, `start`, `end`, `margin`,
|
|
741
|
+
`platform` (and `fade`, `box`, `font*` for the still/text kinds) mapping to
|
|
742
|
+
overlay.py's flags. `"chromakey": "0x00ff00"`, `"chromakey_similarity"` and
|
|
743
|
+
`"chromakey_blend"` key a colour out of the video layer. The stage runs after
|
|
744
|
+
`fit`, so `"scale": 480` is 480 px of the delivered frame, and a template's
|
|
745
|
+
safe margins apply as they do to a logo. Refused (`kind: input`) before the
|
|
746
|
+
first stage: `video` together with `logo`/`image`/`text` in one entry (a
|
|
747
|
+
logo, image and text together still draw the first of them, as before), a
|
|
748
|
+
`chromakey*` key without `video`, `chromakey_similarity`/`_blend` without
|
|
749
|
+
`chromakey`, and a layer file that does not exist (under `--dry-run` too: no
|
|
750
|
+
earlier stage writes it). The layer's content is in the stage's `--cache`
|
|
751
|
+
key, so replacing the file re-runs `overlays` and what follows it; paths are
|
|
752
|
+
relative to the project file; `--export-timeline` lists the layer under
|
|
753
|
+
`not_exported`.
|
|
754
|
+
|
|
738
755
|
`"chapters"` is a chapters file path, or an inline list of `{"at": TIME,
|
|
739
756
|
"title": STR}`; it runs `metadata.py` on the delivered file as the last stage
|
|
740
757
|
before `check`, so the markers are in the file that ships (streams copied). The
|
|
@@ -1025,12 +1042,14 @@ missing one is `kind: missing_tool`; an installed one that fails is `kind: input
|
|
|
1025
1042
|
engine_failed`, with its own error line in `detail`). Without a language whisper.cpp is passed
|
|
1026
1043
|
`-l auto` (its own default is English), else the named or detected language, and `-mc 0` (no
|
|
1027
1044
|
previous-text context, so one hallucinated line is not repeated to the end of the file; #318). `--model` is the
|
|
1028
|
-
whisper model; a Parakeet engine takes it only when it names a Parakeet model
|
|
1045
|
+
whisper model (default `FFMPEG_SKILL_ASR_MODEL`, this machine's default, else `base`); a Parakeet engine takes it only when it names a Parakeet model
|
|
1029
1046
|
(`mlx-community/parakeet-*` or a `.gguf`), else `PARAKEET_MODEL` / `PARAKEET_CPP_MODEL`, else the
|
|
1030
|
-
English `tdt-0.6b-v2`. A
|
|
1047
|
+
English `tdt-0.6b-v2`. A Parakeet model in `FFMPEG_SKILL_ASR_MODEL` goes to the Parakeet engines
|
|
1048
|
+
only (the Whisper engines keep `base`); one given with `--model` goes to whichever engine runs.
|
|
1049
|
+
A non-English `--language` on an English-only Parakeet model is refused; a
|
|
1031
1050
|
multilingual (v3) model assumes no language, so it gets no English-assumed note. The result's
|
|
1032
1051
|
`transcription` names the engine, model and routing. `silence.py --filler --transcribe` takes
|
|
1033
|
-
the same `--engine`.
|
|
1052
|
+
the same `--engine` and `--model`.
|
|
1034
1053
|
Nothing is required: without an engine it prints install hints and the user can supply `--text`
|
|
1035
1054
|
cues instead. When engines are installed but none transcribed, the refusal names each and why
|
|
1036
1055
|
instead (`reason: engine_failed`, or `english_only` for a Parakeet engine auto passed over for
|
package/scripts/_common/asr.py
CHANGED
|
@@ -28,6 +28,9 @@ WHISPER_ENGINES = ("whisper.cpp", "faster-whisper", "openai-whisper")
|
|
|
28
28
|
ASR_ENGINES = PARAKEET_ENGINES + WHISPER_ENGINES
|
|
29
29
|
ENGINE_CHOICES = ("auto",) + ASR_ENGINES
|
|
30
30
|
ASR_ENGINE_ENV = "FFMPEG_SKILL_ASR_ENGINE"
|
|
31
|
+
# This machine's default --model (#328); --model wins, and without either the model is "base".
|
|
32
|
+
ASR_MODEL_ENV = "FFMPEG_SKILL_ASR_MODEL"
|
|
33
|
+
DEFAULT_WHISPER_MODEL = "base"
|
|
31
34
|
# whisper.cpp conditions every window on the text before it, and once a hallucinated line is in that
|
|
32
35
|
# context it can repeat it to the end of the file (#318: one sentence ~300 times, ~7 minutes of speech
|
|
33
36
|
# lost). -mc / --max-context 0 keeps no previous text. Given to the commands that write a transcript,
|
|
@@ -200,10 +203,11 @@ def write_srt(cues: List[Tuple[float, float, str]], path: str) -> None:
|
|
|
200
203
|
fh.write(f"{i}\n{fmt_srt_time(s)} --> {fmt_srt_time(e)}\n{t}\n\n")
|
|
201
204
|
|
|
202
205
|
|
|
203
|
-
def transcribe_result(video: str, out_srt: str, language: Optional[str], model: str, audio_stream: int = 0,
|
|
206
|
+
def transcribe_result(video: str, out_srt: str, language: Optional[str], model: Optional[str], audio_stream: int = 0,
|
|
204
207
|
engine: Optional[str] = None) -> Transcription:
|
|
205
208
|
"""Optional local ASR bridge: the SRT written to `out_srt`, and a Transcription saying what
|
|
206
|
-
produced it. `
|
|
209
|
+
produced it. `model` (else $FFMPEG_SKILL_ASR_MODEL, else base: requested_model()) is the
|
|
210
|
+
model; `engine` (else $FFMPEG_SKILL_ASR_ENGINE, else auto) picks the engine; auto tries
|
|
207
211
|
Parakeet (parakeet-mlx, parakeet.cpp) for English speech, then whisper-cli / main
|
|
208
212
|
(whisper.cpp), faster-whisper (python), whisper (openai-whisper CLI). No engine installed ->
|
|
209
213
|
clear error with install hints; the skill never depends on one."""
|
|
@@ -213,14 +217,16 @@ def transcribe_result(video: str, out_srt: str, language: Optional[str], model:
|
|
|
213
217
|
from _common import require_tool
|
|
214
218
|
ffmpeg = require_tool("ffmpeg")
|
|
215
219
|
wanted = requested_engine(engine)
|
|
220
|
+
model, parakeet_model = requested_model(model)
|
|
216
221
|
tmpdir = tempfile.mkdtemp(prefix="ffskill_asr_")
|
|
217
222
|
try:
|
|
218
|
-
return _transcribe_in(tmpdir, video, out_srt, language, model, audio_stream, ffmpeg, shutil, subprocess, wanted
|
|
223
|
+
return _transcribe_in(tmpdir, video, out_srt, language, model, audio_stream, ffmpeg, shutil, subprocess, wanted,
|
|
224
|
+
parakeet_model=parakeet_model)
|
|
219
225
|
finally:
|
|
220
226
|
shutil.rmtree(tmpdir, ignore_errors=True)
|
|
221
227
|
|
|
222
228
|
|
|
223
|
-
def transcribe(video: str, out_srt: str, language: Optional[str], model: str, audio_stream: int = 0,
|
|
229
|
+
def transcribe(video: str, out_srt: str, language: Optional[str], model: Optional[str], audio_stream: int = 0,
|
|
224
230
|
engine: Optional[str] = None) -> List[Tuple[float, float, str]]:
|
|
225
231
|
"""transcribe_result()'s cues alone: the call shape caption.transcribe has always had. A caller
|
|
226
232
|
that reports which engine ran (caption.py does) calls transcribe_result() instead."""
|
|
@@ -304,8 +310,12 @@ def _skipped_parakeet(route: "Route", language: Optional[str], language_flag: st
|
|
|
304
310
|
|
|
305
311
|
|
|
306
312
|
def _transcribe_in(tmpdir: str, video: str, out_srt: str, language: Optional[str], model: str, audio_stream: int,
|
|
307
|
-
ffmpeg: str, shutil, subprocess, wanted: str = "auto"
|
|
313
|
+
ffmpeg: str, shutil, subprocess, wanted: str = "auto",
|
|
314
|
+
parakeet_model: Optional[str] = None) -> Transcription:
|
|
315
|
+
"""`model` is what the Whisper engines run; the Parakeet engines are offered `parakeet_model`
|
|
316
|
+
(default `model`), which differs only for a Parakeet model from $FFMPEG_SKILL_ASR_MODEL."""
|
|
308
317
|
from _common import run_analysis, STATE
|
|
318
|
+
pmodel = model if parakeet_model is None else parakeet_model
|
|
309
319
|
language = engine_language(language)
|
|
310
320
|
wav = os.path.join(tmpdir, "audio.wav")
|
|
311
321
|
# A wav in our own temp dir: a measurement input for the engine, not a deliverable, so it
|
|
@@ -318,7 +328,7 @@ def _transcribe_in(tmpdir: str, video: str, out_srt: str, language: Optional[str
|
|
|
318
328
|
tried: "List[Dict[str, str]]" = []
|
|
319
329
|
|
|
320
330
|
def parakeet(engines: "List[str]", facts: Dict[str, Any]) -> Optional[Transcription]:
|
|
321
|
-
done = _parakeet_attempt(engines, facts, wanted, language, wav, tmpdir,
|
|
331
|
+
done = _parakeet_attempt(engines, facts, wanted, language, wav, tmpdir, pmodel, shutil, subprocess, video, "--language",
|
|
322
332
|
tried=tried)
|
|
323
333
|
if not done:
|
|
324
334
|
return None
|
|
@@ -328,13 +338,13 @@ def _transcribe_in(tmpdir: str, video: str, out_srt: str, language: Optional[str
|
|
|
328
338
|
return run
|
|
329
339
|
|
|
330
340
|
# 0. Parakeet (English), when auto-routing picks it or it was asked for by name
|
|
331
|
-
route = parakeet_route(wanted, language, wav, shutil, subprocess, model)
|
|
341
|
+
route = parakeet_route(wanted, language, wav, shutil, subprocess, pmodel, whisper_model=model)
|
|
332
342
|
done = parakeet(route.first, route.facts)
|
|
333
343
|
if done:
|
|
334
344
|
return done
|
|
335
345
|
tried.extend(_skipped_parakeet(route, language, "--language"))
|
|
336
346
|
if wanted in PARAKEET_ENGINES:
|
|
337
|
-
die_engine_failed(wanted, video, tried, _parakeet_available(wanted, shutil,
|
|
347
|
+
die_engine_failed(wanted, video, tried, _parakeet_available(wanted, shutil, pmodel))
|
|
338
348
|
facts = route.facts
|
|
339
349
|
if route.first:
|
|
340
350
|
# auto chose Parakeet and no Parakeet engine transcribed it: the routing says why Whisper ran
|
|
@@ -665,6 +675,25 @@ def requested_engine(engine: Optional[str]) -> str:
|
|
|
665
675
|
return name
|
|
666
676
|
|
|
667
677
|
|
|
678
|
+
def requested_model(model: Optional[str]) -> Tuple[str, str]:
|
|
679
|
+
"""(the model the Whisper engines run, the model the Parakeet engines are offered): --model,
|
|
680
|
+
else $FFMPEG_SKILL_ASR_MODEL, else base.
|
|
681
|
+
|
|
682
|
+
--model is one call's choice and goes to whichever engine runs, as it always has. The
|
|
683
|
+
variable is this machine's default for every call, so it never hands an engine a model that
|
|
684
|
+
engine cannot load: a Parakeet model there (an mlx-community/parakeet-* repo or a .gguf) goes
|
|
685
|
+
to the Parakeet engines only, and the Whisper engines keep base. A Whisper name reaches the
|
|
686
|
+
Parakeet engines too, which pass over it for their own model, as they do for --model."""
|
|
687
|
+
if model:
|
|
688
|
+
return model, model
|
|
689
|
+
name = os.environ.get(ASR_MODEL_ENV, "").strip()
|
|
690
|
+
if not name:
|
|
691
|
+
return DEFAULT_WHISPER_MODEL, DEFAULT_WHISPER_MODEL
|
|
692
|
+
if _names_parakeet_model(name):
|
|
693
|
+
return DEFAULT_WHISPER_MODEL, name
|
|
694
|
+
return name, name
|
|
695
|
+
|
|
696
|
+
|
|
668
697
|
def _is_english(language: Optional[str]) -> bool:
|
|
669
698
|
# "eng" too: caption.py's --language is also the mux track tag, where ISO 639-2 is natural
|
|
670
699
|
return (language or "").lower().split("-")[0].split("_")[0] in ("en", "eng", "english")
|
|
@@ -926,7 +955,7 @@ def _heard_nothing(text: str, key: str) -> bool:
|
|
|
926
955
|
return isinstance(doc, dict) and doc.get(key) == []
|
|
927
956
|
|
|
928
957
|
|
|
929
|
-
def transcribe_words(video: str, language: "Optional[str]" = None, model: str =
|
|
958
|
+
def transcribe_words(video: str, language: "Optional[str]" = None, model: "Optional[str]" = None,
|
|
930
959
|
audio_stream: int = 0, engine: "Optional[str]" = None) -> "Tuple[List[Dict[str, Any]], Optional[str]]":
|
|
931
960
|
"""transcribe_words_result()'s (words, engine) alone: the call shape this function has had
|
|
932
961
|
since 1.17. A caller that reports which engine ran (silence.py does) calls
|
|
@@ -935,11 +964,12 @@ def transcribe_words(video: str, language: "Optional[str]" = None, model: str =
|
|
|
935
964
|
return run.words, run.engine
|
|
936
965
|
|
|
937
966
|
|
|
938
|
-
def transcribe_words_result(video: str, language: "Optional[str]" = None, model: str =
|
|
967
|
+
def transcribe_words_result(video: str, language: "Optional[str]" = None, model: "Optional[str]" = None,
|
|
939
968
|
audio_stream: int = 0, engine: "Optional[str]" = None,
|
|
940
969
|
language_flag: str = "--filler-lang") -> Transcription:
|
|
941
970
|
"""[{word, start, end}, ...] from a local speech engine, as a Transcription naming the engine.
|
|
942
971
|
|
|
972
|
+
`model` (else $FFMPEG_SKILL_ASR_MODEL, else base: requested_model()) is the model and
|
|
943
973
|
`engine` (else $FFMPEG_SKILL_ASR_ENGINE, else auto) picks it; auto tries Parakeet for English
|
|
944
974
|
speech first (parakeet-mlx sub-word tokens merged into words, parakeet.cpp `--json` words),
|
|
945
975
|
then drives whichever whisper is installed with ITS word-timestamp option -- whisper.cpp
|
|
@@ -957,7 +987,7 @@ def transcribe_words_result(video: str, language: "Optional[str]" = None, model:
|
|
|
957
987
|
ffmpeg = require_tool("ffmpeg")
|
|
958
988
|
wanted = requested_engine(engine)
|
|
959
989
|
language = engine_language(language)
|
|
960
|
-
parakeet_model =
|
|
990
|
+
model, parakeet_model = requested_model(model)
|
|
961
991
|
tmpdir = _tempfile.mkdtemp(prefix="ffskill_asrw_")
|
|
962
992
|
try:
|
|
963
993
|
wav = os.path.join(tmpdir, "audio.wav")
|
|
@@ -39,7 +39,7 @@ _NOT_EXPORTED = {
|
|
|
39
39
|
"silence": "the silence cut (render.py measures it at render time; the timeline keeps the clips whole)",
|
|
40
40
|
"captions": "captions (burn them with render.py, or import the .srt into the editor)",
|
|
41
41
|
"graphics": "graphics (titles, lower thirds, social templates)",
|
|
42
|
-
"overlays": "overlays (logos and
|
|
42
|
+
"overlays": "overlays (logos, images, text and video layers)",
|
|
43
43
|
"fit": "fit / reframe to the delivery aspect (set the sequence frame in the editor)",
|
|
44
44
|
"audiogram": "the audiogram picture",
|
|
45
45
|
"loudness": "loudness normalisation",
|
package/scripts/caption.py
CHANGED
|
@@ -916,9 +916,10 @@ def main() -> int:
|
|
|
916
916
|
"(default: none, so no player burns in a language the viewer did not ask for)")
|
|
917
917
|
src.add_argument("--offset", default="0", help="shift every cue by TIME (seconds, mm:ss, hh:mm:ss.ms or "
|
|
918
918
|
"hh:mm:ss:ff; a leading - shifts earlier); works for --text, --srt and --ass")
|
|
919
|
-
src.add_argument("--model", default=
|
|
920
|
-
help="speech model for --transcribe: a whisper model name/path (default
|
|
921
|
-
"model (an mlx-community/parakeet-* repo or a .gguf) for the
|
|
919
|
+
src.add_argument("--model", default=None,
|
|
920
|
+
help="speech model for --transcribe: a whisper model name/path (default: $FFMPEG_SKILL_ASR_MODEL, "
|
|
921
|
+
"else base), or a Parakeet model (an mlx-community/parakeet-* repo or a .gguf) for the "
|
|
922
|
+
"Parakeet engines")
|
|
922
923
|
src.add_argument("--engine", choices=ENGINE_CHOICES, default=None,
|
|
923
924
|
help="speech engine for --transcribe (default: $FFMPEG_SKILL_ASR_ENGINE, else auto = Parakeet "
|
|
924
925
|
"for English speech when installed, else whisper.cpp / faster-whisper / openai-whisper)")
|
package/scripts/render.py
CHANGED
|
@@ -22,7 +22,8 @@ Project format (all keys optional except clips):
|
|
|
22
22
|
],
|
|
23
23
|
"overlays": [
|
|
24
24
|
{"logo": true},
|
|
25
|
-
{"text": "Episode 12", "position": "bottom", "start": 1, "end": 5, "fade": 0.3, "box": true}
|
|
25
|
+
{"text": "Episode 12", "position": "bottom", "start": 1, "end": 5, "fade": 0.3, "box": true},
|
|
26
|
+
{"video": "webcam.mp4", "position": "bottom-right", "scale": 480, "chromakey": "0x00ff00"}
|
|
26
27
|
],
|
|
27
28
|
"audio": {"voice": "medium", "music": "bed.mp3", "music_volume": -16, "duck": true, "music_fade_out": 2,
|
|
28
29
|
"effects": "sfx.wav", "stems": {"dialogue": 0, "music": -18, "effects": -22}},
|
|
@@ -37,6 +38,9 @@ Stages run in this order: clips (cut) → join → silence → fit → captions
|
|
|
37
38
|
graphics → overlays → audio → loudness → export → chapters → check. Missing stages are
|
|
38
39
|
skipped. "brand" points caption/graphics/overlay at a brand.json (fonts,
|
|
39
40
|
colours, logo, safe margin); {"logo": true} in overlays places the brand logo.
|
|
41
|
+
An overlay entry draws one of logo, image, text or video (a picture-in-picture
|
|
42
|
+
layer, overlay.py --video; "chromakey", "chromakey_similarity" and
|
|
43
|
+
"chromakey_blend" key a colour out of that video and need "video").
|
|
40
44
|
|
|
41
45
|
"check" mirrors check.py's own exit code: a delivery-spec FAIL (or check.py
|
|
42
46
|
itself failing to run) exits 1, same as running check.py directly would --
|
|
@@ -140,8 +144,10 @@ OBJECT_KEYS: Dict[str, frozenset] = {
|
|
|
140
144
|
"graphics[]": frozenset({"template", "name", "title", "subtitle", "start", "end", "position",
|
|
141
145
|
"from", "scale", "primary", "text_color", "lang",
|
|
142
146
|
"text", "top", "bottom", "duration", "margin", "platform"}),
|
|
147
|
+
# #326: "video" is a fourth layer kind (overlay.py --video), keyed with the chromakey* keys
|
|
143
148
|
"overlays[]": frozenset({"logo", "image", "text", "position", "start", "end", "fade", "opacity",
|
|
144
|
-
"scale", "font_size", "font", "font_file", "margin", "box", "platform"
|
|
149
|
+
"scale", "font_size", "font", "font_file", "margin", "box", "platform",
|
|
150
|
+
"video", "chromakey", "chromakey_similarity", "chromakey_blend"}),
|
|
145
151
|
"audio": frozenset({"music", "replace", "music_volume", "fade_in", "fade_out", "music_fade_out",
|
|
146
152
|
"gain", "duck_amount", "duck_threshold", "duck_attack", "duck_release",
|
|
147
153
|
"voice", "denoise", "duck", "music_loop", "stereo", "mono", "downmix",
|
|
@@ -410,6 +416,28 @@ def check_keys(obj: Any, schema: str, label: str) -> None:
|
|
|
410
416
|
else f" (valid keys: {', '.join(sorted(valid))})"))
|
|
411
417
|
|
|
412
418
|
|
|
419
|
+
def check_overlay(ov: Any, label: str) -> None:
|
|
420
|
+
"""#326: a video layer is one kind of overlay, as overlay.py's --image/--text/--logo/--video
|
|
421
|
+
are one mutually exclusive choice, and the chromakey* keys only act on it. A logo, image and
|
|
422
|
+
text in one entry still draw the first of them, as before; only a combination with the new
|
|
423
|
+
"video" key is refused, so no project 2.8.6 rendered is refused now."""
|
|
424
|
+
if not isinstance(ov, dict):
|
|
425
|
+
return
|
|
426
|
+
if ov.get("video"):
|
|
427
|
+
others = [k for k in ("logo", "image", "text") if ov.get(k)]
|
|
428
|
+
if others:
|
|
429
|
+
die(f"{label}: give one of logo, image, text or video per entry, not video and {others[0]} "
|
|
430
|
+
"(add a second overlays[] entry for the other layer)")
|
|
431
|
+
else:
|
|
432
|
+
keyed = [k for k in ("chromakey", "chromakey_similarity", "chromakey_blend") if ov.get(k) is not None]
|
|
433
|
+
if keyed:
|
|
434
|
+
die(f"{label}: {keyed[0]} keys a colour out of a video layer and needs \"video\" (overlay.py --chromakey needs --video)")
|
|
435
|
+
if ov.get("chromakey") is None:
|
|
436
|
+
tuned = [k for k in ("chromakey_similarity", "chromakey_blend") if ov.get(k) is not None]
|
|
437
|
+
if tuned:
|
|
438
|
+
die(f"{label}: {tuned[0]} tunes \"chromakey\", which this entry does not set")
|
|
439
|
+
|
|
440
|
+
|
|
413
441
|
def require_object(obj: Any, label: str) -> None:
|
|
414
442
|
"""Refuse a value that is not an object where the run reads it with .get(): "export": "reels"
|
|
415
443
|
in a render, a string frame, a clip written as a bare path. 2.2.1 died there with a traceback
|
|
@@ -470,6 +498,8 @@ def validate_project(proj: Dict[str, Any], timeline: bool = False, stop_after: O
|
|
|
470
498
|
die(f"{name}[{i}]: must be an object {{...}}, got {type(item).__name__} {item!r:.60}")
|
|
471
499
|
for i, item in enumerate(items if isinstance(items, list) else []):
|
|
472
500
|
check_keys(item, f"{name}[]", f"{name}[{i}]")
|
|
501
|
+
if name == "overlays":
|
|
502
|
+
check_overlay(item, f"overlays[{i}]")
|
|
473
503
|
if name != "clips":
|
|
474
504
|
continue
|
|
475
505
|
# every clip stage starts from rel(c["src"]): without one it was a KeyError traceback
|
|
@@ -954,6 +984,15 @@ def main() -> int:
|
|
|
954
984
|
output = rel(proj.get("output") or "final.mp4")
|
|
955
985
|
# the final stage is a copy from the work dir, so run()'s own guard never sees the sources (review 5)
|
|
956
986
|
refuse_output_is_input(output, *[rel(c.get("src")) for c in clips if c.get("src")])
|
|
987
|
+
if _reaches("overlays", args.stop_after):
|
|
988
|
+
# #326: an overlay video is the caller's file, never an earlier stage's output, so a
|
|
989
|
+
# missing one is refused here, under --dry-run too, not after every stage before it ran
|
|
990
|
+
for i, ov in enumerate(proj.get("overlays") or []):
|
|
991
|
+
if ov.get("video"):
|
|
992
|
+
layer = rel(ov["video"])
|
|
993
|
+
refuse_output_is_input(output, layer)
|
|
994
|
+
if not os.path.isfile(layer):
|
|
995
|
+
die(f"overlays[{i}].video: file not found: {layer}", kind="input")
|
|
957
996
|
# 2.0 refuses an existing output without --overwrite; the final place_output() would, but
|
|
958
997
|
# only after every stage had run -- say so before the first one instead
|
|
959
998
|
_check_existing_output(["ffmpeg", output])
|
|
@@ -1248,9 +1287,13 @@ def main() -> int:
|
|
|
1248
1287
|
argv += ["--image", rel(ov["image"])]
|
|
1249
1288
|
elif ov.get("text"):
|
|
1250
1289
|
argv += ["--text", ov["text"]]
|
|
1290
|
+
elif ov.get("video"):
|
|
1291
|
+
# #326: sized and placed here, after fit, against the frame that ships
|
|
1292
|
+
argv += ["--video", rel(ov["video"])]
|
|
1251
1293
|
else:
|
|
1252
|
-
die(f"overlays[{i}] needs image or
|
|
1253
|
-
for k, flag in (("position", "--position"), ("start", "--start"), ("end", "--end"), ("fade", "--fade"), ("opacity", "--opacity"), ("scale", "--scale"), ("font_size", "--font-size"), ("font", "--font"), ("font_file", "--font-file"), ("margin", "--margin"), ("platform", "--platform")
|
|
1294
|
+
die(f"overlays[{i}] needs logo, image, text or video")
|
|
1295
|
+
for k, flag in (("position", "--position"), ("start", "--start"), ("end", "--end"), ("fade", "--fade"), ("opacity", "--opacity"), ("scale", "--scale"), ("font_size", "--font-size"), ("font", "--font"), ("font_file", "--font-file"), ("margin", "--margin"), ("platform", "--platform"),
|
|
1296
|
+
("chromakey", "--chromakey"), ("chromakey_similarity", "--chromakey-similarity"), ("chromakey_blend", "--chromakey-blend")):
|
|
1254
1297
|
if ov.get(k) is not None:
|
|
1255
1298
|
argv += [flag, str(ov[k])]
|
|
1256
1299
|
if ov.get("box"):
|
package/scripts/silence.py
CHANGED
|
@@ -127,7 +127,7 @@ def resolve_filler(args, meta):
|
|
|
127
127
|
# cannot answer this question: a cue has a start and an end, a word does not.
|
|
128
128
|
# The call's own result, not module state: engine, routing and notes belong to this run.
|
|
129
129
|
asr_run = _asr.transcribe_words_result(args.input, args.filler_lang if args.filler_lang != "auto" else None,
|
|
130
|
-
engine=args.engine)
|
|
130
|
+
model=args.model, engine=args.engine)
|
|
131
131
|
words, engine = asr_run.words, asr_run.engine
|
|
132
132
|
transcription = dict(asr_run.facts)
|
|
133
133
|
args._asr_notes = list(asr_run.notes)
|
|
@@ -277,6 +277,9 @@ def main() -> int:
|
|
|
277
277
|
fil.add_argument("--engine", choices=_asr.ENGINE_CHOICES, default=None,
|
|
278
278
|
help="speech engine for --transcribe (default: $FFMPEG_SKILL_ASR_ENGINE, else auto = Parakeet "
|
|
279
279
|
"for English speech when installed, else whisper)")
|
|
280
|
+
fil.add_argument("--model", default=None,
|
|
281
|
+
help="speech model for --transcribe, as caption.py's: a whisper model name/path (default: "
|
|
282
|
+
"$FFMPEG_SKILL_ASR_MODEL, else base), or a Parakeet model for the Parakeet engines")
|
|
280
283
|
fil.add_argument("--filler-list", action="store_true",
|
|
281
284
|
help="report what --filler would remove and write nothing")
|
|
282
285
|
fil.add_argument("--max-cuts", type=int, default=400,
|