ffmpeg-skill 1.23.0 → 1.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/SKILL.md +53 -53
- package/docs/contract.md +28 -14
- package/mcp/server.py +12 -1
- package/package.json +1 -1
- package/scripts/_contract.py +86 -0
- package/scripts/caption.py +111 -14
- package/scripts/loudness.py +2 -0
package/README.md
CHANGED
|
@@ -345,6 +345,8 @@ On Windows, `python3` is only on PATH if Python was installed from the Microsoft
|
|
|
345
345
|
|
|
346
346
|
`FFMPEG_SKILL_MCP_LEAN=1` in the server's environment drops `json` and `progress` from every `inputSchema`: they are transport flags the server sets itself, not tool arguments, and 2.0 drops them unconditionally. It is opt-in, independent of `FFMPEG_SKILL_MCP_FULL`, and the default `tools/list` stays byte-identical (aside from the core-12 filter) to the CLI surface the contract promises.
|
|
347
347
|
|
|
348
|
+
The server also advertises a `prompts` capability: five canned workflow recipes an agent can ask for by name instead of composing the individual tool calls itself — `reel`, `podcast`, `multicam`, `delivery_check`, `hdr`. `prompts/list` returns each one's name, description and arguments; `prompts/get {name, arguments}` fills the matching template and returns one text message naming the actual command lines to run and what to check afterward. These are recipes, not new tool calls — every command line a prompt names is one `tools/call` (or the CLI) can already run.
|
|
349
|
+
|
|
348
350
|
### Capability detection
|
|
349
351
|
|
|
350
352
|
```bash
|
package/SKILL.md
CHANGED
|
@@ -62,93 +62,93 @@ Timestamp flags (`--start`, `--end`, `--at`, `--from`, `--duration`, `--offset`,
|
|
|
62
62
|
|
|
63
63
|
| User says | Do |
|
|
64
64
|
|-----------|----|
|
|
65
|
-
| "what's in this file", "how long is it" | `probe.py input.mp4` |
|
|
66
65
|
| "cut from 1:20 to 2:05", "trim the first 10 s" | `cut.py input.mp4 --start 1:20 --end 2:05` |
|
|
67
66
|
| "keep only these parts", "remove the middle" | `cut.py input.mp4 --segments 0-1:00,1:30-2:00` |
|
|
68
67
|
| "make it exactly 60 seconds" | `fit.py input.mp4 --duration 60` (speed) or `--method trim` |
|
|
68
|
+
| "cut this and make it HEVC / AV1 / ProRes" (output codec named) | `cut.py input.mp4 --start 0:10 --end 0:40 --codec hevc` (`--codec`/`--quality` on any re-encoding tool; ProRes needs `-o NAME.mov`) |
|
|
69
|
+
| "cut out the pauses", "jump cuts" | `silence.py input.mp4 [--threshold -40 --min-silence 0.8]` |
|
|
70
|
+
| "cut the ums and uhs", "remove the filler words" | `silence.py input.mp4 --filler --words words.json` (measured word timings; `--transcribe` makes them) |
|
|
71
|
+
| "don't cut inside a sentence, just the real pauses" | `silence.py input.mp4 --speech-aware` — a breath under `--min-silence` inside a sentence is kept, only sentence-boundary pauses cut; composes with `--filler` into one list |
|
|
72
|
+
| "a 60 s highlight from this hour" | `scenes.py long.mp4 --highlights 6 --target 60 --edl picks.txt` → `cut.py --segments` |
|
|
73
|
+
| "cut on the beat", "edit it to the music" | `scenes.py track.mp4 --beats --json > beats.json`, then `cut.py input.mp4 --segments ... --snap beats --snap-source beats.json` (`--snap-source` carries the measured grid over) |
|
|
74
|
+
| "speed up here, slow-mo there" (known segments) | `speedramp.py action.mp4 --segment 0-3:1.0 --segment 3-4:0.25 --segment 4-8:2.0` |
|
|
75
|
+
| "hold on this frame", "freeze the last frame" | `freeze.py clip.mp4 --hold 2` |
|
|
76
|
+
| "smooth slow motion", "half speed but fluid" | `fit.py input.mp4 --duration 2x --smooth interpolate` (slow) or `--smooth blend` |
|
|
77
|
+
| "reverse this clip" | `reverse.py input.mp4` |
|
|
78
|
+
| "loop this clip to fill 30 seconds" | `loop.py bg_loop.mp4 --duration 30` |
|
|
69
79
|
| "make it vertical / 9:16 / square" | `fit.py input.mp4 --aspect 9:16 --fit pad` (or `--fit crop`; `--pad-fill blur` for blurred bars) |
|
|
70
80
|
| "resize to a height/width" | `fit.py input.mp4 --height 1080` (or `--width`, or both for an exact frame) |
|
|
71
81
|
| "crop to this exact box" (known x/y/w/h) | `crop.py input.mp4 --x 100 --y 0 --width 1080 --height 1920` |
|
|
72
|
-
| "
|
|
82
|
+
| "blurred background instead of black bars" | `fit.py input.mp4 --aspect 9:16 --fit blur` (whole picture kept, borders a blurred, darkened copy) |
|
|
83
|
+
| "rotate 90 degrees", "mirror it" | `fit.py input.mp4 --rotate 90` / `fit.py input.mp4 --flip h` |
|
|
84
|
+
| "the horizon is tilted" (known degrees) | `straighten.py tilted.mp4 --degrees -2.5` |
|
|
85
|
+
| "flat view out of this 360 video" (known yaw/pitch/fov) | `sphere.py insta360.mp4 --yaw 90 --pitch 0 --h-fov 100 --v-fov 70` |
|
|
73
86
|
| "this old footage is interlaced" | `deinterlace.py input.mp4` |
|
|
74
87
|
| "it's grainy/noisy, clean it up" | `denoise.py input.mp4 --strength medium` |
|
|
75
88
|
| "blur/pixelate this face/plate" (known box) | `redact.py input.mp4 --x 820 --y 140 --width 240 --height 240 --mode pixelate` |
|
|
76
|
-
| "flat view out of this 360 video" (known yaw/pitch/fov) | `sphere.py insta360.mp4 --yaw 90 --pitch 0 --h-fov 100 --v-fov 70` |
|
|
77
|
-
| "the horizon is tilted" (known degrees) | `straighten.py tilted.mp4 --degrees -2.5` |
|
|
78
|
-
| "turn this image into a clip", "title card" | `insert.py title.png --duration 3` |
|
|
79
|
-
| "slow zoom on a photo", "Ken Burns" | `insert.py photo.jpg --duration 6 --zoom in --pan right --width 1920 --height 1080` |
|
|
80
|
-
| "rotate 90 degrees", "mirror it" | `fit.py input.mp4 --rotate 90` / `fit.py input.mp4 --flip h` |
|
|
81
|
-
| "reverse this clip" | `reverse.py input.mp4` |
|
|
82
89
|
| "stabilize this shaky footage" | `stabilize.py input.mp4` |
|
|
83
|
-
| "
|
|
84
|
-
| "
|
|
85
|
-
| "
|
|
86
|
-
| "
|
|
87
|
-
| "
|
|
88
|
-
| "
|
|
89
|
-
| "loop this clip to fill 30 seconds" | `loop.py bg_loop.mp4 --duration 30` |
|
|
90
|
-
| "cut to the product shot 0:12-0:16", "B-roll over this bit" | `broll.py talk.mp4 --insert product.mp4 --at 12 --end 16` (repeat `--insert/--at`; `--audio b\|mix`) |
|
|
91
|
-
| "add chapters", "chapter markers for YouTube" | `metadata.py episode.mp4 --chapters chapters.txt` (`TIME TITLE` per line; streams copied; a `render.py` project spells it `"chapters"`). `--auto-chapters` proposes them from measured pauses/scene cuts, titled `Chapter N`, renameable |
|
|
92
|
-
| "set the title / artist / comment" | `metadata.py episode.mp4 --title "Episode 12" --artist "Studio"` |
|
|
93
|
-
| "put these videos in a 4x2 grid" | `grid.py t1.mp4 ... t8.mp4 --cols 4 --rows 2` |
|
|
90
|
+
| "are there black bars on this?" | `cropdetect.py input.mp4` |
|
|
91
|
+
| "iPhone Dolby Vision clip looks wrong" | `color.py clip.mov --to-sdr` or `--strip-dovi` (keep HDR, drop the DV layer) |
|
|
92
|
+
| "the colours look washed out / iPhone HDR" | `color.py input.mov --to-sdr` (probe shows `hdr: true`) |
|
|
93
|
+
| "apply this LUT", "convert the S-Log footage" | `color.py input.mp4 --lut grade.cube [--lut-strength 0.7]` |
|
|
94
|
+
| "the colours are tagged wrong" | `color.py input.mp4 --retag bt709` (stream copy; re-encodes only if the copy can't carry it — see `reencoded`) |
|
|
95
|
+
| "brighten it / punch up contrast / fix white balance" | `color.py input.mp4 --correct --exposure 0.3 --contrast 1.1 --saturation 1.05 --temperature 5600 --tint -0.05` |
|
|
94
96
|
| "add subtitles from this SRT", "burn in captions" | `caption.py input.mp4 --srt subs.srt` |
|
|
95
97
|
| "caption it with these lines" (text with times) | `caption.py input.mp4 --text cues.txt` |
|
|
96
98
|
| "keep the subtitles toggleable", "mux in an SRT" | `caption.py input.mp4 --srt subs.srt --mode mux`; repeat `--srt file:lang` for several languages, `.mkv` for more than two |
|
|
97
99
|
| "the captions are tiny / three lines on a Short", "don't chop the sentence" | `caption.py` shrinks the size until the cue fits `--max-lines` before splitting it (`--fit-size off` for 1.16 behaviour, `--min-size` sets the floor). Keep the template's `--max-lines` (2 on a vertical) and let the size drop; raising it to dodge a shrink stacks two words per line. A word still wider than the column at the floor is sliced at the edge (hyphen preferred), never rewritten, automatic; `broken_inside_word` in the JSON counts it |
|
|
100
|
+
| "transcribe it and caption it" | `caption.py input.mp4 --transcribe --animate pop --karaoke` (needs a local whisper; else `--text`) |
|
|
101
|
+
| "TikTok-style captions with the words popping" | `caption.py input.mp4 --text cues.txt --animate pop --karaoke` |
|
|
98
102
|
| "our logo top-right", "a watermark" | `overlay.py input.mp4 --image logo.png --position top-right --scale 200` |
|
|
99
103
|
| "a title for the first 4 seconds" | `overlay.py input.mp4 --text "Title" --position top --start 0 --end 4 --fade 0.4` |
|
|
100
104
|
| "webcam clip in the corner", "picture-in-picture" | `overlay.py input.mp4 --video webcam.mp4 --position bottom-right --scale 480` |
|
|
101
105
|
| "remove the green screen" | `overlay.py bg.mp4 --video greenscreen.mp4 --chromakey 0x00ff00` |
|
|
102
|
-
| "
|
|
106
|
+
| "a lower third with my name", "countdown intro", "progress bar" | `graphics.py input.mp4 --template lower-third --name "..." --title "..." --start 2 --end 8` |
|
|
107
|
+
| "a sticker", "a hook card for the first 3 s", "meme text" | `graphics.py input.mp4 --template sticker --text "NEW" --platform tiktok` / `--template hook --title "..." --duration 3` / `--template meme --top "..." --bottom "..."` |
|
|
108
|
+
| "use our brand fonts/colours/logo" | `--brand brand.json` on caption/overlay/graphics, or `"brand"` in project.json |
|
|
103
109
|
| "sync the lav mic", "line up two cameras" | `sync.py camera.mp4 mic.wav --replace-audio` / `sync.py camA.mp4 camB.mp4 --trim-second`; a third+ recorder is another positional (`sync.py ref.mp4 mic.wav cam2.mp4`), one offsets JSON |
|
|
104
110
|
| "fix the audio levels", "normalise to -14 LUFS" | `loudness.py input.mp4` (`-I -16 --tp -1.5` podcast, `-I -23` broadcast; `--lra N` for the range) |
|
|
105
|
-
| "
|
|
111
|
+
| "clean up the audio", "remove the hiss" | `audio.py input.mp4 --voice` (speech; `--voice light\|medium\|strong`) or `--denoise` |
|
|
112
|
+
| "add background music under the talking" | `audio.py input.mp4 --music bed.mp3 --duck --fade-out 3` (`--effects sfx.wav` adds a third bed, never ducked; project levels: `audio.stems`) |
|
|
113
|
+
| "make the music duck harder / come back faster" | add `--duck-amount 18 --duck-threshold -30 --duck-release 250` (`--duck-attack` too) |
|
|
114
|
+
| "the mix sounds narrow", "wider stereo" | `audio.py band.wav --stereo-widen 0.5` (needs a real stereo source; 5.1 needs `--downmix`) |
|
|
115
|
+
| "convert the 5.1 to stereo" | `audio.py input.mov --downmix` |
|
|
116
|
+
| "swap in the narration track" | `audio.py input.mp4 --replace narration.wav` |
|
|
117
|
+
| "pull the audio out", "give me the sound as WAV" | `audio.py input.mp4 -o input.wav` (an audio extension drops the picture; `--audio-stream 1` picks a track) |
|
|
118
|
+
| "compress the voice", "limit the peaks", "gate the room noise" | `audio.py input.mp4 --compress --comp-threshold -20 --comp-ratio 4` / `--limit --limit-ceiling -1` / `--gate --gate-threshold -45` |
|
|
119
|
+
| "the audio drifts out of sync over the hour" | `sync.py camera.mp4 recorder.wav --fix-drift --replace-audio` |
|
|
120
|
+
| "turn this podcast into a video", "audiogram" | `render.py --template audiogram ep.m4a --image cover.png` — waveform over a still or colour plate; give an image or colour, nothing is fetched |
|
|
106
121
|
| "make this a TikTok / Reel / Short / YouTube / X / LinkedIn / podcast" | `render.py --template tiktok\|reels\|shorts\|youtube-shorts\|youtube\|x\|linkedin\|facebook\|podcast input.mp4 [--cues cues.txt] [--title "..."]` — frame, captions, loudness, export and check in one command (`--list-templates`, `--write-project` to edit first) |
|
|
107
122
|
| "post it everywhere", "one edit for every platform" | `render.py --template all input.mp4 --cues cues.txt` (or a comma list) → one file per destination plus `<name>_pack.md` (`report.py --pack` renders the HTML) |
|
|
108
123
|
| "export for YouTube / Reels / X", "a ProRes master" | `export.py input.mp4 --preset youtube\|reels\|tiktok\|shorts\|linkedin\|facebook\|x\|prores\|h265` (`--normalize` hits the loudness spec in the same call; `youtube-hdr` keeps HDR, `youtube-av1` writes AV1) |
|
|
109
124
|
| "make a GIF preview" | `export.py input.mp4 --preset gif` |
|
|
110
125
|
| "a small proxy / cheap preview file" | `proxy.py input.mp4 [--width 640 --no-audio]` — not a delivery preset (that is `export.py`) |
|
|
111
|
-
| "cut out the pauses", "jump cuts" | `silence.py input.mp4 [--threshold -40 --min-silence 0.8]` |
|
|
112
|
-
| "cut the ums and uhs", "remove the filler words" | `silence.py input.mp4 --filler --words words.json` (measured word timings; `--transcribe` makes them) |
|
|
113
|
-
| "don't cut inside a sentence, just the real pauses" | `silence.py input.mp4 --speech-aware` — a breath under `--min-silence` inside a sentence is kept, only sentence-boundary pauses cut; composes with `--filler` into one list |
|
|
114
|
-
| "stitch these clips", "add a crossfade" | `join.py a.mp4 b.mp4 c.mp4 --transition fade --duration 0.5` |
|
|
115
|
-
| "show me what it looks like", "are the captions readable" | `look.py output.mp4 --tiles 3x2`, then view the PNG |
|
|
116
|
-
| "what would you run?", "don't render yet" | any script with `--dry-run` |
|
|
117
|
-
| "a 60 s highlight from this hour" | `scenes.py long.mp4 --highlights 6 --target 60 --edl picks.txt` → `cut.py --segments` |
|
|
118
|
-
| "cut on the beat", "edit it to the music" | `scenes.py track.mp4 --beats --json > beats.json`, then `cut.py input.mp4 --segments ... --snap beats --snap-source beats.json` (`--snap-source` carries the measured grid over) |
|
|
119
|
-
| "which shots are static vs moving", "volume peaks per second", "is this speech or music" | `scenes.py input.mp4 --shots` (static/pan/motion per shot, measured flow) / `--audio-peaks` (dBFS list) / `--speech` (a speech-vs-music ratio, not a classification) — combine, or alone |
|
|
120
|
-
| "where does the subject move, so I can crop it myself" | `cropdetect.py input.mp4 --motion-centre` — motion centroid per second, report-only; the calling agent picks the crop |
|
|
121
126
|
| "is this OK to upload?" | `check.py final.mp4 --platform reels` |
|
|
122
127
|
| "a podcast episode with chapters" | `loudness.py ep.wav -I -16 --tp -1.5` → `metadata.py ep.m4a --chapters chapters.txt` → `check.py ep.m4a --platform podcast` (chapters and channels rows) |
|
|
128
|
+
| "send me a summary of what you did" | `report.py --before raw.mov --after final.mp4 --platform youtube -o report.html` |
|
|
129
|
+
| "turn this image into a clip", "title card" | `insert.py title.png --duration 3` |
|
|
130
|
+
| "slow zoom on a photo", "Ken Burns" | `insert.py photo.jpg --duration 6 --zoom in --pan right --width 1920 --height 1080` |
|
|
131
|
+
| "make a blank/colour background clip" | `background.py -o bg.mp4 --duration 3 --width 1920 --height 1080 --color 0x101010` |
|
|
132
|
+
| "turn these numbered frames into a video" | `sequence.py --dir frames --pattern "frame_%04d.png" --fps 24` |
|
|
133
|
+
| "waveform/spectrum video for this track" | `waveform.py podcast.wav -o waveform.mp4` |
|
|
134
|
+
| "add black at the start" | `pad.py clip.mp4 --start 1.5` |
|
|
135
|
+
| "cut to the product shot 0:12-0:16", "B-roll over this bit" | `broll.py talk.mp4 --insert product.mp4 --at 12 --end 16` (repeat `--insert/--at`; `--audio b\|mix`) |
|
|
136
|
+
| "add chapters", "chapter markers for YouTube" | `metadata.py episode.mp4 --chapters chapters.txt` (`TIME TITLE` per line; streams copied; a `render.py` project spells it `"chapters"`). `--auto-chapters` proposes them from measured pauses/scene cuts, titled `Chapter N`, renameable |
|
|
137
|
+
| "set the title / artist / comment" | `metadata.py episode.mp4 --title "Episode 12" --artist "Studio"` |
|
|
138
|
+
| "put these videos in a 4x2 grid" | `grid.py t1.mp4 ... t8.mp4 --cols 4 --rows 2` |
|
|
139
|
+
| "stitch these clips", "add a crossfade" | `join.py a.mp4 b.mp4 c.mp4 --transition fade --duration 0.5` |
|
|
123
140
|
| "several changes to the same edit", 3+ steps | `render.py --init project.json`, edit, `render.py project.json` |
|
|
124
141
|
| "I changed one stage, don't redo the rest" | `render.py project.json --cache DIR` — identical stages come from the cache (`--from STAGE` starts there) |
|
|
125
|
-
| "a lower third with my name", "countdown intro", "progress bar" | `graphics.py input.mp4 --template lower-third --name "..." --title "..." --start 2 --end 8` |
|
|
126
|
-
| "a sticker", "a hook card for the first 3 s", "meme text" | `graphics.py input.mp4 --template sticker --text "NEW" --platform tiktok` / `--template hook --title "..." --duration 3` / `--template meme --top "..." --bottom "..."` |
|
|
127
|
-
| "blurred background instead of black bars" | `fit.py input.mp4 --aspect 9:16 --fit blur` (whole picture kept, borders a blurred, darkened copy) |
|
|
128
|
-
| "use our brand fonts/colours/logo" | `--brand brand.json` on caption/overlay/graphics, or `"brand"` in project.json |
|
|
129
|
-
| "send me a summary of what you did" | `report.py --before raw.mov --after final.mp4 --platform youtube -o report.html` |
|
|
130
142
|
| "do this to every file in the folder", "use all the cores" | `batch.py FOLDER --recipe batch.json --jobs auto` (steps or a render project; cached) |
|
|
131
|
-
| "transcribe it and caption it" | `caption.py input.mp4 --transcribe --animate pop --karaoke` (needs a local whisper; else `--text`) |
|
|
132
143
|
| "three cameras, cut between them" | `multicam.py camA.mp4 camB.mp4 camC.mp4 --switch "0-20:0,20-40:1,40-60:2"` (manual) or `--switch energy` (auto-cuts to the loudest camera, `--min-shot`, `--edl`) |
|
|
133
|
-
| "
|
|
144
|
+
| "what's in this file", "how long is it" | `probe.py input.mp4` |
|
|
145
|
+
| "show me what it looks like", "are the captions readable" | `look.py output.mp4 --tiles 3x2`, then view the PNG |
|
|
146
|
+
| "what would you run?", "don't render yet" | any script with `--dry-run` |
|
|
147
|
+
| "which shots are static vs moving", "volume peaks per second", "is this speech or music" | `scenes.py input.mp4 --shots` (static/pan/motion per shot, measured flow) / `--audio-peaks` (dBFS list) / `--speech` (a speech-vs-music ratio, not a classification) — combine, or alone |
|
|
148
|
+
| "where does the subject move, so I can crop it myself" | `cropdetect.py input.mp4 --motion-centre` — motion centroid per second, report-only; the calling agent picks the crop |
|
|
134
149
|
| "does it look like Log / S-Log?" | `probe.py clip.mp4 --analyze` (`looks_like_log`) then `color.py --lut` |
|
|
135
150
|
| "test the tool on my real files" | `verify.py ~/Footage --report verify.md` |
|
|
136
151
|
| "show me progress", "quick preview first" | any encoding script with `--progress` and/or `--fast` |
|
|
137
|
-
| "the colours look washed out / iPhone HDR" | `color.py input.mov --to-sdr` (probe shows `hdr: true`) |
|
|
138
|
-
| "apply this LUT", "convert the S-Log footage" | `color.py input.mp4 --lut grade.cube [--lut-strength 0.7]` |
|
|
139
|
-
| "the colours are tagged wrong" | `color.py input.mp4 --retag bt709` (stream copy; re-encodes only if the copy can't carry it — see `reencoded`) |
|
|
140
|
-
| "brighten it / punch up contrast / fix white balance" | `color.py input.mp4 --correct --exposure 0.3 --contrast 1.1 --saturation 1.05 --temperature 5600 --tint -0.05` |
|
|
141
|
-
| "clean up the audio", "remove the hiss" | `audio.py input.mp4 --voice` (speech; `--voice light\|medium\|strong`) or `--denoise` |
|
|
142
|
-
| "add background music under the talking" | `audio.py input.mp4 --music bed.mp3 --duck --fade-out 3` (`--effects sfx.wav` adds a third bed, never ducked; project levels: `audio.stems`) |
|
|
143
|
-
| "make the music duck harder / come back faster" | add `--duck-amount 18 --duck-threshold -30 --duck-release 250` (`--duck-attack` too) |
|
|
144
|
-
| "the mix sounds narrow", "wider stereo" | `audio.py band.wav --stereo-widen 0.5` (needs a real stereo source; 5.1 needs `--downmix`) |
|
|
145
|
-
| "convert the 5.1 to stereo" | `audio.py input.mov --downmix` |
|
|
146
|
-
| "swap in the narration track" | `audio.py input.mp4 --replace narration.wav` |
|
|
147
|
-
| "pull the audio out", "give me the sound as WAV" | `audio.py input.mp4 -o input.wav` (an audio extension drops the picture; `--audio-stream 1` picks a track) |
|
|
148
|
-
| "compress the voice", "limit the peaks", "gate the room noise" | `audio.py input.mp4 --compress --comp-threshold -20 --comp-ratio 4` / `--limit --limit-ceiling -1` / `--gate --gate-threshold -45` |
|
|
149
|
-
| "the audio drifts out of sync over the hour" | `sync.py camera.mp4 recorder.wav --fix-drift --replace-audio` |
|
|
150
|
-
| "smooth slow motion", "half speed but fluid" | `fit.py input.mp4 --duration 2x --smooth interpolate` (slow) or `--smooth blend` |
|
|
151
|
-
| "TikTok-style captions with the words popping" | `caption.py input.mp4 --text cues.txt --animate pop --karaoke` |
|
|
152
152
|
| "it's a phone video with variable frame rate" | nothing extra: re-encodes conform VFR to constant fps; `fit.py --fps 30` picks the rate |
|
|
153
153
|
|
|
154
154
|
## Audio-only files
|
package/docs/contract.md
CHANGED
|
@@ -21,7 +21,7 @@ The contract is derived from the code that runs, not maintained beside it:
|
|
|
21
21
|
| Field | Meaning | Changes when |
|
|
22
22
|
|---|---|---|
|
|
23
23
|
| `contract_version` | shape of this document (`1.0`) | a key is renamed, removed or changes meaning |
|
|
24
|
-
| `skill.version` | the npm / package.json version (`1.
|
|
24
|
+
| `skill.version` | the npm / package.json version (`1.24.1`) | any release |
|
|
25
25
|
|
|
26
26
|
A release that adds a tool or a flag keeps `contract_version`; a breaking change to the
|
|
27
27
|
ToolSpec shape bumps it. Consumers pin on `contract_version` and read `skill.version`
|
|
@@ -88,19 +88,19 @@ spelling keeps working until 2.0.
|
|
|
88
88
|
|
|
89
89
|
| What 2.0 removes | Since | Replacement | To be ready today |
|
|
90
90
|
|---|---|---|---|
|
|
91
|
-
| The per-tool v1 success keys next to `result_v2` (`output`, `probe`, `commands`, `verified`, `verification` and each tool's own keys at the top level) | 1.
|
|
92
|
-
| `--crf` as an alias of `--quality` on every re-encoding tool that takes `--quality` (`export.py` keeps `--crf`: its preset chooses the encoder) | 1.
|
|
93
|
-
| `json` and `progress` in the MCP `inputSchema` | 1.
|
|
94
|
-
| `hdr` meaning "BT.2020 primaries *or* a PQ/HLG transfer" in `probe` | 1.
|
|
95
|
-
| Overwriting an existing output with only a warning | 1.
|
|
91
|
+
| The per-tool v1 success keys next to `result_v2` (`output`, `probe`, `commands`, `verified`, `verification` and each tool's own keys at the top level) | 1.24.1 | `result_v2`, promoted to the top level in 2.0 | Run with `FFMPEG_SKILL_RESULT_V2=1` and read `result_v2` (`metrics`, `notes`, `details`) instead of the top-level keys |
|
|
92
|
+
| `--crf` as an alias of `--quality` on every re-encoding tool that takes `--quality` (`export.py` keeps `--crf`: its preset chooses the encoder) | 1.24.1 | `--quality N` (the same CRF scale, codec-neutral) | Pass `--quality`; `--crf` warns on stderr and is marked in `--help` |
|
|
93
|
+
| `json` and `progress` in the MCP `inputSchema` | 1.24.1 | nothing: the transport sets them itself | Stop sending them from an MCP client; run the server with `FFMPEG_SKILL_MCP_LEAN=1` to see the 2.0 schema |
|
|
94
|
+
| `hdr` meaning "BT.2020 primaries *or* a PQ/HLG transfer" in `probe` | 1.24.1 | `hdr_signal` (true only for PQ / HLG / Dolby Vision); in 2.0 `hdr` takes that meaning | Key on `hdr_signal` for "is this a real HDR signal" and on `hdr_format` for the `BT.2020 SDR` case |
|
|
95
|
+
| Overwriting an existing output with only a warning | 1.24.1 | `--overwrite` as explicit consent (refused without it from 2.0) | Set `FFMPEG_SKILL_NO_OVERWRITE=1` (the recommended agent setting) and pass `--overwrite` where a replacement is intended |
|
|
96
96
|
|
|
97
97
|
## Skill
|
|
98
98
|
|
|
99
99
|
```json
|
|
100
100
|
{
|
|
101
101
|
"contract_version": "1.0",
|
|
102
|
-
"deprecated": [{"what": "...", "since": "1.
|
|
103
|
-
"skill": {"id": "ffmpeg-skill", "version": "1.
|
|
102
|
+
"deprecated": [{"what": "...", "since": "1.24.1", "replacement": "...", "removed_in": "2.0.0", "where": "cli | json | mcp | behaviour"}],
|
|
103
|
+
"skill": {"id": "ffmpeg-skill", "version": "1.24.1", "execution_mode": "local", "kind": "execution",
|
|
104
104
|
"entrypoints": {"cli": "...", "mcp": "...", "contract": "...", "doctor": "..."},
|
|
105
105
|
"not_provided": ["AI reasoning", "decisions", "production plans", "project IR", "approvals", "network access", "transcription engine"]},
|
|
106
106
|
"requirements": {"python": ">=3.9 (standard library only)", "ffmpeg": ">=5.0", "ffprobe": ">=5.0"},
|
|
@@ -120,7 +120,7 @@ One entry per tool under `tools`, sorted by id. Tool ids are stable:
|
|
|
120
120
|
| Field | Meaning |
|
|
121
121
|
|---|---|
|
|
122
122
|
| `id`, `name`, `version`, `executable` | `ffmpeg-skill/cut`, `cut`, skill version, `scripts/cut.py` |
|
|
123
|
-
| `examples[]` | `{prompts[], command}` (1.
|
|
123
|
+
| `examples[]` | `{prompts[], command}` (1.24.1): SKILL.md's own "User says" / "Do" table rows for this tool, machine-readable — `prompts` is the row's phrasing(s), `command` the row's `Do` text with the markdown backticks stripped. Every tool has at least one; parsed from SKILL.md at read time, so it never drifts from the table a person reads |
|
|
124
124
|
| `role` | `analysis`, `analysis_and_execution`, `execution` or `verification` (see below) |
|
|
125
125
|
| `capabilities.required` | ffmpeg components the tool always needs |
|
|
126
126
|
| `capabilities.optional[]` | `{capability, when}`: needed only for that flag or input |
|
|
@@ -129,7 +129,7 @@ One entry per tool under `tools`, sorted by id. Tool ids are stable:
|
|
|
129
129
|
| `output_schema` | what `--json` prints on stdout |
|
|
130
130
|
| `supports_dry_run`, `dry_run` | whether `--dry-run` plans without running ffmpeg or writing files |
|
|
131
131
|
| `supports_json` | whether `--json` exists |
|
|
132
|
-
| `supports_json_brief` | whether `--json-brief` exists (1.
|
|
132
|
+
| `supports_json_brief` | whether `--json-brief` exists (1.24.1): the same success document with `probe` replaced by a compact `summary` (`duration_s`, `width`, `height`, `fps`, `vcodec`, `acodec`, `channels`, and `lufs` when the tool measured one), `commands` replaced by the number of commands run, and the per-step `verification` list dropped (its verdict stays in `verified`). Tool-specific keys are unchanged, `--json`'s own output is unchanged, and a failure prints the same failure document either way |
|
|
133
133
|
| `mutates_input` | always `false`: no tool overwrites its input |
|
|
134
134
|
| `produces_artifact` | writes a file (media, PNG, HTML, EDL) |
|
|
135
135
|
| `verification` | `{required, tools}`: which tools to run on the output afterwards |
|
|
@@ -435,7 +435,7 @@ given a different type):
|
|
|
435
435
|
|
|
436
436
|
| key | tool | what it holds |
|
|
437
437
|
|---|---|---|
|
|
438
|
-
| `caption` | `caption.py` | the cue-layout counts the run only printed before (`shifted`, `wrapped`, `rebalanced`, `split`, `extended`, `dropped`) plus `wrap` (`"phrase"` or `"measured"`) and `phrase_breaks`, the number of cues a phrase rule broke somewhere the 1.15 width rule would not. `broken_inside_word` (1.
|
|
438
|
+
| `caption` | `caption.py` | the cue-layout counts the run only printed before (`shifted`, `wrapped`, `rebalanced`, `split`, `extended`, `dropped`) plus `wrap` (`"phrase"` or `"measured"`) and `phrase_breaks`, the number of cues a phrase rule broke somewhere the 1.15 width rule would not. `broken_inside_word` (1.24.1) counts atoms hard-sliced at the live column's edge because they did not fit alone even at the size floor — never a rewrite, the sliced pieces are the exact original characters. `overlong` is now residual: it fires only when even a single character is wider than the column |
|
|
439
439
|
| `tracks`, `subtitle_tracks` | `caption.py --mode mux` | one entry per subtitle stream in the output — `{index, file, language, title, codec, default, cues, kept_from_input}`; a stream the input already carried has `file: null` and `kept_from_input: true`. `subtitle_tracks` is the total. Every field describes the file as written, not as asked for: an MPEG-4 output reports `title: null` (the muxer stores none) and `default: true` on its first track (the muxer always enables it), each with a `notes` line |
|
|
440
440
|
| `auto_chapters` | `metadata.py --auto-chapters` | `{source, min_chapter, max_chapters, proposed, kept, titles, chapters, description_block, files}`. `titles` is always `"placeholder"`: the machine-readable form of "the skill did not name these". Each chapter carries its `evidence` (`start`, `silence`, `scene`, or `silence+scene` with the span, its length and the cut time) |
|
|
441
441
|
| `audiogram` | `waveform.py` (every run) | `{style, background, image, position, vis_height, platform, captions, title, stages, verified}`. `background` is `"image"` or `"color"`; `verified` is true when the render probes at the asked-for frame size, frame rate and within 0.05 s of the source audio, and is `false` under `--dry-run`, where nothing was rendered to verify |
|
|
@@ -451,7 +451,7 @@ Per-tool keys added in 1.17, all additive:
|
|
|
451
451
|
| `jobs`, `jobs_requested`, `wall_seconds`, `item_seconds_total`, `timed_out` | `batch.py` | the parallelism actually applied and the number asked for, the batch's wall clock, the sum of the per-item times (so the speed-up can be quoted), and whether the shared timeout budget ran out. A timed-out item carries `"skipped": "timeout"` in its result row |
|
|
452
452
|
| `cache` | `render.py --cache` | `{dir, ffmpeg, hits, misses, saved_seconds, entries}`, plus `would_hit` under `--dry-run`. The ffmpeg build banner, the skill version, the contract version, the forwarded flags (`--fast`, `--codec`, …) and the output's extension are all part of every key, so a cache is never reused across any of them — a `--fast` draft is never served to a run that did not ask for one |
|
|
453
453
|
|
|
454
|
-
Per-tool keys added in 1.
|
|
454
|
+
Per-tool keys added in 1.24.1, all additive:
|
|
455
455
|
|
|
456
456
|
| key | tool | what it holds |
|
|
457
457
|
|---|---|---|
|
|
@@ -459,7 +459,7 @@ Per-tool keys added in 1.23.0, all additive:
|
|
|
459
459
|
| `text_unchanged` | `caption.py` | a sibling inside the `caption` block, **burn mode only** (`--mode mux` never touches the text and omits the key): `true` when the drawn text equals the cues that were handed in — nothing transcribed, no cue dropped, no cue **split** across two consecutive cues and no glyph stripped (`--emoji none`). Wrapping, line breaks and timing do not count: the words are the same. This tool never rewrites, shortens or translates a cue, so the key is a statement of what happened, not a judgement of the text |
|
|
460
460
|
|
|
461
461
|
|
|
462
|
-
Per-tool keys added in 1.
|
|
462
|
+
Per-tool keys added in 1.24.1, all additive:
|
|
463
463
|
|
|
464
464
|
| key | tool | what it holds |
|
|
465
465
|
|---|---|---|
|
|
@@ -593,7 +593,21 @@ most often across the corpus in `evals/agent_prompts*.json`. Every tool -- inclu
|
|
|
593
593
|
-- is still callable by name through `tools/call` regardless of what `tools/list` advertised; the
|
|
594
594
|
contract (`ffmpeg-skill contract --json`) still describes all 42 unconditionally. Set
|
|
595
595
|
`FFMPEG_SKILL_MCP_FULL=1` (anything but "" or `0`) in the server's environment to make `tools/list`
|
|
596
|
-
return all 42, as every version before 1.
|
|
596
|
+
return all 42, as every version before 1.24.1 did.
|
|
597
|
+
|
|
598
|
+
### `prompts` capability: five workflow recipes
|
|
599
|
+
|
|
600
|
+
Roadmap 1.20.0 "agent ergonomics": the server advertises `capabilities.prompts` in `initialize`
|
|
601
|
+
and answers `prompts/list` / `prompts/get` with five canned workflows an agent can ask for by
|
|
602
|
+
name instead of composing the individual tool calls itself: `reel`, `podcast`, `multicam`,
|
|
603
|
+
`delivery_check`, `hdr`. `prompts/list` returns each one's `name`, `description` and `arguments`
|
|
604
|
+
(from `_contract.MCP_PROMPTS`); `prompts/get {name, arguments}` fills the matching template and
|
|
605
|
+
returns one user-role text message naming the actual command lines to run and what to check
|
|
606
|
+
afterward -- the same facts SKILL.md's own request table states, addressed by workflow instead of
|
|
607
|
+
by request phrasing. A missing required argument is a JSON-RPC error (`prompts/get` never guesses
|
|
608
|
+
a path or a platform); an unknown prompt name is a JSON-RPC error too, distinct from "method not
|
|
609
|
+
found" since `prompts/get` itself is a real method. These are recipes, not new tool calls: every
|
|
610
|
+
command line a prompt names is one `tools/call` (or the CLI) can already run.
|
|
597
611
|
|
|
598
612
|
## Consuming the contract from an agent
|
|
599
613
|
|
package/mcp/server.py
CHANGED
|
@@ -150,13 +150,24 @@ def handle(req: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
150
150
|
# 1.20.0: the structured-arguments note used to be appended to every one of the 42 tool
|
|
151
151
|
# descriptions verbatim; it's server-wide, not per-tool, so it now lives here once instead
|
|
152
152
|
# -- `tools/list` descriptions stay each tool's own one-line sentence.
|
|
153
|
-
return {"protocolVersion": PROTOCOL_VERSION, "capabilities": {"tools": {}},
|
|
153
|
+
return {"protocolVersion": PROTOCOL_VERSION, "capabilities": {"tools": {}, "prompts": {}},
|
|
154
154
|
"serverInfo": {"name": "ffmpeg-skill", "version": version()},
|
|
155
155
|
"instructions": _contract.MCP_STRUCTURED_NOTE}
|
|
156
156
|
if method == "tools/list":
|
|
157
157
|
return {"tools": tool_list()}
|
|
158
158
|
if method == "tools/call":
|
|
159
159
|
return call_tool(params.get("name", ""), params.get("arguments") or {})
|
|
160
|
+
if method == "prompts/list":
|
|
161
|
+
# roadmap 1.20.0: the five workflow recipes (reel, podcast, multicam, delivery_check, hdr)
|
|
162
|
+
return {"prompts": _contract.mcp_prompt_list()}
|
|
163
|
+
if method == "prompts/get":
|
|
164
|
+
try:
|
|
165
|
+
return _contract.mcp_prompt_get(params.get("name", ""), params.get("arguments") or {})
|
|
166
|
+
except KeyError as exc:
|
|
167
|
+
# not "method not found" (-32601, handle()'s own KeyError below): the *method*
|
|
168
|
+
# prompts/get exists, only this prompt *name* doesn't -- a caller mistake, same as
|
|
169
|
+
# call_tool()'s "unknown tool" text, not a transport-level error.
|
|
170
|
+
raise ValueError(f"unknown prompt: {exc}") from exc
|
|
160
171
|
if method == "ping":
|
|
161
172
|
return {}
|
|
162
173
|
raise KeyError(method)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ffmpeg-skill",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.24.1",
|
|
4
4
|
"description": "Agent Skill that gives coding agents (Claude Code, Cursor, Codex) a local video editor: 42 FFmpeg tools with a machine-readable contract, contract-derived MCP server, FFmpeg capability detection, probe-first / verify-last workflow. Cut, join, silence removal, fit, captions and karaoke, overlays, motion graphics, HDR to SDR, LUTs, audio clean-up and typed dynamics, sync with drift correction, multicam, loudness, delivery checks, project rendering, batch. No API keys, no cloud, no dependencies.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ffmpeg",
|
package/scripts/_contract.py
CHANGED
|
@@ -1128,6 +1128,92 @@ MCP_STRUCTURED_NOTE = ("Structured arguments: keys are the input_schema property
|
|
|
1128
1128
|
"are passed by name, output -> -o. Or argv: the raw CLI list (non-canonical; all other keys are then ignored). "
|
|
1129
1129
|
"Media paths must be absolute.")
|
|
1130
1130
|
|
|
1131
|
+
# roadmap 1.20.0 "agent ergonomics": a `prompts` capability with the five workflows an agent asks
|
|
1132
|
+
# for most often (per the SKILL.md request table these are built from -- see skill_examples()'s
|
|
1133
|
+
# own source, not a separate hand-picked list). Each `template` is filled with `{name}` -> the
|
|
1134
|
+
# matching argument's value; a missing optional argument's `{name}` is dropped along with the
|
|
1135
|
+
# surrounding text named in `optional` (so "no chapters file" doesn't leave a dangling clause).
|
|
1136
|
+
# The five are recipes, not new tool calls -- every command line named is one this skill already
|
|
1137
|
+
# runs; `prompts/get` hands back the same facts SKILL.md's own table states, just addressed by
|
|
1138
|
+
# workflow instead of by request phrasing.
|
|
1139
|
+
MCP_PROMPTS: List[Dict[str, Any]] = [
|
|
1140
|
+
{
|
|
1141
|
+
"name": "reel", "description": "Turn a clip into a captioned vertical delivery (TikTok/Reels/Shorts) and verify it.",
|
|
1142
|
+
"arguments": [{"name": "input", "description": "path to the source video", "required": True},
|
|
1143
|
+
{"name": "platform", "description": "tiktok | reels | shorts | youtube-shorts (default reels)", "required": False, "default": "reels"},
|
|
1144
|
+
{"name": "cues", "description": "path to a captions cue file, if the clip needs captions", "required": False}],
|
|
1145
|
+
"optional": {"cues": " with cues from {cues}"},
|
|
1146
|
+
"template": ("Turn {input} into a {platform} deliverable: run `render.py --template {platform} {input}`{cues}, "
|
|
1147
|
+
"which handles frame, captions, loudness and export in one command. Then verify with "
|
|
1148
|
+
"`check.py <output> --platform {platform}` before calling it done -- format rows (codec, pixel "
|
|
1149
|
+
"format, size, true peak, colour tags, VFR) are safe to fix; judgement rows (duration, aspect, "
|
|
1150
|
+
"fps, loudness) change content, so fix them only when the request implies the answer, otherwise "
|
|
1151
|
+
"state the choice and its cost. Mention WARNs; do not chase them."),
|
|
1152
|
+
},
|
|
1153
|
+
{
|
|
1154
|
+
"name": "podcast", "description": "Turn an audio recording into a loudness-normalised podcast episode with chapters.",
|
|
1155
|
+
"arguments": [{"name": "input", "description": "path to the source audio", "required": True},
|
|
1156
|
+
{"name": "chapters", "description": "path to a chapters timing file", "required": False}],
|
|
1157
|
+
"optional": {"chapters": " with chapters from `metadata.py <output> --chapters {chapters}`"},
|
|
1158
|
+
"template": ("Turn {input} into a podcast episode: normalise loudness with `loudness.py {input} -I -16 --tp -1.5`"
|
|
1159
|
+
"{chapters}, then verify with `check.py <output> --platform podcast` (its chapters and channels rows)."),
|
|
1160
|
+
},
|
|
1161
|
+
{
|
|
1162
|
+
"name": "multicam", "description": "Cut between several camera angles of the same scene.",
|
|
1163
|
+
"arguments": [{"name": "inputs", "description": "the camera files, space-separated", "required": True}],
|
|
1164
|
+
"optional": {},
|
|
1165
|
+
"template": ("Cut between the cameras in {inputs}: `multicam.py {inputs} --switch energy` auto-cuts to the "
|
|
1166
|
+
"loudest camera at each moment (`--min-shot` sets the shortest allowed shot); a manual spec "
|
|
1167
|
+
"looks like `--switch \"0-20:0,20-40:1,40-60:2\"`. Check the reported sync confidence per camera "
|
|
1168
|
+
"-- under 0.3 (or a large offset) is suspect and needs a manual look; these align audio, never lip sync."),
|
|
1169
|
+
},
|
|
1170
|
+
{
|
|
1171
|
+
"name": "delivery_check", "description": "Verify a finished file is ready to upload to a platform.",
|
|
1172
|
+
"arguments": [{"name": "input", "description": "path to the file to check", "required": True},
|
|
1173
|
+
{"name": "platform", "description": "youtube | reels | tiktok | shorts | x | linkedin | facebook | podcast", "required": True}],
|
|
1174
|
+
"optional": {},
|
|
1175
|
+
"template": ("Verify {input} is ready for {platform}: run `check.py {input} --platform {platform}`. Format "
|
|
1176
|
+
"rows (codec, pixel format, size, true peak, colour tags, VFR) are safe to fix automatically. "
|
|
1177
|
+
"Judgement rows (duration, aspect, fps, loudness) change content -- fix only when the request "
|
|
1178
|
+
"implies the answer, otherwise state the choice and its cost. Mention WARNs; do not chase them."),
|
|
1179
|
+
},
|
|
1180
|
+
{
|
|
1181
|
+
"name": "hdr", "description": "Bring an HDR or Dolby Vision clip down to SDR (or strip only the DV layer).",
|
|
1182
|
+
"arguments": [{"name": "input", "description": "path to the HDR/Dolby Vision source", "required": True},
|
|
1183
|
+
{"name": "strip_dovi", "description": "true to keep HDR and drop only the Dolby Vision layer, instead of converting to SDR", "required": False}],
|
|
1184
|
+
"optional": {},
|
|
1185
|
+
"template": ("Bring {input} from HDR/Dolby Vision down to SDR: `color.py {input} --to-sdr` (or `--strip-dovi` "
|
|
1186
|
+
"to keep HDR and only drop the Dolby Vision layer, when strip_dovi is asked for). A flat, washed-out "
|
|
1187
|
+
"look after a re-encode usually means HDR metadata was lost upstream; `probe` shows `hdr: true` "
|
|
1188
|
+
"for BT.2020 primaries or a PQ/HLG transfer, and `hdr_signal: true` only for a real PQ/HLG/DV transfer."),
|
|
1189
|
+
},
|
|
1190
|
+
]
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
def mcp_prompt_list() -> List[Dict[str, Any]]:
|
|
1194
|
+
"""`prompts/list`: name, description and arguments for each of MCP_PROMPTS -- everything but
|
|
1195
|
+
the template, which `prompts/get` fills."""
|
|
1196
|
+
return [{"name": p["name"], "description": p["description"], "arguments": p["arguments"]} for p in MCP_PROMPTS]
|
|
1197
|
+
|
|
1198
|
+
|
|
1199
|
+
def mcp_prompt_get(name: str, arguments: Dict[str, Any]) -> Dict[str, Any]:
|
|
1200
|
+
"""`prompts/get`: the filled template as a single user-role text message. Raises KeyError for
|
|
1201
|
+
an unknown prompt name (mirrors call_tool's own unknown-tool handling) and ValueError naming
|
|
1202
|
+
the missing argument when a required one is absent -- both are caller mistakes, not server
|
|
1203
|
+
failures, so the transport turns them into a JSON-RPC error rather than a crash."""
|
|
1204
|
+
by_name = {p["name"]: p for p in MCP_PROMPTS}
|
|
1205
|
+
if name not in by_name:
|
|
1206
|
+
raise KeyError(name)
|
|
1207
|
+
prompt = by_name[name]
|
|
1208
|
+
missing = [a["name"] for a in prompt["arguments"] if a["required"] and not (arguments or {}).get(a["name"])]
|
|
1209
|
+
if missing:
|
|
1210
|
+
raise ValueError(f"prompt {name!r} needs {', '.join(missing)}")
|
|
1211
|
+
fields: Dict[str, str] = {a["name"]: str((arguments or {}).get(a["name"]) or a.get("default", "")) for a in prompt["arguments"]}
|
|
1212
|
+
for opt_name, clause in prompt["optional"].items():
|
|
1213
|
+
fields[opt_name] = clause.format(**fields) if fields.get(opt_name) else ""
|
|
1214
|
+
text = prompt["template"].format(**fields)
|
|
1215
|
+
return {"description": prompt["description"], "messages": [{"role": "user", "content": {"type": "text", "text": text}}]}
|
|
1216
|
+
|
|
1131
1217
|
|
|
1132
1218
|
def mcp_input_schema(spec: Dict[str, Any]) -> Dict[str, Any]:
|
|
1133
1219
|
"""Translate a ToolSpec.input_schema into the JSON Schema an MCP tools/list entry carries.
|
package/scripts/caption.py
CHANGED
|
@@ -25,6 +25,7 @@ Text-to-SRT input format (one cue per line, blank lines ignored):
|
|
|
25
25
|
Examples:
|
|
26
26
|
python3 caption.py input.mp4 --srt subs.srt
|
|
27
27
|
python3 caption.py input.mp4 --text cues.txt --animate pop --karaoke # word-by-word highlight, TikTok style
|
|
28
|
+
python3 caption.py input.mp4 --text cues.txt --karaoke --karaoke-style word # active word scaled + emboldened, past/upcoming colours
|
|
28
29
|
python3 caption.py input.mp4 --srt subs.srt --font "Noto Sans CJK JP" --size 28 --position top
|
|
29
30
|
python3 caption.py --text cues.txt --write-srt cues.srt # only produce the SRT
|
|
30
31
|
python3 caption.py input.mp4 --text cues.txt # generate + burn in one go
|
|
@@ -464,9 +465,18 @@ def max_line_em(args, play_w: Optional[int], play_h: Optional[int]) -> Optional[
|
|
|
464
465
|
|
|
465
466
|
--size is in ASS points against a 288-line script (what libass's force_style uses), so the
|
|
466
467
|
rendered pixel size is size * play_h / 288.
|
|
468
|
+
|
|
469
|
+
--karaoke-style word scales its active word up by --karaoke-scale and freezes the line break
|
|
470
|
+
(WrapStyle: 2 in write_ass) so libass can never re-wrap around that scale-up. The budget here
|
|
471
|
+
is narrowed by the same factor so the longest wrapped line -- computed at the UNscaled width --
|
|
472
|
+
still fits once its active word is drawn --karaoke-scale%% larger; without this, a line that
|
|
473
|
+
exactly filled the safe width would overflow it the moment its emphasised word appears.
|
|
467
474
|
"""
|
|
468
|
-
|
|
469
|
-
|
|
475
|
+
em = line_em_for_size(args.size, play_w, play_h,
|
|
476
|
+
safe_fraction=safe_width_fraction(args, play_w))
|
|
477
|
+
if em is not None and getattr(args, "karaoke", False) and getattr(args, "karaoke_style", "sweep") == "word":
|
|
478
|
+
em = em / max(1.0, getattr(args, "karaoke_scale", 112) / 100.0)
|
|
479
|
+
return em
|
|
470
480
|
|
|
471
481
|
|
|
472
482
|
def parse_ass_dialogue(path: str) -> str:
|
|
@@ -526,6 +536,23 @@ def word_durations_from_timings(words: List[Tuple[float, float, str]], start: fl
|
|
|
526
536
|
return out
|
|
527
537
|
|
|
528
538
|
|
|
539
|
+
def _karaoke_word_durations(args, video: Optional[str], start: float, end: float, n_words: int) -> List[int]:
|
|
540
|
+
"""Centiseconds per word for one cue, shared by --karaoke-style sweep and word.
|
|
541
|
+
|
|
542
|
+
Real word timings from the transcript beat both the energy estimate and the even split --
|
|
543
|
+
they are what the speaker actually did, not a proxy for it.
|
|
544
|
+
"""
|
|
545
|
+
dur_cs = max(1, int(round((end - start) * 100)))
|
|
546
|
+
durs = word_durations_from_timings(getattr(args, "_word_timings", None) or [], start, end, n_words)
|
|
547
|
+
if durs is None:
|
|
548
|
+
if getattr(args, "karaoke_timing", "even") == "energy" and video:
|
|
549
|
+
durs = word_durations_from_audio(video, start, end, n_words, getattr(args, "audio_stream", 0))
|
|
550
|
+
else:
|
|
551
|
+
per = max(1, dur_cs // max(1, n_words))
|
|
552
|
+
durs = [per] * n_words
|
|
553
|
+
return durs
|
|
554
|
+
|
|
555
|
+
|
|
529
556
|
def write_ass(cues: List[Tuple[float, float, str]], path: str, args, play_w: int, play_h: int, video: str = None) -> None:
|
|
530
557
|
"""Write a styled ASS file with optional animation and word-by-word highlight."""
|
|
531
558
|
def t(sec: float) -> str:
|
|
@@ -544,8 +571,20 @@ def write_ass(cues: List[Tuple[float, float, str]], path: str, args, play_w: int
|
|
|
544
571
|
secondary = ass_color(args.color)
|
|
545
572
|
outline = ass_color(args.outline_color)
|
|
546
573
|
back = ass_color(args.outline_color, 0x80)
|
|
574
|
+
word_style = args.karaoke and getattr(args, "karaoke_style", "sweep") == "word"
|
|
575
|
+
word_past = ass_color(args.color)
|
|
576
|
+
word_active = ass_color(args.highlight_color)
|
|
577
|
+
word_upcoming = ass_color(args.upcoming_color)
|
|
578
|
+
word_bord = f"{args.outline * scale * (max(1.0, args.karaoke_scale) / 100.0):.2f}"
|
|
579
|
+
word_scale = int(round(args.karaoke_scale))
|
|
580
|
+
# --karaoke-style word emits one Dialogue per word carrying the WHOLE cue with explicit \N
|
|
581
|
+
# breaks (the layout's own fixed line list -- see max_line_em's docstring). WrapStyle: 2 tells
|
|
582
|
+
# libass to honour only those explicit breaks and never re-wrap: with WrapStyle: 0 (the sweep
|
|
583
|
+
# default, unchanged), scaling the active word up would re-flow the line and it would visibly
|
|
584
|
+
# jump every time the highlight advances (the trap the issue this implements is filed against).
|
|
547
585
|
header = [
|
|
548
|
-
"[Script Info]", "ScriptType: v4.00+", f"PlayResX: {play_w}", f"PlayResY: {play_h}",
|
|
586
|
+
"[Script Info]", "ScriptType: v4.00+", f"PlayResX: {play_w}", f"PlayResY: {play_h}",
|
|
587
|
+
f"WrapStyle: {2 if word_style else 0}", "ScaledBorderAndShadow: yes", "",
|
|
549
588
|
"[V4+ Styles]",
|
|
550
589
|
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
|
|
551
590
|
f"Style: Default,{ass_font_name(args.font)},{size},{primary},{secondary},{outline},{back},{-1 if args.bold else 0},0,0,0,100,100,0,0,{3 if args.box else 1},{args.outline * scale:.1f},{args.shadow * scale:.1f},{ALIGN[args.position]},{margin_l},{margin_r},{margin},1",
|
|
@@ -579,20 +618,66 @@ def write_ass(cues: List[Tuple[float, float, str]], path: str, args, play_w: int
|
|
|
579
618
|
own = own_sizes[cue_index] if cue_index < len(own_sizes) else None
|
|
580
619
|
if own and own != args.size:
|
|
581
620
|
fx += "{\\fs%d}" % int(round(own * scale))
|
|
621
|
+
if args.karaoke and word_style:
|
|
622
|
+
# One Dialogue event per WORD, each carrying the whole cue -- \k/\kf can only
|
|
623
|
+
# interpolate colour, so scaling/emboldening the active word (and a third "upcoming"
|
|
624
|
+
# colour) needs a real per-word override block instead. The line list is `body`'s own
|
|
625
|
+
# \N split, already frozen by layout_cues (this function never re-wraps it), so every
|
|
626
|
+
# event uses the identical break -- WrapStyle: 2 above stops libass re-wrapping around
|
|
627
|
+
# the active word's larger \fscx/\fscy, which is the jump the issue is filed against.
|
|
628
|
+
line_words = [[w for w in seg.split(" ") if w] for seg in body.split("\\N")]
|
|
629
|
+
flat = [w for lw in line_words for w in lw]
|
|
630
|
+
real_idx = [i for i, w in enumerate(flat) if w.strip(EMOJI_SENTINEL)]
|
|
631
|
+
if not real_idx:
|
|
632
|
+
# nothing to highlight (an all-emoji or empty cue) -- one plain event, same as
|
|
633
|
+
# today's karaoke-off rendering, rather than zero Dialogue lines for this cue
|
|
634
|
+
out_body = body
|
|
635
|
+
if EMOJI_SENTINEL in out_body:
|
|
636
|
+
out_body = out_body.replace(EMOJI_SENTINEL, emoji_placeholder(getattr(args, "_emoji_box_px", size)))
|
|
637
|
+
lines.append(f"Dialogue: 0,{t(start)},{t(end)},Default,,0,0,0,,{fx}{out_body}")
|
|
638
|
+
continue
|
|
639
|
+
n_words = len(real_idx)
|
|
640
|
+
durs = _karaoke_word_durations(args, video, start, end, n_words)
|
|
641
|
+
starts_cs, acc = [], 0
|
|
642
|
+
for d in durs:
|
|
643
|
+
starts_cs.append(acc)
|
|
644
|
+
acc += d
|
|
645
|
+
for j in range(n_words):
|
|
646
|
+
w_start = start + starts_cs[j] / 100.0
|
|
647
|
+
w_end = start + (starts_cs[j] + durs[j]) / 100.0
|
|
648
|
+
# colour/scale each real word by its rank against the active one (j); an
|
|
649
|
+
# emoji-placeholder token is never active and just keeps its plain text
|
|
650
|
+
styled, flat_pos = [], 0
|
|
651
|
+
for lw in line_words:
|
|
652
|
+
styled_line = []
|
|
653
|
+
for w in lw:
|
|
654
|
+
if not w.strip(EMOJI_SENTINEL):
|
|
655
|
+
styled_line.append(w)
|
|
656
|
+
else:
|
|
657
|
+
rank = real_idx.index(flat_pos)
|
|
658
|
+
if rank < j:
|
|
659
|
+
styled_line.append(f"{{\\c{word_past}}}{w}")
|
|
660
|
+
elif rank == j:
|
|
661
|
+
styled_line.append(
|
|
662
|
+
f"{{\\c{word_active}\\fscx{word_scale}\\fscy{word_scale}\\bord{word_bord}}}{w}{{\\r}}")
|
|
663
|
+
else:
|
|
664
|
+
styled_line.append(f"{{\\c{word_upcoming}}}{w}")
|
|
665
|
+
flat_pos += 1
|
|
666
|
+
styled.append(" ".join(styled_line))
|
|
667
|
+
out_body = "\\N".join(styled)
|
|
668
|
+
if EMOJI_SENTINEL in out_body:
|
|
669
|
+
out_body = out_body.replace(EMOJI_SENTINEL, emoji_placeholder(getattr(args, "_emoji_box_px", size)))
|
|
670
|
+
# the size/fit override (\fsN) belongs on every event; the entrance animation
|
|
671
|
+
# (\fad/\move/pop) only on the cue's FIRST word event -- replaying it on every
|
|
672
|
+
# later word would re-fade/re-pop the whole cue each time the highlight advances
|
|
673
|
+
event_fx = fx if j == 0 else (fx[fx.index("{\\fs"):] if "{\\fs" in fx else "")
|
|
674
|
+
lines.append(f"Dialogue: 0,{t(w_start)},{t(w_end)},Default,,0,0,0,,{event_fx}{out_body}")
|
|
675
|
+
continue
|
|
582
676
|
if args.karaoke:
|
|
583
677
|
# split each line into words and give every word an equal share of the cue (\k is in centiseconds)
|
|
584
|
-
dur_cs = max(1, int(round((end - start) * 100)))
|
|
585
678
|
segments = body.split("\\N")
|
|
586
679
|
words = [w for seg in segments for w in seg.split(" ") if w and w.strip(EMOJI_SENTINEL)]
|
|
587
|
-
|
|
588
|
-
# split -- they are what the speaker actually did, not a proxy for it
|
|
589
|
-
durs = word_durations_from_timings(getattr(args, "_word_timings", None) or [], start, end, len(words))
|
|
590
|
-
if durs is None:
|
|
591
|
-
if getattr(args, "karaoke_timing", "even") == "energy" and video:
|
|
592
|
-
durs = word_durations_from_audio(video, start, end, len(words), getattr(args, "audio_stream", 0))
|
|
593
|
-
else:
|
|
594
|
-
per = max(1, dur_cs // max(1, len(words)))
|
|
595
|
-
durs = [per] * len(words)
|
|
680
|
+
durs = _karaoke_word_durations(args, video, start, end, len(words))
|
|
596
681
|
it = iter(durs)
|
|
597
682
|
out_segments = []
|
|
598
683
|
for seg in segments:
|
|
@@ -840,7 +925,16 @@ def main() -> int:
|
|
|
840
925
|
anim = ap.add_argument_group("animation (generates ASS; needs --text or --srt input)")
|
|
841
926
|
anim.add_argument("--animate", choices=["none", "fade", "pop", "slide"], default=None, help="per-cue entrance animation (default none, or brand caption.animate)")
|
|
842
927
|
anim.add_argument("--karaoke", action="store_true", help="word-by-word highlight (fills from --color to --highlight-color across each cue)")
|
|
843
|
-
anim.add_argument("--
|
|
928
|
+
anim.add_argument("--karaoke-style", choices=["sweep", "word"], default="sweep",
|
|
929
|
+
help="'sweep' (default): today's \\kf colour fill, one Dialogue event per cue -- unchanged. "
|
|
930
|
+
"'word': one Dialogue event per word, the active word scaled by --karaoke-scale and "
|
|
931
|
+
"emboldened, past words in --color, upcoming words in --upcoming-color -- \\k/\\kf can only "
|
|
932
|
+
"interpolate colour, so this is the only way to also scale/embolden the active word")
|
|
933
|
+
anim.add_argument("--highlight-color", default=None, help="karaoke fill/active-word colour RRGGBB (default FFD200 or brand primary)")
|
|
934
|
+
anim.add_argument("--upcoming-color", default=None,
|
|
935
|
+
help="--karaoke-style word: colour RRGGBB for words not yet spoken (default B4B4B4)")
|
|
936
|
+
anim.add_argument("--karaoke-scale", type=float, default=112.0,
|
|
937
|
+
help="--karaoke-style word: the active word's size as a percentage of the caption size (default 112)")
|
|
844
938
|
anim.add_argument("--karaoke-timing", choices=["even", "energy"], default="energy",
|
|
845
939
|
help="how words are timed inside a cue: 'energy' follows the speech loudness in the audio (default), 'even' splits time equally")
|
|
846
940
|
anim.add_argument("--write-ass", help="where to save the generated ASS (default: next to the output)")
|
|
@@ -886,6 +980,9 @@ def main() -> int:
|
|
|
886
980
|
# to be applied anyway and then refused as "animation is burn only" -- ignore it there
|
|
887
981
|
args.animate = args.animate or (bcap.get("animate", "none") if args.brand and args.mode != "mux" else "none")
|
|
888
982
|
args.highlight_color = color_hex(args.highlight_color or bc.get("primary", "FFD200"))
|
|
983
|
+
args.upcoming_color = color_hex(args.upcoming_color or bc.get("upcoming", "B4B4B4"))
|
|
984
|
+
if args.karaoke_scale <= 0:
|
|
985
|
+
die("--karaoke-scale must be positive")
|
|
889
986
|
if args.brand and bcap.get("bold") and not args.bold:
|
|
890
987
|
args.bold = True
|
|
891
988
|
if args.brand and bcap.get("karaoke") and not args.karaoke:
|
package/scripts/loudness.py
CHANGED
|
@@ -105,6 +105,8 @@ def main() -> int:
|
|
|
105
105
|
cmd += ["-vn"] + audio_codec_for(output, bitrate)
|
|
106
106
|
else:
|
|
107
107
|
cmd += ["-map", "0:v:0", "-map", "0:a:0", "-c:v", "copy", "-c:a", "aac", "-b:a", bitrate]
|
|
108
|
+
if ext in (".mp4", ".mov", ".m4v"):
|
|
109
|
+
cmd += ["-movflags", "+faststart"]
|
|
108
110
|
dropped_streams = run_keeping_subtitles(cmd, output)
|
|
109
111
|
return
|
|
110
112
|
cmd.append(output)
|