ffmpeg-skill 3.4.3 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/docs/contract.md +4 -2
- package/package.json +1 -1
- package/references/scripts.md +24 -2
- package/scripts/_contract.py +5 -1
- package/scripts/render.py +3 -2
- package/scripts/silence.py +160 -11
package/README.md
CHANGED
|
@@ -587,9 +587,10 @@ Where it can be installed from today. Each route is spelled out below the table.
|
|
|
587
587
|
| Codex plugin | Codex CLI | `codex plugin marketplace add kajisho5/ffmpeg-skill --sparse .claude-plugin`, then `codex plugin add ffmpeg-skill@ffmpeg-skill` |
|
|
588
588
|
| Gemini CLI extension | Gemini CLI | `gemini extensions install https://github.com/kajisho5/ffmpeg-skill` |
|
|
589
589
|
| MCP server | clients without a shell | `npx -y ffmpeg-skill mcp` (see [MCP](#mcp)) |
|
|
590
|
+
| MCP Registry | clients that install from the registry | `io.github.kajisho5/ffmpeg-skill` ([entry](https://registry.modelcontextprotocol.io/v0/servers/io.github.kajisho5%2Fffmpeg-skill/versions/latest)); the release workflow publishes a new version there after npm, and a failed registry step leaves the entry on the previous version |
|
|
590
591
|
| GitHub release | a pinned copy | [releases](https://github.com/kajisho5/ffmpeg-skill/releases) |
|
|
591
592
|
|
|
592
|
-
Not listed in a directory yet: the Claude plugin directory, the Cursor Marketplace, the Gemini CLI extension gallery
|
|
593
|
+
Not listed in a directory yet: the Claude plugin directory, the Cursor Marketplace, the Gemini CLI extension gallery and Glama (each is submitted and waiting on its review or crawl). This table gains a row when a listing goes live; until then the routes above are the whole list.
|
|
593
594
|
|
|
594
595
|
```bash
|
|
595
596
|
npx ffmpeg-skill # Claude Code → ~/.claude/skills/ffmpeg-skill
|
package/docs/contract.md
CHANGED
|
@@ -21,7 +21,7 @@ The contract is derived from the code that runs, not maintained beside it:
|
|
|
21
21
|
| Field | Meaning | Changes when |
|
|
22
22
|
|---|---|---|
|
|
23
23
|
| `contract_version` | shape of this document (`2.0`) | a key is renamed, removed or changes meaning |
|
|
24
|
-
| `skill.version` | the npm / package.json version (`3.
|
|
24
|
+
| `skill.version` | the npm / package.json version (`3.6.0`) | any release |
|
|
25
25
|
|
|
26
26
|
A release that adds a tool or a flag keeps `contract_version`; a breaking change to the
|
|
27
27
|
ToolSpec shape bumps it. Consumers pin on `contract_version` and read `skill.version`
|
|
@@ -149,7 +149,7 @@ JSON would have been rewritten for that. The flat keys, typed per tool in `outpu
|
|
|
149
149
|
"deprecated": [],
|
|
150
150
|
"removed": [{"what": "...", "since": "1.10.0", "replacement": "...", "removed_in": "2.0.0", "where": "cli | json | mcp | behaviour"},
|
|
151
151
|
{"what": "running on FFmpeg 5.x (the requirements.ffmpeg floor)", "since": "2.14.0", "replacement": "FFmpeg 6.1 or later; ...", "removed_in": "3.0.0", "where": "behaviour"}],
|
|
152
|
-
"skill": {"id": "ffmpeg-skill", "version": "3.
|
|
152
|
+
"skill": {"id": "ffmpeg-skill", "version": "3.6.0", "execution_mode": "local", "kind": "execution",
|
|
153
153
|
"entrypoints": {"cli": "...", "mcp": "...", "contract": "...", "doctor": "..."},
|
|
154
154
|
"not_provided": ["AI reasoning", "decisions", "production plans", "project IR", "approvals", "network access", "transcription engine"]},
|
|
155
155
|
"requirements": {"python": ">=3.10 (standard library only)", "ffmpeg": ">=6.1", "ffprobe": ">=6.1"},
|
|
@@ -752,6 +752,8 @@ silence a tool reported as fine while another measurement disagreed):
|
|
|
752
752
|
| `verification[]` step `duration` | `export.py` (a preset with a platform maximum, without `--allow-long`) | the written file's length against the maximum, measured as `check.py` measures it (the container's duration, `tolerance` 0). A length that cannot be read fails the step with `reason: not_measured` |
|
|
753
753
|
| `verification[]` step `duration` (`property: audio_duration`) | `loudness.py` | the output's sound against the input's, to one AAC frame (at least 0.03 s): a levels pass changes no timing. A length that cannot be read fails the step with `reason: not_measured` |
|
|
754
754
|
| `inputs_duration`, `transition_shortening`; step `duration`'s `inputs_total` | `join.py` | the clips' own total and how much the transitions took out of it. `expected_duration` is derived from the transition the join applied, so it cannot show that on its own |
|
|
755
|
+
| `margins` | `silence.py` | `{lead, tail}`: the seconds of each silence kept before speech starts (`--margin-lead`) and after it ends (`--margin-tail`); both equal `--margin` unless one is named. Absent under `--filler` with neither `--speech-aware` nor a sentence flag, which removes no silences |
|
|
756
|
+
| `silence_positions`, `sentence` | `silence.py --sentence-min-silence` / `--sentence-margin-tail` | per entry of `silences`, `sentence_end` when the transcript word before it ends in sentence-final punctuation, else `inside`; and `{min_silence, margin_tail, source, word_timings, sentence_end_words, silences}`. `sentence_end_words: 0` means the transcript had no such punctuation, so every silence used `--min-silence` and `--margin-tail` (warning `sentence_ends_not_found`) |
|
|
755
757
|
| `silence_kinds`, `silence_breakdown` | `silence.py` | per detected silence, `digital` (at or below -90 dBFS) or `below_threshold` (quiet sound under `--threshold`), and the count and seconds of each with the level that defines it |
|
|
756
758
|
|
|
757
759
|
Keys added after 2.21, all additive (#367 follow-up, and a report of a filename with a space that
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ffmpeg-skill",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.6.0",
|
|
4
4
|
"mcpName": "io.github.kajisho5/ffmpeg-skill",
|
|
5
5
|
"description": "Agent Skill that gives coding agents (Claude Code, Cursor, Codex) a local video editor: 42 FFmpeg tools with a machine-readable contract, contract-derived MCP server, FFmpeg capability detection, probe-first / verify-last workflow. Cut, join, silence removal, fit, captions and karaoke, overlays, motion graphics, HDR to SDR, LUTs, audio clean-up and typed dynamics, sync with drift correction, multicam, loudness, delivery checks, project rendering, batch. No API keys, no cloud, no dependencies.",
|
|
6
6
|
"keywords": [
|
package/references/scripts.md
CHANGED
|
@@ -697,7 +697,8 @@ each shorter clip's last frame (with silence) out to the longest.
|
|
|
697
697
|
|
|
698
698
|
### silence.py — remove dead air / jump cuts
|
|
699
699
|
```
|
|
700
|
-
silence.py INPUT [--threshold -35] [--min-silence 0.6] [--margin 0.2] [--
|
|
700
|
+
silence.py INPUT [--threshold -35] [--min-silence 0.6] [--margin 0.2] [--margin-lead S] [--margin-tail S] [--min-keep 0.2] [--list]
|
|
701
|
+
[--sentence-min-silence S] [--sentence-margin-tail S] (with --words FILE or --transcribe) [--edl keep.txt] [--timemap map.json|map.csv] [--speech-aware] [-o OUT]
|
|
701
702
|
silence.py INPUT --filler --words transcript.json [--filler-lang auto|en|ja|ko|zh|es|de|fr|pt|it|nl|pl|ru|tr]
|
|
702
703
|
[--filler-words FILE] [--filler-extra W,W] [--filler-keep W,W] [--filler-pad 0.02]
|
|
703
704
|
[--transcribe] [--engine auto|parakeet-mlx|parakeet.cpp|whisper.cpp|faster-whisper|openai-whisper]
|
|
@@ -713,10 +714,31 @@ to `-45`; noisy ones `-30`. Always tell the user how many seconds were removed.
|
|
|
713
714
|
|
|
714
715
|
**Breaths and weak word endings.** `--margin` is the padding kept on each side of every cut
|
|
715
716
|
(default 0.2 s since 2.22, 0.15 s before; 0.3 s for a speaker who trails off). It is kept around
|
|
716
|
-
silences only: a filler word's cut is the word plus `--filler-pad` (since 3.1.6). `--
|
|
717
|
+
silences only: a filler word's cut is the word plus `--filler-pad` (since 3.1.6). `--margin-tail`
|
|
718
|
+
is what is kept after speech ends and `--margin-lead` what is kept before it starts again; each
|
|
719
|
+
defaults to `--margin`, and the result's `margins` gives the two values used. A last syllable that
|
|
720
|
+
is mostly breath wants more tail than the next word wants lead (`--margin-lead 0.1 --margin-tail
|
|
721
|
+
0.4`). `--speech-aware` keeps the short
|
|
717
722
|
breaths inside a sentence and cuts only at sentence-boundary pauses. A `--threshold` set too high
|
|
718
723
|
(-25, say) counts a weak word ending as silence: the result then warns.
|
|
719
724
|
|
|
725
|
+
**Pauses inside a sentence and after one.** With word timings (`--words FILE` or `--transcribe`),
|
|
726
|
+
`--sentence-min-silence S` and `--sentence-margin-tail S` give a silence that follows a sentence
|
|
727
|
+
end its own floor and its own tail; every other silence keeps `--min-silence` and `--margin-tail`.
|
|
728
|
+
A sentence end is a transcript word whose text ends in `. ! ? 。 ! ? …` (also `।` and `؟`), read
|
|
729
|
+
from the transcript and never from the sound; the word before a silence is the last one
|
|
730
|
+
whose own midpoint lies before the silence's midpoint, which holds when the engine stretched a
|
|
731
|
+
word's timing into the pause. `--words talk.json --min-silence 0.2 --sentence-min-silence
|
|
732
|
+
0.3 --margin 0.1 --sentence-margin-tail 0.3` tightens pauses inside a sentence from 0.2 s and
|
|
733
|
+
after one from 0.3 s, leaving the sentence end more room. The result lists `silence_positions`
|
|
734
|
+
(`sentence_end` or `inside`, one per entry of `silences`) and `sentence` (`min_silence`,
|
|
735
|
+
`margin_tail`, `source`, `word_timings`, `sentence_end_words`, `silences`). With no such punctuation
|
|
736
|
+
in the transcript no silence is a sentence end: every one uses `--min-silence` and `--margin-tail`,
|
|
737
|
+
and the run warns `sentence_ends_not_found`. Not with `--speech-aware`,
|
|
738
|
+
which takes a sentence boundary from the pause's length instead; without a transcript the flags
|
|
739
|
+
are refused. With `--filler` they turn silence removal on as well (`--filler` alone removes
|
|
740
|
+
only the words), reading the same transcript.
|
|
741
|
+
|
|
720
742
|
**Cut levels (2.22).** Every run measures the RMS level 50 ms either side of each cut on the
|
|
721
743
|
source (`cut_points`: `source_time`, `output_time`, `edge` out/in, `rms_dbfs` of the loudest
|
|
722
744
|
channel, `null` for digital silence; the whole key is `null`, with a `cut_levels_unmeasured`
|
package/scripts/_contract.py
CHANGED
|
@@ -243,7 +243,7 @@ DRY_RUN_ANALYSIS = {
|
|
|
243
243
|
"scenes": "scene and audio-peak measurement runs; --sheet and --edl are not written",
|
|
244
244
|
"report": "probe, loudness and contact-sheet measurements run; the HTML is not written",
|
|
245
245
|
"cropdetect": "the cropdetect filter runs over the sampled windows to measure bars; this tool never writes a file regardless of --dry-run",
|
|
246
|
-
"silence": "silencedetect runs so the reported silences and keep ranges are real; the cut output is not written. --filler --transcribe also runs the speech engine (its word timings are what the filler spans are planned on), which may download a model on first use",
|
|
246
|
+
"silence": "silencedetect runs so the reported silences and keep ranges are real; the cut output is not written. --filler --transcribe also runs the speech engine (its word timings are what the filler spans are planned on), which may download a model on first use; so does --transcribe with --sentence-min-silence or --sentence-margin-tail",
|
|
247
247
|
"loudness": "the loudnorm measurement pass runs so input_i and the planned pass-2 command are real; the normalised output is not written",
|
|
248
248
|
"check": "read-only tool; the loudness measurement (or, when it is skipped, the audio row's volumedetect) and the --content decode run under --dry-run too, so every row is present",
|
|
249
249
|
"stabilize": "vidstabdetect (pass 1, into a temp file) runs; the stabilised output (pass 2) is not written",
|
|
@@ -500,6 +500,7 @@ DIAGNOSTIC_CODES = {
|
|
|
500
500
|
"sync_low_confidence": ("warnings", "sync.py / multicam.py: a source's offset has a correlation confidence below 0.1; {path, confidence}"),
|
|
501
501
|
"cropdetect_windows_failed": ("warnings", "cropdetect.py: sampled windows ffmpeg could not decode (windows_failed); {failed, windows, starts}"),
|
|
502
502
|
"filler_words_untimed": ("warnings", "silence.py --filler: listed words in the transcript with no usable timing were not removed (filler.matched_untimed); {count}"),
|
|
503
|
+
"sentence_ends_not_found": ("warnings", "silence.py --sentence-min-silence / --sentence-margin-tail: the transcript has no sentence-final punctuation, so every silence used --min-silence and --margin-tail (sentence.sentence_end_words is 0)"),
|
|
503
504
|
"cut_near_sound": ("warnings", "silence.py: a cut point's level is within CUT_SOUND_MARGIN_DB of --threshold, so it may land in a breath or a word's weak tail (cut_warnings lists them)"),
|
|
504
505
|
"cut_levels_unmeasured": ("warnings", "silence.py: the source could not be decoded to measure the level at each cut; cut_points and cut_warnings are null (unmeasured, not silent)"),
|
|
505
506
|
"filler_removal_caution": ("warnings", "silence.py --filler: a removed word that is often an ordinary word, or a discourse marker added with --filler-extra (also in filler.warnings)"),
|
|
@@ -652,6 +653,9 @@ def output_schema(name: str, meta: Dict[str, Any], hw: bool = False) -> Dict[str
|
|
|
652
653
|
"beat_grid": {"type": "object", "description": "--beats (1.17): {supported_beats, tempo_bpm, interval, confidence, phase, onsets, supported, unsupported, method, step_s (the envelope's real step: 220/22050 s for --beat-step 0.01, after 3.2.0), range_bpm, usable, onset_refinement (after 3.2.0: pcm-window-ratio when each onset was placed on the decoded samples, else null)}"}}
|
|
653
654
|
elif name == "silence":
|
|
654
655
|
extra = {"silences": {"type": "array"}, "keep": {"type": "array"}, "input_duration": {"type": "number"}, "kept_duration": {"type": "number"}, "removed_seconds": {"type": "number"},
|
|
656
|
+
"margins": {"type": "object", "description": "{lead, tail}: the seconds of each silence kept before speech starts (--margin-lead) and after it ends (--margin-tail); both are --margin unless one is named; absent under --filler with neither --speech-aware nor a sentence flag, which removes no silences"},
|
|
657
|
+
"silence_positions": {"type": "array", "description": "--sentence-min-silence / --sentence-margin-tail: per entry of silences, sentence_end when the transcript word before it ends in sentence-final punctuation, else inside"},
|
|
658
|
+
"sentence": {"type": "object", "description": "--sentence-min-silence / --sentence-margin-tail: {min_silence, margin_tail (the floor and tail given to a silence after a sentence end), source, word_timings, sentence_end_words (transcript words that end a sentence; 0 means none did, so every silence used --min-silence and --margin-tail), silences (how many of silences are sentence_end)}"},
|
|
655
659
|
"removed_seconds_total": {"type": "number", "description": "--filler: seconds removed in all, silence plus filler words (removed_seconds stays the silence alone)"},
|
|
656
660
|
"cut_points": {"type": ["array", "null"], "description": "2.22: per cut {source_time, output_time, edge (out|in), rms_dbfs (the loudest channel 50 ms either side on the source; null for digital silence), filler (true on a removed filler word's edge)}; null when the source could not be decoded for it (cut_levels_unmeasured warning)"},
|
|
657
661
|
"cut_warnings": {"type": ["array", "null"], "description": "the cut_points (not filler cuts) within 10 dB of --threshold: where to listen (a cut_near_sound warning); verified is unchanged"},
|
package/scripts/render.py
CHANGED
|
@@ -134,7 +134,7 @@ OBJECT_KEYS: Dict[str, frozenset] = {
|
|
|
134
134
|
"snap": frozenset({"to", "tolerance", "min_confidence", "source"}),
|
|
135
135
|
"frame": frozenset({"aspect", "width", "height", "fps", "fit"}),
|
|
136
136
|
"transition": frozenset({"type", "duration"}),
|
|
137
|
-
"silence": frozenset({"threshold", "min_silence", "margin"}),
|
|
137
|
+
"silence": frozenset({"threshold", "min_silence", "margin", "margin_lead", "margin_tail"}),
|
|
138
138
|
# 1.16: the picture an audio-only source gets before the rest of the chain can work on it
|
|
139
139
|
"audiogram": frozenset({"image", "image_fit", "style", "position", "vis_height", "opacity",
|
|
140
140
|
"platform", "title", "color", "background", "width", "height", "fps"}),
|
|
@@ -1428,7 +1428,8 @@ def main() -> int:
|
|
|
1428
1428
|
if sil:
|
|
1429
1429
|
nxt = str(work / f"tight{mid}")
|
|
1430
1430
|
argv = [current, "-o", nxt]
|
|
1431
|
-
for k, flag in (("threshold", "--threshold"), ("min_silence", "--min-silence"), ("margin", "--margin")
|
|
1431
|
+
for k, flag in (("threshold", "--threshold"), ("min_silence", "--min-silence"), ("margin", "--margin"),
|
|
1432
|
+
("margin_lead", "--margin-lead"), ("margin_tail", "--margin-tail")):
|
|
1432
1433
|
if sil.get(k) is not None:
|
|
1433
1434
|
argv += [flag, str(sil[k])]
|
|
1434
1435
|
sh("silence.py", *argv, stage="silence")
|
package/scripts/silence.py
CHANGED
|
@@ -9,6 +9,9 @@ cut.py --segments --accurate cuts them (both streams on one set of frame-grid bo
|
|
|
9
9
|
Examples:
|
|
10
10
|
python3 silence.py talk.mp4 # -35 dB, gaps >= 0.6 s, 0.2 s margin
|
|
11
11
|
python3 silence.py talk.mp4 --threshold -40 --min-silence 1 --margin 0.25
|
|
12
|
+
python3 silence.py talk.mp4 --margin-lead 0.1 --margin-tail 0.4 # less before a word, more after one
|
|
13
|
+
python3 silence.py talk.mp4 --words talk.json --min-silence 0.2 --sentence-min-silence 0.3 --sentence-margin-tail 0.3
|
|
14
|
+
# pauses inside a sentence from 0.2 s, after one from 0.3 s with a longer tail
|
|
12
15
|
python3 silence.py talk.mp4 --list # print the silences and the resulting cut list, no output
|
|
13
16
|
python3 silence.py talk.mp4 --edl keep.txt # also save the kept ranges (START-END per line, cut.py --segments format)
|
|
14
17
|
"""
|
|
@@ -58,6 +61,76 @@ CUT_LEVEL_RATE = 8000
|
|
|
58
61
|
CUT_LEVEL_CHANNELS = 2 # measured one by one; a surround source is folded to two first
|
|
59
62
|
DIGITAL_SILENCE_DB = -90.0 # 2.20: at or below this a span is digital silence (16-bit dither sits near -96)
|
|
60
63
|
BREATH_FLOOR = 0.12 # shortest gap silencedetect is asked for under --speech-aware
|
|
64
|
+
# a transcript word that ends in one of these (after any closing quote or bracket) ends a sentence
|
|
65
|
+
SENTENCE_END = ".!?\u3002\uff01\uff1f\u2026\u0964\u061f"
|
|
66
|
+
SENTENCE_CLOSERS = "\"'\u201d\u2019\u300d\u300f\uff09)]\u3011 "
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def ends_sentence(word: str) -> bool:
|
|
70
|
+
"""Whether a transcript word carries sentence-final punctuation: read off the text the speech
|
|
71
|
+
engine wrote, never guessed from the sound."""
|
|
72
|
+
text = str(word or "").rstrip(SENTENCE_CLOSERS)
|
|
73
|
+
return bool(text) and text[-1] in SENTENCE_END
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def silence_positions(silences: List[Tuple[float, float]], words: "list",
|
|
77
|
+
duration: Optional[float] = None) -> List[str]:
|
|
78
|
+
"""Per silence, `sentence_end` when the word spoken before it ends a sentence, else `inside`.
|
|
79
|
+
The word before a silence is the last one whose own midpoint lies before the silence's
|
|
80
|
+
midpoint; a silence that runs to the end of the file ends at `duration`. An engine
|
|
81
|
+
stretches a word's timing into the pause beside it, the end of the word before or the start
|
|
82
|
+
of the word after (on demos/footage/ai_talk_en.mp4 three word starts sit 0.45 to 0.5 s before
|
|
83
|
+
the sound does), and a word's midpoint stays on its own side of the pause either way. A
|
|
84
|
+
silence with no word before it is `inside`."""
|
|
85
|
+
timed = sorted(words, key=lambda w: w["start"] + w["end"])
|
|
86
|
+
out: List[str] = []
|
|
87
|
+
for s, e in silences:
|
|
88
|
+
if e == float("inf"):
|
|
89
|
+
e = max(s, duration) if duration is not None else s
|
|
90
|
+
ref = (s + e) / 2
|
|
91
|
+
before = [w for w in timed if (w["start"] + w["end"]) / 2 < ref]
|
|
92
|
+
out.append("sentence_end" if before and ends_sentence(before[-1]["word"]) else "inside")
|
|
93
|
+
return out
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def sentence_words(args) -> "Tuple[list, str]":
|
|
97
|
+
"""(word timings, where they came from) for --sentence-min-silence / --sentence-margin-tail.
|
|
98
|
+
Under --filler the transcript that run read or made is reused; otherwise --words is read, or
|
|
99
|
+
--transcribe runs the local engine. Refuses without one: where a sentence ends is taken from
|
|
100
|
+
the transcript's punctuation, and there is no reading it off a pause."""
|
|
101
|
+
def usable(words: "list", source: str) -> "Tuple[list, str]":
|
|
102
|
+
# a word with no length or a time that is not a number cannot be placed against a pause
|
|
103
|
+
timed = [w for w in words if math.isfinite(w["start"]) and math.isfinite(w["end"]) and w["end"] > w["start"]]
|
|
104
|
+
if not timed:
|
|
105
|
+
die(f"no usable word timings in the transcript ({source}): every word's start and end must be "
|
|
106
|
+
"numbers with the end after the start, or there is nothing to place a sentence end against a pause",
|
|
107
|
+
kind="input")
|
|
108
|
+
return timed, source
|
|
109
|
+
|
|
110
|
+
if getattr(args, "_words", None) is not None:
|
|
111
|
+
return usable(args._words, args._words_source)
|
|
112
|
+
if args.words:
|
|
113
|
+
try:
|
|
114
|
+
raw = json.loads(read_text_or_die(args.words, "--words"))
|
|
115
|
+
except ValueError as exc:
|
|
116
|
+
die(f"--words {args.words}: not readable JSON ({exc})", kind="input")
|
|
117
|
+
words, _had_segments, _untimed = _words_from_transcript(raw)
|
|
118
|
+
if not words:
|
|
119
|
+
die(f"no word timings in {args.words}: the sentence flags need "
|
|
120
|
+
'{"words": [{"word": ..., "start": ..., "end": ...}]} (or the same inside "segments").',
|
|
121
|
+
kind="input")
|
|
122
|
+
return usable(words, f"whisper-json:{args.words}")
|
|
123
|
+
if args.transcribe:
|
|
124
|
+
asr_run = _asr.transcribe_words_result(args.input, None, model=args.model, engine=args.engine)
|
|
125
|
+
args._asr_notes = list(asr_run.notes)
|
|
126
|
+
if not asr_run.words:
|
|
127
|
+
die(f"{asr_run.engine or 'the local engine'} ran but produced no word-level timings, so there is "
|
|
128
|
+
"nothing to tell a sentence end from. Pass a transcript with word timings as --words.", kind="input")
|
|
129
|
+
family = "parakeet" if asr_run.engine in _asr.PARAKEET_ENGINES else "whisper"
|
|
130
|
+
return usable(asr_run.words, f"{family}:{asr_run.engine}" if asr_run.engine else "whisper")
|
|
131
|
+
die("--sentence-min-silence and --sentence-margin-tail need word timings: pass --words transcript.json "
|
|
132
|
+
"(a whisper JSON with word timestamps) or --transcribe. A sentence end is read from the transcript's "
|
|
133
|
+
"punctuation; a pause's length does not say where one is.", kind="input")
|
|
61
134
|
|
|
62
135
|
|
|
63
136
|
def speech_aware_silences(path: str, threshold: float, min_silence: float) -> "Tuple[List[Tuple[float, float]], List[Tuple[float, float]]]":
|
|
@@ -83,9 +156,12 @@ def speech_aware_silences(path: str, threshold: float, min_silence: float) -> "T
|
|
|
83
156
|
return boundaries, breaths
|
|
84
157
|
|
|
85
158
|
|
|
86
|
-
def inside_margin(silences: List[Tuple[float, float]], duration: float, margin: float
|
|
87
|
-
|
|
88
|
-
|
|
159
|
+
def inside_margin(silences: List[Tuple[float, float]], duration: float, margin: float,
|
|
160
|
+
lead: Optional[float] = None) -> List[Tuple[float, float]]:
|
|
161
|
+
"""Each silence with a margin kept on both sides: the stretch actually removed. `margin` is
|
|
162
|
+
kept after the speech that ends where the silence starts (--margin-tail), `lead` before the
|
|
163
|
+
speech that starts where it ends (--margin-lead; the same as `margin` when not given). A
|
|
164
|
+
silence too short to hold both removes nothing (it is dropped, never turned inside out).
|
|
89
165
|
|
|
90
166
|
The margin is applied to the silences alone, before they are merged with filler spans (#414):
|
|
91
167
|
a filler word's span is the word itself plus --filler-pad, and taking a 0.2 s margin off each
|
|
@@ -94,7 +170,7 @@ def inside_margin(silences: List[Tuple[float, float]], duration: float, margin:
|
|
|
94
170
|
out: List[Tuple[float, float]] = []
|
|
95
171
|
for s, e in silences:
|
|
96
172
|
a = s + margin
|
|
97
|
-
b = duration if e == float("inf") else min(duration, e - margin)
|
|
173
|
+
b = duration if e == float("inf") else min(duration, e - (margin if lead is None else lead))
|
|
98
174
|
if b > a:
|
|
99
175
|
out.append((a, b))
|
|
100
176
|
return out
|
|
@@ -308,6 +384,7 @@ def resolve_filler(args, meta):
|
|
|
308
384
|
"faster-whisper word_timestamps=True / openai-whisper --word_timestamps True) and "
|
|
309
385
|
"pass the result with --words.", kind="input")
|
|
310
386
|
|
|
387
|
+
args._words, args._words_source = words, source # the sentence flags read the same transcript
|
|
311
388
|
lang = args.filler_lang
|
|
312
389
|
reported_lang = lang
|
|
313
390
|
if lang == "auto":
|
|
@@ -455,6 +532,20 @@ def main() -> int:
|
|
|
455
532
|
ap.add_argument("--threshold", type=float, default=-35.0, help="silence level in dBFS (default -35; use -40..-45 for quiet rooms)")
|
|
456
533
|
ap.add_argument("--min-silence", type=float, default=0.6, help="only remove gaps at least this long in seconds (default 0.6)")
|
|
457
534
|
ap.add_argument("--margin", type=float, default=0.2, help="seconds of silence to keep on each side of speech (default 0.2; 0.15 before 2.22)")
|
|
535
|
+
ap.add_argument("--margin-lead", type=float, default=None, metavar="SECONDS",
|
|
536
|
+
help="seconds of silence to keep before speech starts, when it should differ from --margin "
|
|
537
|
+
"(default: --margin)")
|
|
538
|
+
ap.add_argument("--margin-tail", type=float, default=None, metavar="SECONDS",
|
|
539
|
+
help="seconds of silence to keep after speech ends, when it should differ from --margin "
|
|
540
|
+
"(default: --margin); a speaker whose last syllable trails off needs more here than "
|
|
541
|
+
"before the next word")
|
|
542
|
+
ap.add_argument("--sentence-min-silence", type=float, default=None, metavar="SECONDS",
|
|
543
|
+
help="for a silence after a word that ends a sentence, remove it only when it is at least this "
|
|
544
|
+
"long, instead of --min-silence. Needs --words or --transcribe: a sentence end is the "
|
|
545
|
+
"transcript's punctuation")
|
|
546
|
+
ap.add_argument("--sentence-margin-tail", type=float, default=None, metavar="SECONDS",
|
|
547
|
+
help="for a silence after a word that ends a sentence, keep this much after the speech instead "
|
|
548
|
+
"of --margin-tail. Needs --words or --transcribe")
|
|
458
549
|
ap.add_argument("--min-keep", type=float, default=0.2, help="drop kept pieces shorter than this (default 0.2)")
|
|
459
550
|
ap.add_argument("--speech-aware", action="store_true",
|
|
460
551
|
help="keep breaths shorter than --min-silence when they sit inside a sentence "
|
|
@@ -485,7 +576,7 @@ def main() -> int:
|
|
|
485
576
|
fil.add_argument("--filler-pad", type=float, default=FILLER_PAD,
|
|
486
577
|
help=f"seconds trimmed either side of a filler word (default {FILLER_PAD})")
|
|
487
578
|
fil.add_argument("--words", metavar="FILE",
|
|
488
|
-
help="a whisper JSON with word-level timings, for --filler")
|
|
579
|
+
help="a whisper JSON with word-level timings, for --filler and the sentence flags")
|
|
489
580
|
fil.add_argument("--transcribe", action="store_true",
|
|
490
581
|
help="produce the word timings with a local speech engine, Parakeet or whisper (never required; the same "
|
|
491
582
|
"bridge caption.py uses)")
|
|
@@ -525,6 +616,26 @@ def main() -> int:
|
|
|
525
616
|
if not math.isfinite(args.margin) or args.margin < 0:
|
|
526
617
|
# nan made inside_margin() drop every silence and re-encode the whole file unchanged
|
|
527
618
|
die(f"--margin must be a finite number of seconds, 0 or more (got {args.margin:g})", kind="input")
|
|
619
|
+
for flag, value in (("--margin-lead", args.margin_lead), ("--margin-tail", args.margin_tail)):
|
|
620
|
+
if value is not None and (not math.isfinite(value) or value < 0):
|
|
621
|
+
die(f"{flag} must be a finite number of seconds, 0 or more (got {value:g})", kind="input")
|
|
622
|
+
# what is kept after speech ends and before it starts again; --margin is the value of both
|
|
623
|
+
# unless one is named
|
|
624
|
+
tail = args.margin if args.margin_tail is None else args.margin_tail
|
|
625
|
+
lead = args.margin if args.margin_lead is None else args.margin_lead
|
|
626
|
+
by_sentence = args.sentence_min_silence is not None or args.sentence_margin_tail is not None
|
|
627
|
+
for flag, value in (("--sentence-min-silence", args.sentence_min_silence),
|
|
628
|
+
("--sentence-margin-tail", args.sentence_margin_tail)):
|
|
629
|
+
if value is not None and (not math.isfinite(value) or value < 0):
|
|
630
|
+
die(f"{flag} must be a finite number of seconds, 0 or more (got {value:g})", kind="input")
|
|
631
|
+
if by_sentence and args.speech_aware:
|
|
632
|
+
die("--speech-aware and the sentence flags answer the same question two ways: --speech-aware takes a "
|
|
633
|
+
"pause of --min-silence or longer as a sentence boundary, --sentence-min-silence and "
|
|
634
|
+
"--sentence-margin-tail take the transcript's punctuation. Pass one of them.", kind="input")
|
|
635
|
+
if by_sentence and not (args.words or args.transcribe):
|
|
636
|
+
sentence_words(args) # refuses: no transcript to read a sentence end from
|
|
637
|
+
sentence_min = args.min_silence if args.sentence_min_silence is None else args.sentence_min_silence
|
|
638
|
+
sentence_tail = tail if args.sentence_margin_tail is None else args.sentence_margin_tail
|
|
528
639
|
if args.filler_pad < 0:
|
|
529
640
|
die("--filler-pad cannot be negative: a negative pad turns each word's span inside out "
|
|
530
641
|
"(end before start) and the span is then silently dropped, so nothing is removed",
|
|
@@ -540,16 +651,44 @@ def main() -> int:
|
|
|
540
651
|
breaths: List[Tuple[float, float]] = []
|
|
541
652
|
if args.speech_aware:
|
|
542
653
|
silences, breaths = speech_aware_silences(args.input, args.threshold, args.min_silence)
|
|
543
|
-
elif args.filler:
|
|
654
|
+
elif args.filler and not by_sentence:
|
|
544
655
|
silences = [] # --filler alone: remove only the filler-word spans, not generic dead air
|
|
545
656
|
else:
|
|
546
|
-
|
|
657
|
+
# with the sentence flags each kind of pause has its own floor: measure at the shorter one
|
|
658
|
+
silences = detect(args.input, args.threshold, min(args.min_silence, sentence_min) if by_sentence else args.min_silence)
|
|
547
659
|
filler_info, filler_ranges = resolve_filler(args, meta) if args.filler else (None, [])
|
|
660
|
+
positions: List[str] = []
|
|
661
|
+
sentence_info: Optional[Dict[str, Any]] = None
|
|
662
|
+
if by_sentence:
|
|
663
|
+
words, words_source = sentence_words(args)
|
|
664
|
+
found = silence_positions(silences, words, duration)
|
|
665
|
+
# a silence that runs to the end of the file is as long as what is left of the file
|
|
666
|
+
kept_pairs = [(span, pos) for span, pos in zip(silences, found)
|
|
667
|
+
if (duration if span[1] == float("inf") else span[1]) - span[0]
|
|
668
|
+
>= (sentence_min if pos == "sentence_end" else args.min_silence)]
|
|
669
|
+
silences = [span for span, _ in kept_pairs]
|
|
670
|
+
positions = [pos for _, pos in kept_pairs]
|
|
671
|
+
marked = sum(1 for w in words if ends_sentence(w["word"]))
|
|
672
|
+
sentence_info = {"min_silence": sentence_min, "margin_tail": sentence_tail, "source": words_source,
|
|
673
|
+
"word_timings": len(words), "sentence_end_words": marked,
|
|
674
|
+
"silences": positions.count("sentence_end")}
|
|
675
|
+
if not marked:
|
|
676
|
+
warn("the transcript has no sentence-final punctuation, so no silence was taken as a sentence end: "
|
|
677
|
+
"every silence used --min-silence and --margin-tail", "sentence_ends_not_found")
|
|
678
|
+
|
|
679
|
+
def margined(spans: List[Tuple[float, float]]) -> List[Tuple[float, float]]:
|
|
680
|
+
"""The stretch of each silence that is removed: its own tail for one after a sentence end."""
|
|
681
|
+
if not by_sentence:
|
|
682
|
+
return inside_margin(spans, duration, tail, lead)
|
|
683
|
+
out: List[Tuple[float, float]] = []
|
|
684
|
+
for span, pos in zip(spans, positions):
|
|
685
|
+
out += inside_margin([span], duration, sentence_tail if pos == "sentence_end" else tail, lead)
|
|
686
|
+
return out
|
|
548
687
|
# One sorted, merged removal list through the graph the tool already has: filler removal IS
|
|
549
688
|
# time-range removal, so it reuses keep_ranges() and the same select chain.
|
|
550
689
|
# --margin is kept around the silences only (inside_margin), not around filler spans, whose
|
|
551
690
|
# own trim is --filler-pad (#414); the merged list then goes through with no further margin.
|
|
552
|
-
removals = merge_spans(
|
|
691
|
+
removals = merge_spans(margined(silences) + list(filler_ranges))
|
|
553
692
|
keeps = keep_ranges(removals, duration, 0.0, args.min_keep)
|
|
554
693
|
# what the render holds (on a picture, each bound on its nearest frame): the report, the EDL
|
|
555
694
|
# and the time map all describe these, so a re-cut or a sum of `keep` is the rendered edit
|
|
@@ -564,7 +703,7 @@ def main() -> int:
|
|
|
564
703
|
# the silences alone; everything removed is `removed_seconds_total`.
|
|
565
704
|
silence_only = removed
|
|
566
705
|
if args.filler:
|
|
567
|
-
silence_keeps = keep_ranges(merge_spans(
|
|
706
|
+
silence_keeps = keep_ranges(merge_spans(margined(silences)), duration, 0.0, args.min_keep)
|
|
568
707
|
silence_only = max(0.0, duration - sum(e - s for s, e in rendered_ranges(silence_keeps, meta, duration, video_out)))
|
|
569
708
|
kinds = silence_kinds(args.input, silences, duration, args.threshold)
|
|
570
709
|
summary = {
|
|
@@ -574,6 +713,15 @@ def main() -> int:
|
|
|
574
713
|
"kept_duration": round(kept, 3),
|
|
575
714
|
"removed_seconds": round(silence_only, 3),
|
|
576
715
|
}
|
|
716
|
+
if silences or not args.filler or args.speech_aware or by_sentence:
|
|
717
|
+
# the seconds of each silence this run kept: `tail` from where the silence starts (after
|
|
718
|
+
# the speech before it), `lead` up to where it ends (before the speech after it). Absent
|
|
719
|
+
# under --filler with neither --speech-aware nor a sentence flag: no silence is detected.
|
|
720
|
+
summary["margins"] = {"lead": lead, "tail": tail}
|
|
721
|
+
if sentence_info is not None:
|
|
722
|
+
# which of `silences` follow a sentence end, and the floor and tail those were given
|
|
723
|
+
summary["silence_positions"] = positions
|
|
724
|
+
summary["sentence"] = sentence_info
|
|
577
725
|
if kinds:
|
|
578
726
|
summary.update(kinds)
|
|
579
727
|
if args.speech_aware:
|
|
@@ -605,7 +753,7 @@ def main() -> int:
|
|
|
605
753
|
die(f"{len(keeps)} keep ranges is above --max-cuts {args.max_cuts}: the filter graph grows "
|
|
606
754
|
"with every range and a graph this size is slow and fragile. Raise --max-cuts if you "
|
|
607
755
|
"mean it, or use --min-silence/--filler-pad to merge the short ones.", kind="input")
|
|
608
|
-
filler_only = args.filler and not args.speech_aware
|
|
756
|
+
filler_only = args.filler and not args.speech_aware and not by_sentence
|
|
609
757
|
if filler_only:
|
|
610
758
|
info(f"--filler only (no --speech-aware): skipping generic silence detection, "
|
|
611
759
|
f"keeping {len(keeps)} range(s): {kept:.2f}s of {duration:.2f}s (removing {removed:.2f}s, filler only)")
|
|
@@ -641,7 +789,8 @@ def main() -> int:
|
|
|
641
789
|
if near:
|
|
642
790
|
where = ", ".join("%.2fs" % c["source_time"] for c in near[:5]) + (", ..." if len(near) > 5 else "")
|
|
643
791
|
warn(f"{len(near)} cut point(s) sit within {CUT_SOUND_MARGIN_DB:g} dB of --threshold ({where}): "
|
|
644
|
-
"a breath or a word's weak tail may be cut; listen there, or lower --threshold / raise --margin"
|
|
792
|
+
"a breath or a word's weak tail may be cut; listen there, or lower --threshold / raise --margin "
|
|
793
|
+
"(--margin-tail for an `out` edge, --margin-lead for an `in` edge)",
|
|
645
794
|
"cut_near_sound", cuts=len(near))
|
|
646
795
|
|
|
647
796
|
def finish_time_map() -> None:
|