ffmpeg-skill 1.17.3 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/contract.md +22 -10
- package/package.json +1 -1
- package/references/ci-platform-pitfalls.md +1 -1
- package/references/scripts.md +85 -10
- package/scripts/_common/__init__.py +8 -5
- package/scripts/_common/decision.py +86 -0
- package/scripts/_common/drawtext.py +125 -0
- package/scripts/_common/emoji.py +350 -0
- package/scripts/_common/fonts.py +437 -0
- package/scripts/_common/probe.py +26 -0
- package/scripts/_common/text.py +59 -1642
- package/scripts/_common/wrap.py +778 -0
- package/scripts/_contract.py +8 -3
- package/scripts/cropdetect.py +59 -1
- package/scripts/multicam.py +85 -5
- package/scripts/scenes.py +87 -2
- package/scripts/silence.py +46 -1
- package/scripts/sync.py +100 -52
package/scripts/_contract.py
CHANGED
|
@@ -411,11 +411,16 @@ def output_schema(name: str, meta: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
411
411
|
extra = {"audiogram": {"type": "object", "description": "{style, background ('image' or 'color'), image, position, vis_height, platform, captions, title, stages, verified} -- present on every run, so a plain waveform answers background 'color' (1.16)"},
|
|
412
412
|
"notes": {"type": "array", "items": {"type": "string"}}}
|
|
413
413
|
elif name == "scenes":
|
|
414
|
-
extra = {"file": {"type": "string"}, "duration": {"type": "number"}, "scene_count": {"type": "integer"}, "scenes": {"type": "array"}, "audio_peaks": {"type": "array"}
|
|
414
|
+
extra = {"file": {"type": "string"}, "duration": {"type": "number"}, "scene_count": {"type": "integer"}, "scenes": {"type": "array"}, "audio_peaks": {"type": "array"},
|
|
415
|
+
"shots": {"type": "array", "description": "--shots (1.18): [{start, end, label, flow_magnitude}]"},
|
|
416
|
+
"audio_peaks_db": {"type": "array", "description": "--audio-peaks (1.18): [{time, level}] measured dBFS, distinct from audio_peaks above"},
|
|
417
|
+
"speech": {"type": "array", "description": "--speech (1.18): [{time, speech_music_ratio}]"}}
|
|
415
418
|
elif name == "silence":
|
|
416
|
-
extra = {"silences": {"type": "array"}, "keep": {"type": "array"}, "input_duration": {"type": "number"}, "kept_duration": {"type": "number"}, "removed_seconds": {"type": "number"}
|
|
419
|
+
extra = {"silences": {"type": "array"}, "keep": {"type": "array"}, "input_duration": {"type": "number"}, "kept_duration": {"type": "number"}, "removed_seconds": {"type": "number"},
|
|
420
|
+
"speech_aware": {"type": "object", "description": "--speech-aware (1.18): {min_silence, floor, breaths_kept, breaths_kept_seconds, breaths}"}}
|
|
417
421
|
elif name == "sync":
|
|
418
|
-
extra = {"reference": {"type": "string"}, "second": {"type": "string"}, "offset_seconds": {"type": "number"}, "confidence": {"type": "number"}, "meaning": {"type": "string"}, "drift": {"type": "object"}
|
|
422
|
+
extra = {"reference": {"type": "string"}, "second": {"type": "string"}, "offset_seconds": {"type": "number"}, "confidence": {"type": "number"}, "meaning": {"type": "string"}, "drift": {"type": "object"},
|
|
423
|
+
"sources": {"type": "array", "description": "1.18: [{path, offset_s, confidence, drift_ppm}], one per SOURCE; the only per-source shape once more than one SOURCE is given"}}
|
|
419
424
|
elif name == "look":
|
|
420
425
|
extra = {"outputs": {"type": "array", "items": {"type": "string"}}}
|
|
421
426
|
elif name == "render":
|
package/scripts/cropdetect.py
CHANGED
|
@@ -33,7 +33,10 @@ import sys
|
|
|
33
33
|
from collections import Counter
|
|
34
34
|
from typing import Dict, List, Tuple
|
|
35
35
|
|
|
36
|
-
from _common import add_common, apply_common, die, emit, info, print_json, probe, require_tool, run_analysis
|
|
36
|
+
from _common import add_common, apply_common, die, emit, info, print_json, probe, require_tool, run_analysis, decode_gray_frames, frame_flow
|
|
37
|
+
|
|
38
|
+
MOTION_CENTRE_FPS = 2.0
|
|
39
|
+
MOTION_CENTRE_W, MOTION_CENTRE_H = 64, 36
|
|
37
40
|
|
|
38
41
|
CROP_RE = re.compile(r"crop=(\d+):(\d+):(\d+):(\d+)")
|
|
39
42
|
|
|
@@ -61,6 +64,52 @@ def detect(path: str, seconds: float, samples: int, limit: float, round_to: int,
|
|
|
61
64
|
return rects
|
|
62
65
|
|
|
63
66
|
|
|
67
|
+
def motion_centre(path: str, seconds: float, samples: int, duration: float, sw: int, sh: int) -> List[Dict]:
|
|
68
|
+
"""Per-second motion centroid: {time, x, y, x_frac, y_frac} -- a report-only measurement,
|
|
69
|
+
the same sampled-window approach as detect() above. x/y are pixel coordinates in the SOURCE
|
|
70
|
+
frame (consistent with the --round crop rectangle this tool already reports in source
|
|
71
|
+
pixels); x_frac/y_frac are the same position as a 0..1 fraction of source_width/height, for a
|
|
72
|
+
caller that wants to reframe without first knowing the source size. This never picks a
|
|
73
|
+
subject -- it reports where in the frame the measured pixel motion was concentrated, which is
|
|
74
|
+
not the same thing as where the interesting subject is (a moving background behind a still
|
|
75
|
+
speaker centres the motion on the background)."""
|
|
76
|
+
per_window = max(0.5, seconds / max(1, samples))
|
|
77
|
+
cell = 8 # NxN diff grid at decode resolution
|
|
78
|
+
cw, ch = MOTION_CENTRE_W / cell, MOTION_CENTRE_H / cell
|
|
79
|
+
out: List[Dict] = []
|
|
80
|
+
for i in range(samples):
|
|
81
|
+
start = 0.0 if duration <= 0 else (duration - per_window) * i / max(1, samples - 1) if samples > 1 else 0.0
|
|
82
|
+
start = max(0.0, start)
|
|
83
|
+
frames = decode_gray_frames(path, MOTION_CENTRE_FPS, MOTION_CENTRE_W, MOTION_CENTRE_H,
|
|
84
|
+
start=start, seconds=per_window, check=False)
|
|
85
|
+
for k in range(len(frames) - 1):
|
|
86
|
+
prev, cur = frames[k], frames[k + 1]
|
|
87
|
+
wsum = wx = wy = 0.0
|
|
88
|
+
for gy in range(cell):
|
|
89
|
+
for gx in range(cell):
|
|
90
|
+
x0, x1 = int(gx * cw), int((gx + 1) * cw)
|
|
91
|
+
y0, y1 = int(gy * ch), int((gy + 1) * ch)
|
|
92
|
+
diff = 0
|
|
93
|
+
for y in range(y0, y1):
|
|
94
|
+
row = y * MOTION_CENTRE_W
|
|
95
|
+
for x in range(x0, x1):
|
|
96
|
+
diff += abs(prev[row + x] - cur[row + x])
|
|
97
|
+
cx = (gx + 0.5) / cell
|
|
98
|
+
cy = (gy + 0.5) / cell
|
|
99
|
+
wsum += diff
|
|
100
|
+
wx += diff * cx
|
|
101
|
+
wy += diff * cy
|
|
102
|
+
t = start + (k + 1) / MOTION_CENTRE_FPS
|
|
103
|
+
if wsum <= 0:
|
|
104
|
+
out.append({"time": round(t, 2), "x": None, "y": None, "x_frac": None, "y_frac": None, "motion": 0.0})
|
|
105
|
+
continue
|
|
106
|
+
xf, yf = wx / wsum, wy / wsum
|
|
107
|
+
out.append({"time": round(t, 2), "x": round(xf * sw), "y": round(yf * sh),
|
|
108
|
+
"x_frac": round(xf, 3), "y_frac": round(yf, 3), "motion": round(wsum / (MOTION_CENTRE_W * MOTION_CENTRE_H), 2)})
|
|
109
|
+
out.sort(key=lambda r: r["time"])
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
|
|
64
113
|
def main() -> int:
|
|
65
114
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
66
115
|
ap.add_argument("input")
|
|
@@ -68,6 +117,10 @@ def main() -> int:
|
|
|
68
117
|
ap.add_argument("--samples", type=int, default=5, help="number of windows spread across the file (default 5)")
|
|
69
118
|
ap.add_argument("--limit", type=float, default=0.0941176, help="black-pixel threshold, 0..1 (default ~0.094, cropdetect's own default)")
|
|
70
119
|
ap.add_argument("--round", type=int, default=16, dest="round_to", help="the reported width/height are rounded to a multiple of this (default 16)")
|
|
120
|
+
ap.add_argument("--motion-centre", action="store_true",
|
|
121
|
+
help="report the motion centroid per second, sampled the same way as the crop "
|
|
122
|
+
"detection above (report only -- this tool never picks a reframe, it hands "
|
|
123
|
+
"the calling agent numbers to reframe with)")
|
|
71
124
|
add_common(ap)
|
|
72
125
|
args = ap.parse_args()
|
|
73
126
|
apply_common(args)
|
|
@@ -102,6 +155,11 @@ def main() -> int:
|
|
|
102
155
|
info(f"detected crop={w}:{h}:{x}:{y} (source {sw}x{sh}, confidence {result['confidence']:.0%}) -- "
|
|
103
156
|
f"crop.py {args.input} --x {x} --y {y} --width {w} --height {h}")
|
|
104
157
|
|
|
158
|
+
if args.motion_centre:
|
|
159
|
+
centre = motion_centre(args.input, args.seconds, args.samples, duration, sw, sh)
|
|
160
|
+
result["motion_centre"] = centre
|
|
161
|
+
info(f"--motion-centre: {len(centre)} measured points")
|
|
162
|
+
|
|
105
163
|
if args.json:
|
|
106
164
|
emit(None, **result)
|
|
107
165
|
else:
|
package/scripts/multicam.py
CHANGED
|
@@ -26,12 +26,16 @@ Examples:
|
|
|
26
26
|
python3 multicam.py camA.mp4 camB.mp4 --auto 8 -o edit.mp4 # alternate cameras every 8 s
|
|
27
27
|
"""
|
|
28
28
|
import argparse
|
|
29
|
+
import math
|
|
29
30
|
import sys
|
|
30
|
-
from typing import List, Tuple
|
|
31
|
+
from typing import Dict, List, Tuple
|
|
31
32
|
|
|
32
|
-
from _common import video_args, aac_args, add_common, apply_common, default_output, die, emit, ffmpeg_base, info, time_arg, probe, run, x264_args, X264_PRESETS, fmt_secs
|
|
33
|
+
from _common import video_args, aac_args, add_common, apply_common, default_output, die, emit, ffmpeg_base, info, time_arg, probe, run, x264_args, X264_PRESETS, fmt_secs, decode_pcm_mono, rms_envelope, STATE
|
|
33
34
|
from sync import measure_offset
|
|
34
35
|
|
|
36
|
+
ENERGY_WINDOW_S = 0.25 # --switch energy: loudness measured over this window on the reference timeline
|
|
37
|
+
ENERGY_RATE = 8000 # decode rate for the per-camera loudness envelope
|
|
38
|
+
|
|
35
39
|
|
|
36
40
|
def parse_switch(spec: str, n: int) -> List[Tuple[float, float, int]]:
|
|
37
41
|
out = []
|
|
@@ -55,12 +59,75 @@ def parse_switch(spec: str, n: int) -> List[Tuple[float, float, int]]:
|
|
|
55
59
|
return out
|
|
56
60
|
|
|
57
61
|
|
|
62
|
+
def energy_switch(inputs: List[str], metas: List["Dict"], offsets: List[float], ratios: List[float],
|
|
63
|
+
ref_dur: float, min_shot: float, window_s: float = ENERGY_WINDOW_S) -> List[Tuple[float, float, int]]:
|
|
64
|
+
"""Auto-switch cuts: at every `window_s` step on the reference timeline, cut to whichever
|
|
65
|
+
camera (of those WITH video) measures the loudest audio at that moment, then merge the
|
|
66
|
+
winners into runs and fold any run shorter than `min_shot` into its neighbour. This is a
|
|
67
|
+
measured loudest-camera pick, the same spirit as scenes.py --rank-by audio: it is a proxy for
|
|
68
|
+
"who is talking", not a judgement -- a loud crowd or a camera with a hot mic wins over a
|
|
69
|
+
quiet subject exactly like scenes.py's own audio ranking does."""
|
|
70
|
+
candidates = [c for c, m in enumerate(metas) if m.get("video")]
|
|
71
|
+
if not candidates:
|
|
72
|
+
die("no input has video; --switch energy needs at least one camera with a picture")
|
|
73
|
+
envs: "Dict[int, List[float]]" = {}
|
|
74
|
+
step = max(1, int(ENERGY_RATE * window_s))
|
|
75
|
+
for c in candidates:
|
|
76
|
+
samples = decode_pcm_mono(inputs[c], ENERGY_RATE, check=False)
|
|
77
|
+
envs[c] = rms_envelope(samples, step) if samples else []
|
|
78
|
+
n_windows = max(1, int(math.ceil(ref_dur / window_s)))
|
|
79
|
+
winners: List[int] = []
|
|
80
|
+
for i in range(n_windows):
|
|
81
|
+
t = i * window_s
|
|
82
|
+
best_c, best_v = candidates[0], -1.0
|
|
83
|
+
for c in candidates:
|
|
84
|
+
src_t = (t - offsets[c]) * ratios[c]
|
|
85
|
+
idx = int(src_t / window_s)
|
|
86
|
+
env = envs[c]
|
|
87
|
+
val = env[idx] if 0 <= idx < len(env) else -1.0
|
|
88
|
+
if val > best_v:
|
|
89
|
+
best_v, best_c = val, c
|
|
90
|
+
winners.append(best_c)
|
|
91
|
+
# collapse into runs
|
|
92
|
+
runs: List[List] = []
|
|
93
|
+
for i, c in enumerate(winners):
|
|
94
|
+
t0, t1 = i * window_s, min(ref_dur, (i + 1) * window_s)
|
|
95
|
+
if runs and runs[-1][2] == c:
|
|
96
|
+
runs[-1][1] = t1
|
|
97
|
+
else:
|
|
98
|
+
runs.append([t0, t1, c])
|
|
99
|
+
# fold a run shorter than min_shot into its neighbour: the next run if there is one, else the
|
|
100
|
+
# previous one -- a single pass is enough because folding only ever lengthens a run, and a
|
|
101
|
+
# run just extended is re-checked on the next iteration through the while loop.
|
|
102
|
+
changed = True
|
|
103
|
+
while changed and len(runs) > 1:
|
|
104
|
+
changed = False
|
|
105
|
+
for i, r in enumerate(runs):
|
|
106
|
+
if r[1] - r[0] < min_shot:
|
|
107
|
+
if i + 1 < len(runs):
|
|
108
|
+
runs[i + 1][0] = r[0]
|
|
109
|
+
else:
|
|
110
|
+
runs[i - 1][1] = r[1]
|
|
111
|
+
del runs[i]
|
|
112
|
+
changed = True
|
|
113
|
+
break
|
|
114
|
+
return [(s, e, c) for s, e, c in runs]
|
|
115
|
+
|
|
116
|
+
|
|
58
117
|
def main() -> int:
|
|
59
118
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
60
119
|
ap.add_argument("inputs", nargs="+", help="reference camera first, then other cameras / recorders")
|
|
61
120
|
ap.add_argument("-o", "--output", help="output file (default: <reference>_multicam.mp4)")
|
|
62
|
-
ap.add_argument("--switch", help="switch list START-END:CAM,... on the reference timeline"
|
|
121
|
+
ap.add_argument("--switch", help="switch list START-END:CAM,... on the reference timeline, or "
|
|
122
|
+
"the literal 'energy' to auto-switch to whichever camera is "
|
|
123
|
+
"loudest at each moment (respecting --min-shot)")
|
|
63
124
|
ap.add_argument("--auto", type=float, help="no switch list: alternate through the cameras every N seconds")
|
|
125
|
+
ap.add_argument("--min-shot", type=float, default=1.5,
|
|
126
|
+
help="--switch energy: never cut to a new camera for less than this many "
|
|
127
|
+
"seconds (default 1.5)")
|
|
128
|
+
ap.add_argument("--edl", help="write the resulting cut list to this file, one START-END per line "
|
|
129
|
+
"(cut.py --segments format); the camera index for each cut is in "
|
|
130
|
+
"the JSON `cuts` field alongside it")
|
|
64
131
|
ap.add_argument("--audio", type=int, default=0, help="input index to take audio from (default 0 = reference)")
|
|
65
132
|
ap.add_argument("--offsets-only", action="store_true", help="print the measured offsets and exit")
|
|
66
133
|
ap.add_argument("--max-offset", type=float, default=30.0)
|
|
@@ -137,8 +204,13 @@ def main() -> int:
|
|
|
137
204
|
print(f"{p}: {o:+.3f}s (confidence {c:.2f})")
|
|
138
205
|
return 0
|
|
139
206
|
|
|
207
|
+
if args.min_shot <= 0:
|
|
208
|
+
die(f"--min-shot must be positive, got {args.min_shot:g}")
|
|
140
209
|
ref_dur = metas[0]["duration"] or 0.0
|
|
141
|
-
|
|
210
|
+
energy_mode = args.switch == "energy"
|
|
211
|
+
if energy_mode:
|
|
212
|
+
cuts = energy_switch(args.inputs, metas, offsets, ratios, ref_dur, args.min_shot)
|
|
213
|
+
elif args.switch:
|
|
142
214
|
cuts = parse_switch(args.switch, n)
|
|
143
215
|
elif args.auto:
|
|
144
216
|
if args.auto <= 0:
|
|
@@ -167,6 +239,13 @@ def main() -> int:
|
|
|
167
239
|
if not metas[c].get("video"):
|
|
168
240
|
die(f"camera {c} ({args.inputs[c]}) has no video; it can only be used with --audio")
|
|
169
241
|
|
|
242
|
+
if args.edl:
|
|
243
|
+
if not STATE.dry_run:
|
|
244
|
+
with open(args.edl, "w", encoding="utf-8") as fh:
|
|
245
|
+
for s, e, _c in filled:
|
|
246
|
+
fh.write(f"{s:.3f}-{e:.3f}\n")
|
|
247
|
+
info(f"wrote {args.edl}")
|
|
248
|
+
|
|
170
249
|
v0 = metas[0]["video"]
|
|
171
250
|
w, h = args.width or v0["width"], args.height or v0["height"]
|
|
172
251
|
if v0.get("rotation") in (90, -90, 270, -270) and not (args.width or args.height):
|
|
@@ -212,7 +291,8 @@ def main() -> int:
|
|
|
212
291
|
run(cmd)
|
|
213
292
|
r = probe(output, role="output")
|
|
214
293
|
info(f"wrote {output} ({fmt_secs(r['duration'])}, {len(filled)} cuts, audio from input {a})")
|
|
215
|
-
|
|
294
|
+
extra = {"switch_mode": "energy", "min_shot": args.min_shot} if energy_mode else {}
|
|
295
|
+
emit(output, cuts=[[round(s, 3), round(e, 3), c] for s, e, c in filled], **report, **extra)
|
|
216
296
|
return 0
|
|
217
297
|
|
|
218
298
|
|
package/scripts/scenes.py
CHANGED
|
@@ -25,7 +25,11 @@ from typing import Dict, List, Optional, Tuple
|
|
|
25
25
|
# `detect_scenes` moved into _common/probe.py in 1.16.0 (see silence.py); the body is unchanged.
|
|
26
26
|
from _common import (detect_scenes, STATE, add_common, apply_common, beat_grid, default_font_file, die, emit,
|
|
27
27
|
escape_filter_path, ffmpeg_base, info, print_json, probe, run, decode_pcm_mono,
|
|
28
|
-
rms_envelope, BEAT_MIN_CONFIDENCE)
|
|
28
|
+
rms_envelope, BEAT_MIN_CONFIDENCE, decode_gray_frames, frame_flow, label_shot_flow)
|
|
29
|
+
|
|
30
|
+
SHOT_FPS = 4.0 # frames/second sampled for --shots' flow estimate
|
|
31
|
+
SHOT_W, SHOT_H = 48, 27 # decode size for --shots (16:9-ish; enough blocks, still a few KB/shot)
|
|
32
|
+
SPEECH_STEP_S = 1.0 # --speech energy-ratio window
|
|
29
33
|
|
|
30
34
|
|
|
31
35
|
|
|
@@ -54,6 +58,51 @@ def parse_beat_range(text: str) -> "tuple":
|
|
|
54
58
|
return (lo, hi)
|
|
55
59
|
|
|
56
60
|
|
|
61
|
+
def shot_flow_label(path: str, start: float, end: float) -> Dict:
|
|
62
|
+
"""{start, end, label, flow_magnitude} for one shot: decode it at SHOT_FPS/SHOT_W x SHOT_H
|
|
63
|
+
and block-match consecutive frames (see _common.decision.frame_flow / label_shot_flow). A
|
|
64
|
+
shot under two sampled frames has nothing to compare and is reported static with
|
|
65
|
+
flow_magnitude 0 -- there is no motion measurement to make on a single frame."""
|
|
66
|
+
frames = decode_gray_frames(path, SHOT_FPS, SHOT_W, SHOT_H, start=start, seconds=max(0.0, end - start), check=False)
|
|
67
|
+
flows = [frame_flow(frames[i], frames[i + 1], SHOT_W, SHOT_H) for i in range(len(frames) - 1)]
|
|
68
|
+
label = label_shot_flow(flows)
|
|
69
|
+
return {"start": round(start, 3), "end": round(end, 3), "label": label["label"], "flow_magnitude": label["flow_magnitude"]}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def audio_peaks_db(samples: "List[float]", rate: int, step_s: float = 0.25) -> List[Dict]:
|
|
73
|
+
"""--audio-peaks: local maxima of the loudness envelope reported as measured dBFS, not the
|
|
74
|
+
unitless RMS scenes.py has always put in the (unconditional) `audio_peaks` key -- a
|
|
75
|
+
different unit needs a different key so the existing one keeps meaning what it always has."""
|
|
76
|
+
env = rms_envelope(samples, int(rate * step_s))
|
|
77
|
+
peaks = []
|
|
78
|
+
for i, val in enumerate(env):
|
|
79
|
+
if val <= 1e-6:
|
|
80
|
+
continue
|
|
81
|
+
if (i == 0 or env[i - 1] <= val) and (i == len(env) - 1 or env[i + 1] <= val):
|
|
82
|
+
level = round(20 * math.log10(val), 1)
|
|
83
|
+
peaks.append({"time": round(i * step_s, 2), "level": level})
|
|
84
|
+
return peaks
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def speech_music_ratio(samples: "List[float]", rate: int, step_s: float = SPEECH_STEP_S) -> List[Dict]:
|
|
88
|
+
"""--speech: a per-second zero-crossing-rate ratio, reported as a measured number, not a
|
|
89
|
+
speech/music label. Speech's rapid consonant transients drive the zero-crossing rate up and
|
|
90
|
+
make it jump window to window; sustained tones (music, a held note, room tone) cross zero at
|
|
91
|
+
a steadier rate. ratio = this window's ZCR / the file's median ZCR, so 1.0 is "typical for
|
|
92
|
+
this file" regardless of its overall noisiness -- a proxy, in the same spirit as scenes.py's
|
|
93
|
+
--rank-by, not a classifier: nothing here decides what is speech."""
|
|
94
|
+
step = max(1, int(rate * step_s))
|
|
95
|
+
zcrs: List[float] = []
|
|
96
|
+
for i in range(0, len(samples) - step + 1, step):
|
|
97
|
+
block = samples[i:i + step]
|
|
98
|
+
crossings = sum(1 for a, b in zip(block, block[1:]) if (a >= 0) != (b >= 0))
|
|
99
|
+
zcrs.append(crossings / max(1, len(block) - 1))
|
|
100
|
+
if not zcrs:
|
|
101
|
+
return []
|
|
102
|
+
med = sorted(zcrs)[len(zcrs) // 2] or 1e-9
|
|
103
|
+
return [{"time": round(i * step_s, 2), "speech_music_ratio": round(z / med, 3)} for i, z in enumerate(zcrs)]
|
|
104
|
+
|
|
105
|
+
|
|
57
106
|
def main() -> int:
|
|
58
107
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
59
108
|
ap.add_argument("input")
|
|
@@ -76,6 +125,15 @@ def main() -> int:
|
|
|
76
125
|
ap.add_argument("--min-confidence", type=float, default=BEAT_MIN_CONFIDENCE,
|
|
77
126
|
help="with --beats: below this confidence the grid is still reported, marked "
|
|
78
127
|
f"usable: false (default {BEAT_MIN_CONFIDENCE})")
|
|
128
|
+
ap.add_argument("--shots", action="store_true",
|
|
129
|
+
help="label each detected shot static / pan / motion by a measured optical-flow "
|
|
130
|
+
"proxy (lightweight block matching over sampled frames); reports flow_magnitude too")
|
|
131
|
+
ap.add_argument("--audio-peaks", action="store_true",
|
|
132
|
+
help="report loudness peaks as {time, level} in measured dBFS (separate from the "
|
|
133
|
+
"always-on `audio_peaks` RMS list used to pick --highlights)")
|
|
134
|
+
ap.add_argument("--speech", action="store_true",
|
|
135
|
+
help="report a per-second speech-vs-music energy ratio (zero-crossing-rate proxy, "
|
|
136
|
+
"a measurement, not a speech/music classification)")
|
|
79
137
|
ap.add_argument("--sheet", help="write a contact sheet PNG with the first frame of every scene")
|
|
80
138
|
ap.add_argument("--no-timecode", action="store_true", help="--sheet without the burnt-in timecode stamp (a way out if drawtext itself is unusable, see doctor)")
|
|
81
139
|
add_common(ap)
|
|
@@ -100,7 +158,8 @@ def main() -> int:
|
|
|
100
158
|
# With --beats the file is decoded once, at the finer rate, and both envelopes come from that
|
|
101
159
|
# one pass: the 0.5 s scene blocks are an exact multiple of the 10 ms onset blocks.
|
|
102
160
|
beat_rate = 22050
|
|
103
|
-
|
|
161
|
+
need_fine = args.beats or args.audio_peaks or args.speech
|
|
162
|
+
fine_samples = decode_pcm_mono(args.input, beat_rate, check=False) if (need_fine and meta.get("audio")) else None
|
|
104
163
|
if fine_samples is not None:
|
|
105
164
|
env = audio_envelope(args.input, step_s, rate=beat_rate, samples=fine_samples)
|
|
106
165
|
else:
|
|
@@ -128,6 +187,32 @@ def main() -> int:
|
|
|
128
187
|
result: Dict = {"file": args.input, "duration": round(dur, 3), "scene_count": len(scenes), "scenes": scenes, "audio_peaks": peaks}
|
|
129
188
|
info(f"{len(scenes)} scenes, {len(peaks)} audio peaks over {dur:.1f}s")
|
|
130
189
|
|
|
190
|
+
if args.shots:
|
|
191
|
+
shots = [shot_flow_label(args.input, s, e) for s, e in zip(bounds[:-1], bounds[1:]) if e - s > 0.05]
|
|
192
|
+
result["shots"] = shots
|
|
193
|
+
counts = {}
|
|
194
|
+
for sh in shots:
|
|
195
|
+
counts[sh["label"]] = counts.get(sh["label"], 0) + 1
|
|
196
|
+
info(f"--shots: {len(shots)} shots (" + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())) + ")")
|
|
197
|
+
|
|
198
|
+
if args.audio_peaks:
|
|
199
|
+
if not meta.get("audio"):
|
|
200
|
+
result["audio_peaks_db"] = []
|
|
201
|
+
info("--audio-peaks: no audio stream, nothing to measure")
|
|
202
|
+
else:
|
|
203
|
+
db_samples = fine_samples if fine_samples is not None else decode_pcm_mono(args.input, 22050, check=False)
|
|
204
|
+
result["audio_peaks_db"] = audio_peaks_db(db_samples, 22050 if fine_samples is not None else 22050)
|
|
205
|
+
info(f"--audio-peaks: {len(result['audio_peaks_db'])} peaks")
|
|
206
|
+
|
|
207
|
+
if args.speech:
|
|
208
|
+
if not meta.get("audio"):
|
|
209
|
+
result["speech"] = []
|
|
210
|
+
info("--speech: no audio stream, nothing to measure")
|
|
211
|
+
else:
|
|
212
|
+
sp_samples = fine_samples if fine_samples is not None else decode_pcm_mono(args.input, 22050, check=False)
|
|
213
|
+
result["speech"] = speech_music_ratio(sp_samples, 22050)
|
|
214
|
+
info(f"--speech: {len(result['speech'])} one-second windows")
|
|
215
|
+
|
|
131
216
|
if args.beats:
|
|
132
217
|
# A beat grid is a measurement of the music's periodicity, not a statement about where a
|
|
133
218
|
# cut belongs. scenes.py reports what it measured, including a low confidence: reporting a
|
package/scripts/silence.py
CHANGED
|
@@ -42,6 +42,32 @@ def merge_spans(spans: List[Tuple[float, float]]) -> List[Tuple[float, float]]:
|
|
|
42
42
|
return out
|
|
43
43
|
|
|
44
44
|
|
|
45
|
+
BREATH_FLOOR = 0.12 # shortest gap silencedetect is asked for under --speech-aware
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def speech_aware_silences(path: str, threshold: float, min_silence: float) -> "Tuple[List[Tuple[float, float]], List[Tuple[float, float]]]":
|
|
49
|
+
"""(sentence-boundary silences to remove, breaths kept) for --speech-aware.
|
|
50
|
+
|
|
51
|
+
Re-runs the same silencedetect() already used everywhere else in this file, but with a much
|
|
52
|
+
shorter minimum duration, so short in-sentence breaths are measured at all -- the plain
|
|
53
|
+
`detect()` call above never sees them because its own --min-silence floor filters them out
|
|
54
|
+
before they reach Python. Every gap silencedetect measured that is still shorter than
|
|
55
|
+
--min-silence sits *inside* a sentence -- the two speech-flagged stretches on either side of
|
|
56
|
+
it are close together in time because nothing longer separated them from the gaps around
|
|
57
|
+
them -- and is kept rather than cut; --min-silence keeps the meaning it already has for
|
|
58
|
+
--filler and the plain run: the shortest gap this tool will remove."""
|
|
59
|
+
floor = min(BREATH_FLOOR, min_silence)
|
|
60
|
+
fine = detect(path, threshold, floor)
|
|
61
|
+
boundaries, breaths = [], []
|
|
62
|
+
for s, e in fine:
|
|
63
|
+
length = (e - s) if e != float("inf") else float("inf")
|
|
64
|
+
if length >= min_silence:
|
|
65
|
+
boundaries.append((s, e))
|
|
66
|
+
else:
|
|
67
|
+
breaths.append((s, e))
|
|
68
|
+
return boundaries, breaths
|
|
69
|
+
|
|
70
|
+
|
|
45
71
|
def keep_ranges(silences: List[Tuple[float, float]], duration: float, margin: float, min_keep: float) -> List[Tuple[float, float]]:
|
|
46
72
|
keeps: List[Tuple[float, float]] = []
|
|
47
73
|
cursor = 0.0
|
|
@@ -182,6 +208,11 @@ def main() -> int:
|
|
|
182
208
|
ap.add_argument("--min-silence", type=float, default=0.6, help="only remove gaps at least this long in seconds (default 0.6)")
|
|
183
209
|
ap.add_argument("--margin", type=float, default=0.15, help="seconds of silence to keep on each side of speech (default 0.15)")
|
|
184
210
|
ap.add_argument("--min-keep", type=float, default=0.2, help="drop kept pieces shorter than this (default 0.2)")
|
|
211
|
+
ap.add_argument("--speech-aware", action="store_true",
|
|
212
|
+
help="keep breaths shorter than --min-silence when they sit inside a sentence "
|
|
213
|
+
"(measured by re-running silence detection at a much shorter floor), and "
|
|
214
|
+
"only cut at sentence-boundary pauses (--min-silence or longer). Composes "
|
|
215
|
+
"with --filler through the same keep_ranges() removal list.")
|
|
185
216
|
ap.add_argument("--list", action="store_true", help="only print silences and the kept ranges")
|
|
186
217
|
ap.add_argument("--edl", help="write the kept ranges to this file, one START-END per line")
|
|
187
218
|
fil = ap.add_argument_group("filler words (1.17)")
|
|
@@ -229,7 +260,11 @@ def main() -> int:
|
|
|
229
260
|
if not meta.get("audio"):
|
|
230
261
|
die("input has no audio stream to analyse")
|
|
231
262
|
duration = meta.get("duration") or 0.0
|
|
232
|
-
|
|
263
|
+
breaths: List[Tuple[float, float]] = []
|
|
264
|
+
if args.speech_aware:
|
|
265
|
+
silences, breaths = speech_aware_silences(args.input, args.threshold, args.min_silence)
|
|
266
|
+
else:
|
|
267
|
+
silences = detect(args.input, args.threshold, args.min_silence)
|
|
233
268
|
filler_info, filler_ranges = resolve_filler(args, meta) if args.filler else (None, [])
|
|
234
269
|
# One sorted, merged removal list through the graph the tool already has: filler removal IS
|
|
235
270
|
# time-range removal, so it reuses keep_ranges() and the same aselect/concat chain.
|
|
@@ -253,6 +288,16 @@ def main() -> int:
|
|
|
253
288
|
"kept_duration": round(kept, 3),
|
|
254
289
|
"removed_seconds": round(silence_only, 3),
|
|
255
290
|
}
|
|
291
|
+
if args.speech_aware:
|
|
292
|
+
summary["speech_aware"] = {
|
|
293
|
+
"min_silence": args.min_silence, "floor": min(BREATH_FLOOR, args.min_silence),
|
|
294
|
+
"breaths_kept": len(breaths),
|
|
295
|
+
"breaths_kept_seconds": round(sum((e - s) for s, e in breaths if e != float("inf")), 3),
|
|
296
|
+
"breaths": [[round(s, 3), None if e == float("inf") else round(e, 3)] for s, e in breaths],
|
|
297
|
+
}
|
|
298
|
+
info(f"--speech-aware: {len(breaths)} breath(s) kept "
|
|
299
|
+
f"({summary['speech_aware']['breaths_kept_seconds']:.2f}s), "
|
|
300
|
+
f"{len(silences)} sentence-boundary silence(s) cut")
|
|
256
301
|
if filler_info is not None:
|
|
257
302
|
# removed_seconds above is the silence-only figure, unchanged in meaning; the filler share
|
|
258
303
|
# is reported inside `filler`, and removed_seconds_total is the additive sibling that
|