@dsh-gis/dsh-video-studio-skill 0.6.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -16
- package/index.js +20 -7
- package/package.json +4 -2
- package/schemas/film.schema.json +75 -4
- package/skills/video-studio/SKILL.md +95 -23
- package/skills/video-studio/assets/film.json +7 -14
- package/skills/video-studio/assets/scenes.py +115 -10
- package/skills/video-studio/assets/web/engine.js +18 -1
- package/skills/video-studio/assets/web/film.js +29 -4
- package/skills/video-studio/assets/web/index.html +5 -0
- package/skills/video-studio/references/audio-variation.md +158 -0
- package/skills/video-studio/references/audio.md +67 -5
- package/skills/video-studio/references/change-impact.md +1 -1
- package/skills/video-studio/references/quality-gates.md +1 -1
- package/skills/video-studio/references/web-engine.md +1 -1
- package/skills/video-studio/requirements.txt +2 -1
- package/skills/video-studio/scripts/filmcheck.py +81 -12
- package/skills/video-studio/scripts/filmkit.py +271 -18
- package/skills/video-studio/scripts/gates.py +75 -12
- package/skills/video-studio/scripts/qa_frames.py +109 -19
- package/skills/video-studio/scripts/render.py +420 -51
- package/skills/video-studio/scripts/scorekit.py +289 -46
- package/skills/video-studio/scripts/selfcheck.py +302 -2
- package/skills/video-studio/scripts/sfxkit.py +132 -30
- package/skills/video-studio/scripts/stagewatch.py +141 -31
- package/skills/video-studio/scripts/stylekit.py +94 -6
- package/skills/video-studio/scripts/voicekit.py +8 -0
- package/skills/video-studio/scripts/webengine.py +68 -14
- package/tools/audio_similarity.py +279 -0
- package/tools/ci_check.py +145 -34
- package/tools/estimate.py +133 -54
- package/tools/selftest_noscipy.py +70 -0
|
@@ -52,6 +52,11 @@ PITCH_RE = re.compile(r"^\s*[+-]?\d+\s*Hz\s*$")
|
|
|
52
52
|
ID_RE = re.compile(r"^[A-Za-z0-9_.\-]+$")
|
|
53
53
|
|
|
54
54
|
CPS = 4.2 # 每秒约 4.2 字(voicekit 实测中位口径)
|
|
55
|
+
# 语速区间而不是单一 4.2:voicekit 校准实测 3.84–4.61 cps(9 个 persona,固定校准句),
|
|
56
|
+
# 而 stylekit 的声线策略允许的 cps_band 是 3.6–5.6(rate 覆盖 ±15% 后)。
|
|
57
|
+
# 只用中位数会误报:模板片实测 5.2 字/秒的句子被按 4.2 估 -> 全片多算 5.2 s ->
|
|
58
|
+
# "剧本与 timeline 差 5.23 s"的假 WARN(v0.7.2 修,模板片复现)。
|
|
59
|
+
CPS_LO, CPS_HI = 3.6, 5.6
|
|
55
60
|
DEFAULT_LEAD, DEFAULT_TAIL, DEFAULT_MIN_DUR = 0.62, 0.92, 3.0
|
|
56
61
|
TIMELINE_TOL = 0.5 # 估算与 timeline 差超过这个秒数就 WARN
|
|
57
62
|
|
|
@@ -286,6 +291,11 @@ SCRIPT_SPEC = {
|
|
|
286
291
|
"seed": {"t": "int", "min": 0},
|
|
287
292
|
"seed_from_content": {"t": "bool"},
|
|
288
293
|
"gain_db": {"t": "num"}, "duck_db": {"t": "num"},
|
|
294
|
+
# v0.7.1 旁白优先:mix_film 读这四个(背景在人声处额外让位多少 / 阈值)
|
|
295
|
+
"voice_duck_db": {"t": "num"}, "voice_duck_sfx_db": {"t": "num"},
|
|
296
|
+
"voice_thresh_db": {"t": "num"}, "voice_floor_db": {"t": "num"},
|
|
297
|
+
# v0.7.1 SNR 兜底:混音后实测人声/背景差,不足就整体压背景
|
|
298
|
+
"guard_snr_db": {"t": "num"}, "guard_cap_db": {"t": "num"},
|
|
289
299
|
"seg": {"t": "num", "min": 0.0}, "amp": {"t": "num", "min": 0.0},
|
|
290
300
|
"gain": {"t": "num", "min": 0.0}, "xfade": {"t": "num", "min": 0.0},
|
|
291
301
|
"chords": {"t": "list", "item": {"t": "list", "item": {"t": "num"}}},
|
|
@@ -294,6 +304,13 @@ SCRIPT_SPEC = {
|
|
|
294
304
|
"enabled": {"t": "bool"},
|
|
295
305
|
"gain_db": {"t": "num"},
|
|
296
306
|
"mix_gain_db": {"t": "num"},
|
|
307
|
+
# render.py 的 st_sfx 把它透传给 CueBus.build_audio():按配方名覆盖基础电平。
|
|
308
|
+
# 以前 schema/filmcheck 都没写,用户写了会被当未知键警告(其实真的会生效)。
|
|
309
|
+
"base_db": {"t": "dict", "open": True},
|
|
310
|
+
# v0.7 音频差异化:content_seed 由 CueBus 读(位置派生 vs 内容派生);
|
|
311
|
+
# duck_transitions 由 render.st_sfx 读(段落边界前的转场音效自动让位)。
|
|
312
|
+
"content_seed": {"t": "bool"},
|
|
313
|
+
"duck_transitions": {"t": "bool"},
|
|
297
314
|
"cues": {"t": "list", "item": CUE_SPEC}}},
|
|
298
315
|
"web": {"t": "dict", "props": {
|
|
299
316
|
"entry": {"t": "str"},
|
|
@@ -468,6 +485,18 @@ def check_cross(script, script_path, rep, ctx):
|
|
|
468
485
|
ids.append(lid)
|
|
469
486
|
idset = set(ids)
|
|
470
487
|
|
|
488
|
+
# ---- lines[].tail 当前不生效(保留字段):写了会静默无效,必须说出来 ----
|
|
489
|
+
for i, ln in enumerate(lines):
|
|
490
|
+
if _f(ln.get("tail", 0.0)) > 0:
|
|
491
|
+
lid = ln.get("id") or ("l%02d" % (i + 1))
|
|
492
|
+
rep.warn("lines[%d].tail" % i, (
|
|
493
|
+
"film.json 的逐句 tail 目前**不生效**:filmkit.build_timeline 只读 "
|
|
494
|
+
"tail_map,不读 lines[].tail(渲染跑得通,但句后留白没变)。要逐句"
|
|
495
|
+
"控制句后留白,请写顶层 tail_map(形如 l03: 1.4),或交给 voicekit 的 "
|
|
496
|
+
"voice_plan(--stage tts 落 work/voice_plan.json,st_layout 会并进 "
|
|
497
|
+
"tail_map)。例如 %s: 1.4" % lid),
|
|
498
|
+
"inert_line_tail")
|
|
499
|
+
|
|
471
500
|
# ---- sfx.cues[].at 引用的场景 id 必须在 lines 里 ----
|
|
472
501
|
sfx = script.get("sfx") if isinstance(script.get("sfx"), dict) else {}
|
|
473
502
|
cues = sfx.get("cues") if isinstance(sfx.get("cues"), list) else []
|
|
@@ -497,23 +526,63 @@ def check_cross(script, script_path, rep, ctx):
|
|
|
497
526
|
except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError) as e:
|
|
498
527
|
rep.warn(str(tl_path), "读不了 timeline(%s),跳过时长对账" % e, "timeline_unreadable")
|
|
499
528
|
return
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
529
|
+
# 节拍覆盖的优先级与 stylekit.resolve_script_style 一致:pacing > 顶层字段 > 默认。
|
|
530
|
+
# 不读 pacing 会把"pacing.lead=2.0"当成"没写 lead",估出来的时长偏小一半 -> 假 WARN。
|
|
531
|
+
pace = script.get("pacing") if isinstance(script.get("pacing"), dict) else {}
|
|
532
|
+
|
|
533
|
+
def _pace(key, default):
|
|
534
|
+
for box in (pace, script):
|
|
535
|
+
v = box.get(key) if isinstance(box, dict) else None
|
|
536
|
+
if v is not None:
|
|
537
|
+
return _f(v, default)
|
|
538
|
+
return float(default)
|
|
539
|
+
|
|
540
|
+
lead = _pace("lead", DEFAULT_LEAD)
|
|
541
|
+
tail = _pace("tail", DEFAULT_TAIL)
|
|
542
|
+
min_dur = _pace("min_dur", DEFAULT_MIN_DUR)
|
|
543
|
+
ghold = _pace("hold", 0.0)
|
|
544
|
+
holds, speech, est_lo, est_hi = 0.0, 0.0, 0.0, 0.0
|
|
505
545
|
for ln in lines:
|
|
506
|
-
|
|
546
|
+
chars = _weighted_chars(ln.get("text"))
|
|
507
547
|
hold = _f(ln.get("hold", ghold))
|
|
508
548
|
holds += hold
|
|
509
|
-
speech +=
|
|
510
|
-
|
|
511
|
-
|
|
549
|
+
speech += chars / CPS
|
|
550
|
+
# 快语速 = 时长短(下界),慢语速 = 时长长(上界)
|
|
551
|
+
est_lo += max(min_dur, lead + chars / CPS_HI + tail + hold)
|
|
552
|
+
est_hi += max(min_dur, lead + chars / CPS_LO + tail + hold)
|
|
553
|
+
# bar > 0 把每段向上取整到小节倍数,total 天然比逐句估算大;这时不拿估算对账。
|
|
554
|
+
drift = 0.0
|
|
555
|
+
if not (_f(script.get("bar", 0.0)) > 0):
|
|
556
|
+
if total < est_lo - TIMELINE_TOL:
|
|
557
|
+
drift = total - est_lo
|
|
558
|
+
elif total > est_hi + TIMELINE_TOL:
|
|
559
|
+
drift = total - est_hi
|
|
560
|
+
if drift:
|
|
512
561
|
rep.warn(str(tl_path),
|
|
513
|
-
"剧本 hold 合计 %.2f s + 语音预估 %.2f s(含 lead/tail)≈ 全片
|
|
514
|
-
"
|
|
562
|
+
"剧本 hold 合计 %.2f s + 语音预估 %.2f–%.2f s(含 lead/tail)≈ 全片 "
|
|
563
|
+
"%.2f–%.2f s,与 timeline 的 total=%.2f s 差 %.2f s > %.1f s。"
|
|
515
564
|
"改过台词 / hold / 节奏参数就重跑 --stage layout(别拿旧时间轴直接渲)"
|
|
516
|
-
% (holds,
|
|
565
|
+
% (holds, est_lo, est_hi, est_lo, est_hi, total,
|
|
566
|
+
abs(drift), TIMELINE_TOL), "timeline_drift")
|
|
567
|
+
# 台词内容逐句对账(最可靠的一条:build_timeline 把整条 line 原样拷进 scene,
|
|
568
|
+
# 所以 timeline.scenes[i].text 就是"上次排轴时那句话")。改过台词没重跑 layout,
|
|
569
|
+
# 这里能精确点出来;比按时长估算准得多,也不会被语速/语气差异干扰。
|
|
570
|
+
tl_by_id = {str(s.get("id")): s for s in (tl.get("scenes") or []) if isinstance(s, dict)}
|
|
571
|
+
text_drift = []
|
|
572
|
+
for ln in lines:
|
|
573
|
+
lid = ln.get("id")
|
|
574
|
+
if lid is None or not str(lid).strip():
|
|
575
|
+
continue
|
|
576
|
+
scene = tl_by_id.get(str(lid))
|
|
577
|
+
if not isinstance(scene, dict) or "text" not in scene:
|
|
578
|
+
continue # 老版本 timeline 不带 text:跳过
|
|
579
|
+
if str(scene.get("text")) != str(ln.get("text")):
|
|
580
|
+
text_drift.append(str(lid))
|
|
581
|
+
if text_drift:
|
|
582
|
+
rep.warn(str(tl_path),
|
|
583
|
+
"timeline 里 %d 句的台词与 film.json 不一致(%s):TTS 与字幕都还是旧词,"
|
|
584
|
+
"改完台词必须重跑 --stage tts layout 再渲"
|
|
585
|
+
% (len(text_drift), ", ".join(text_drift[:6])), "timeline_text")
|
|
517
586
|
tl_ids = [str(s.get("id")) for s in (tl.get("scenes") or []) if isinstance(s, dict)]
|
|
518
587
|
if idset and tl_ids and set(tl_ids) != idset:
|
|
519
588
|
miss = sorted(idset - set(tl_ids))
|
|
@@ -591,6 +591,12 @@ def wav_write(path, samples, sr=SR, channels=None):
|
|
|
591
591
|
v0.3.0:支持立体声(配乐/音效要声像)。老调用 wav_write(p, mono) 行为不变。
|
|
592
592
|
"""
|
|
593
593
|
a = np.asarray(samples, dtype=np.float32)
|
|
594
|
+
# 非有限值必须报错:旧实现 np.clip 保留 NaN/Inf,astype(int16) 把 NaN 静默写成 0、
|
|
595
|
+
# 把 Inf 写成 undefined,wav 里出现"随机哑音"且函数还返回 nan 峰值(下游只看得到 no)。
|
|
596
|
+
if a.size and not np.all(np.isfinite(a)):
|
|
597
|
+
bad = int(a.size - np.count_nonzero(np.isfinite(a)))
|
|
598
|
+
raise ValueError("wav_write: 采样里有 %d 个 NaN/Inf(%s)——上游混音/合成有数值错误,"
|
|
599
|
+
"拒绝写出损坏的 WAV" % (bad, path))
|
|
594
600
|
if a.ndim == 2:
|
|
595
601
|
n_ch = a.shape[1]
|
|
596
602
|
if channels is None:
|
|
@@ -722,6 +728,12 @@ class AudioBus:
|
|
|
722
728
|
if i0 >= len(self.bus):
|
|
723
729
|
return self
|
|
724
730
|
s = np.asarray(src, dtype=np.float32)
|
|
731
|
+
if i0 < 0:
|
|
732
|
+
# 负起点 = 从 0 秒开始、裁掉前面 |i0| 个采样(与 sfxkit._place 同口径)。
|
|
733
|
+
# 旧实现把负 i0 直接当负下标切片,n 变成 0 后广播崩(place(src, -0.25) 必炸)。
|
|
734
|
+
if i0 <= -len(s):
|
|
735
|
+
return self
|
|
736
|
+
s, i0 = s[-i0:], 0
|
|
725
737
|
if pan_pos is not None and self.channels == 2 and s.ndim == 1:
|
|
726
738
|
theta = (float(np.clip(pan_pos, -1.0, 1.0)) + 1.0) * np.pi / 4.0
|
|
727
739
|
s = np.stack([s * np.cos(theta), s * np.sin(theta)], axis=1)
|
|
@@ -749,6 +761,11 @@ class AudioBus:
|
|
|
749
761
|
return self
|
|
750
762
|
t = float(start)
|
|
751
763
|
idx = 0
|
|
764
|
+
if not (float(seg) > 0.0):
|
|
765
|
+
# seg=0 时 t 不前进 -> while 变成死循环(100000 次上限只是"慢死",
|
|
766
|
+
# 6 秒片实测 >2 分钟)。schema 只约束 seg>=0,必须在入口拒绝。
|
|
767
|
+
raise ValueError("music_bed: seg 必须 > 0,收到 %r(v0.2 和弦床的段长,默认 4.0)"
|
|
768
|
+
% (seg,))
|
|
752
769
|
while t < self.total:
|
|
753
770
|
freqs = chords[idx % len(chords)]
|
|
754
771
|
dur = min(seg + xfade, max(0.25, self.total - t + xfade))
|
|
@@ -841,6 +858,21 @@ def scenes_anchors(scenes):
|
|
|
841
858
|
start=float(s["start"]), end=float(s["end"])) for s in scenes]
|
|
842
859
|
|
|
843
860
|
|
|
861
|
+
def content_seed(base: int, style: str, content_fp: str, recipe: str, t: float) -> int:
|
|
862
|
+
"""内容派生 seed(v0.7):同影片同 cue 稳定,不同影片同一位置也换音色细节。
|
|
863
|
+
|
|
864
|
+
公式与 sfxkit.content_seed 一致(references/audio-variation.md §3.1):
|
|
865
|
+
sha1("v0.7|<base>|<style>|<content_fp>|<recipe>|<t 三位小数>") -> 前 8 位 hex -> % 9000 + 1000
|
|
866
|
+
|
|
867
|
+
为什么不用 seed_base + i:那样只跟 cue 的**出现顺序**有关,两部内容完全不同的片子
|
|
868
|
+
只要 cues 一样,音效轨就逐样本相同(跨片"音效都一样"的直接来源)。
|
|
869
|
+
"""
|
|
870
|
+
h = hashlib.sha1(
|
|
871
|
+
("v0.7|%d|%s|%s|%s|%.3f" % (int(base), str(style), str(content_fp), str(recipe), float(t)))
|
|
872
|
+
.encode("utf-8")).hexdigest()
|
|
873
|
+
return int(h[:8], 16) % 9000 + 1000
|
|
874
|
+
|
|
875
|
+
|
|
844
876
|
class CueBus:
|
|
845
877
|
"""提示点登记表:画面侧登记 -> 落盘 cues.json -> 音频侧按它合成音效。
|
|
846
878
|
|
|
@@ -918,11 +950,21 @@ class CueBus:
|
|
|
918
950
|
|
|
919
951
|
# -- 合成 --
|
|
920
952
|
def build_audio(self, total=None, sr=SR, base_db=None, default_gain_db=0.0,
|
|
921
|
-
seed_base=1000, verbose=True
|
|
953
|
+
seed_base=1000, verbose=True,
|
|
954
|
+
style=None, content_fp=None, key=None):
|
|
922
955
|
"""按提示点合成音效轨 -> (n,2) float32。
|
|
923
956
|
|
|
924
957
|
依赖 scripts/sfxkit.py(v0.3.0 新增)。缺 sfxkit 时直接报错,不静默出哑片。
|
|
925
958
|
未知提示点类型:告警一次并跳过(不 crash 整片混音)。
|
|
959
|
+
提示点参数非法(配方不认识的关键字 / 取值连枚举校验都过不了,TypeError 或
|
|
960
|
+
ValueError):该条跳过并汇总 [WARN],其余正常落位——不静默改渲染默认音色,
|
|
961
|
+
也不因一条 cue 写错丢整条音效轨。
|
|
962
|
+
|
|
963
|
+
v0.7 音频差异化(见 references/audio-variation.md):
|
|
964
|
+
- style / content_fp 同时给时,每条 cue 的 seed 由
|
|
965
|
+
content_seed(seed_base, style, content_fp, type, t) 派生 -> 不同影片同一 cue
|
|
966
|
+
位置也拿到不同音色细节;任一为 None 时退回 seed_base + i(旧路径逐样本兼容)。
|
|
967
|
+
- key 非空时透传给配方(提示音调性跟随配乐,不再硬编码 Dm)。
|
|
926
968
|
"""
|
|
927
969
|
try:
|
|
928
970
|
import sfxkit as SX
|
|
@@ -937,19 +979,46 @@ class CueBus:
|
|
|
937
979
|
n = int(round(dur * sr)) + int(sr * 0.5)
|
|
938
980
|
bus = np.zeros((n, 2), dtype=np.float32)
|
|
939
981
|
unknown, placed = set(), 0
|
|
982
|
+
bad_cues: list = [] # 参数非法/内部报错的 cue:跳过并汇总 WARN,不拖死整条音效轨
|
|
983
|
+
keyed = style is not None and content_fp is not None
|
|
984
|
+
if keyed and verbose:
|
|
985
|
+
print(" 音效 seed: 内容派生(style=%s / fp=%s)" % (style, str(content_fp)[:8]))
|
|
986
|
+
elif verbose and style is not None:
|
|
987
|
+
print(" 音效 seed: 位置派生(只给了 style 没给 content_fp)")
|
|
940
988
|
for i, c in enumerate(self.resolved()):
|
|
941
989
|
kind = str(c.get("type"))
|
|
942
990
|
if kind not in reg:
|
|
943
991
|
unknown.add(kind)
|
|
944
992
|
continue
|
|
945
993
|
params = {k: v for k, v in c.items()
|
|
946
|
-
if k not in ("t", "type", "src", "gain", "pan", "base_db")}
|
|
994
|
+
if k not in ("t", "type", "src", "gain", "pan", "base_db", "key")}
|
|
947
995
|
params.pop("dur_hint", None)
|
|
996
|
+
if keyed:
|
|
997
|
+
seed = content_seed(seed_base, style, content_fp, kind, float(c["t"]))
|
|
998
|
+
else:
|
|
999
|
+
seed = int(seed_base) + i
|
|
1000
|
+
if key is not None:
|
|
1001
|
+
params["key"] = key
|
|
948
1002
|
try:
|
|
949
|
-
x = SX.render(kind, seed=
|
|
950
|
-
except TypeError:
|
|
951
|
-
#
|
|
952
|
-
|
|
1003
|
+
x = SX.render(kind, seed=seed, **params)
|
|
1004
|
+
except (TypeError, ValueError) as err:
|
|
1005
|
+
# 只对"配方不认识这个关键字"逐个删参重试(v0.4 严格签名配方的兼容);
|
|
1006
|
+
# 参数值非法(cue 里 pitch=None 之类)或配方内部 TypeError 不是签名问题,
|
|
1007
|
+
# 必须报告——旧实现一路退化成"只传 seed",把坏参数静默渲染成默认音色
|
|
1008
|
+
# (用户改了 cue 参数却听不出变化,日志里一个字都没有)。
|
|
1009
|
+
extra = dict(params)
|
|
1010
|
+
while True:
|
|
1011
|
+
hit = re.search(r"unexpected keyword argument '([^']+)'", str(err))
|
|
1012
|
+
if not hit or hit.group(1) not in extra:
|
|
1013
|
+
bad_cues.append((kind, float(c["t"]), err))
|
|
1014
|
+
x = None
|
|
1015
|
+
break
|
|
1016
|
+
extra.pop(hit.group(1))
|
|
1017
|
+
try:
|
|
1018
|
+
x = SX.render(kind, seed=seed, **extra)
|
|
1019
|
+
break
|
|
1020
|
+
except (TypeError, ValueError) as err2:
|
|
1021
|
+
err = err2
|
|
953
1022
|
if x is None or len(x) == 0:
|
|
954
1023
|
continue
|
|
955
1024
|
x = np.asarray(x, dtype=np.float32)
|
|
@@ -968,12 +1037,123 @@ class CueBus:
|
|
|
968
1037
|
placed += 1
|
|
969
1038
|
if unknown and verbose:
|
|
970
1039
|
print(" [WARN] 未知提示点类型(已跳过): %s" % ", ".join(sorted(unknown)))
|
|
1040
|
+
if bad_cues and verbose:
|
|
1041
|
+
# 与"未知 type 跳过"同一条隔离策略:一条 cue 的参数写错不该让整条音效轨消失
|
|
1042
|
+
# (st_sfx 会把 build_audio 的异常当成"音效合成失败"整轨丢弃)。但也不能静默:
|
|
1043
|
+
# 逐条打印 cue 类型/时刻/原因,用户改完参数能立刻看到。
|
|
1044
|
+
print(" [WARN] %d 条提示点参数非法(已跳过,其余正常落位):" % len(bad_cues))
|
|
1045
|
+
for kind_b, t_b, err_b in bad_cues[:6]:
|
|
1046
|
+
print(" %-14s t=%.3fs -> %s" % (kind_b, t_b, str(err_b)[:110]))
|
|
971
1047
|
if verbose:
|
|
972
1048
|
print(" 音效轨:%d 个提示点落位 / %d 个登记(%.2fs)"
|
|
973
1049
|
% (placed, len(self.items), dur))
|
|
974
1050
|
return bus[: int(round(dur * sr))]
|
|
975
1051
|
|
|
976
1052
|
|
|
1053
|
+
def voice_env(voice, sr=SR, block_s=0.01, thresh_db=-42.0, floor_db=-60.0):
|
|
1054
|
+
"""旁白的语音活跃度包络(0~1):给"旁白优先"混音用的侧链源。
|
|
1055
|
+
|
|
1056
|
+
为什么需要它:v0.7 之前配乐基线只比旁白低 1.5 dB(`music.gain_db=-1.5`),
|
|
1057
|
+
而视频是"以旁白为主"的——背景只要不明显低于人声,观众就得费力分辨字。
|
|
1058
|
+
`mix_film` 现在用这条包络在**有人声的地方**把配乐/音效再压下去,
|
|
1059
|
+
没人声的间隙自动让背景回来(否则整片背景像被闷住)。
|
|
1060
|
+
|
|
1061
|
+
实现:10 ms 块级 RMS -> dB -> 在 [floor_db, thresh_db] 之间线性映射成 0~1 ->
|
|
1062
|
+
起音快 / 释放慢的单向平滑(与 `duck()` 同一套向量化手法,无逐采样循环)。
|
|
1063
|
+
"""
|
|
1064
|
+
x = np.asarray(voice, dtype=np.float32)
|
|
1065
|
+
if x.ndim == 2:
|
|
1066
|
+
x = x.mean(axis=1)
|
|
1067
|
+
n = len(x)
|
|
1068
|
+
if n == 0:
|
|
1069
|
+
return np.zeros(0, dtype=np.float32)
|
|
1070
|
+
block = max(1, int(sr * float(block_s)))
|
|
1071
|
+
nb = n // block
|
|
1072
|
+
if nb == 0: # 极短旁白(< 10 ms):整段当一块并补零
|
|
1073
|
+
nb = 1 # (旧实现 reshape(1, block) 直接崩)
|
|
1074
|
+
x = np.concatenate([x, np.zeros(block - n, dtype=np.float32)])
|
|
1075
|
+
seg = x[: nb * block].reshape(nb, block)
|
|
1076
|
+
rms = np.sqrt(np.mean(seg * seg, axis=1) + 1e-12)
|
|
1077
|
+
db = 20.0 * np.log10(rms + 1e-12)
|
|
1078
|
+
lo = float(min(floor_db, thresh_db - 1.0))
|
|
1079
|
+
e = np.clip((db - lo) / (float(thresh_db) - lo), 0.0, 1.0)
|
|
1080
|
+
|
|
1081
|
+
def _smooth(v, tau_s):
|
|
1082
|
+
# 极短旁白时块数可能比核还少:np.convolve(mode="same") 返回 max(len(v), k) 长度,
|
|
1083
|
+
# 两个长度不同的平滑结果再做 np.maximum 会广播失败(v0.7.1 起 voice_env 崩在这里)。
|
|
1084
|
+
k = min(max(1, int(sr * tau_s / block)), len(v))
|
|
1085
|
+
w = np.ones(k, dtype=np.float32) / k
|
|
1086
|
+
return np.convolve(v, w, mode="same")
|
|
1087
|
+
|
|
1088
|
+
e = np.maximum(_smooth(e, 0.06), _smooth(e, 0.45)).astype(np.float32)
|
|
1089
|
+
return np.interp(np.arange(n), np.arange(nb) * block + block / 2.0, e).astype(np.float32)
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
def duck_voice(track, voice, depth_db=6.0, sr=SR, thresh_db=-42.0, floor_db=-60.0):
|
|
1093
|
+
"""旁白说话时压低背景(配乐 / 音效):旁白优先的侧链压缩(只压背景,不碰人声)。
|
|
1094
|
+
|
|
1095
|
+
depth_db 是"人声满电平时背景最多被压多少 dB";间隙里包络回 0,背景恢复原电平。
|
|
1096
|
+
与 `duck()`(音效压配乐)叠加使用:人声 > 音效 > 配乐,三条让位链互不冲突。
|
|
1097
|
+
"""
|
|
1098
|
+
b = np.asarray(track, dtype=np.float32)
|
|
1099
|
+
if b.ndim == 1:
|
|
1100
|
+
b2 = np.stack([b, b], axis=1)
|
|
1101
|
+
else:
|
|
1102
|
+
b2 = b
|
|
1103
|
+
n = len(b2)
|
|
1104
|
+
if n == 0 or depth_db <= 0:
|
|
1105
|
+
return b2
|
|
1106
|
+
e = voice_env(voice, sr=sr, thresh_db=thresh_db, floor_db=floor_db)[:n]
|
|
1107
|
+
if e.size == 0:
|
|
1108
|
+
return b2
|
|
1109
|
+
if e.size < n:
|
|
1110
|
+
# 旁白比背景短(单独调用时可能遇到):旁白结束之后没有侧链源,背景应原样恢复。
|
|
1111
|
+
e = np.concatenate([e, np.zeros(n - e.size, dtype=np.float32)])
|
|
1112
|
+
if float(np.max(e)) <= 1e-6:
|
|
1113
|
+
return b2
|
|
1114
|
+
gains = 10.0 ** (-float(depth_db) * e / 20.0)
|
|
1115
|
+
return (b2 * gains[:, None]).astype(np.float32)
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
def voice_snr(voice, bg, sr=SR, thresh_db=-42.0, floor_db=-60.0):
|
|
1119
|
+
"""在"有人声"的采样区间量 人声/背景 的响度差(dB)。旁白优先的客观指标。
|
|
1120
|
+
|
|
1121
|
+
只在语音活跃度 >= 0.5 的区间统计:静音间隙里背景当然是唯一声源,
|
|
1122
|
+
把间隙算进来会稀释问题(真正让人听不清的,是人说话那一刻背景有多响)。
|
|
1123
|
+
人声总长不足 50 ms 时返回 None(没有可判定的样本)。
|
|
1124
|
+
"""
|
|
1125
|
+
v = np.asarray(voice, dtype=np.float64)
|
|
1126
|
+
b = np.asarray(bg, dtype=np.float64)
|
|
1127
|
+
if v.ndim == 2:
|
|
1128
|
+
v = v.mean(axis=1)
|
|
1129
|
+
if b.ndim == 2:
|
|
1130
|
+
b = b.mean(axis=1)
|
|
1131
|
+
n = min(len(v), len(b))
|
|
1132
|
+
if n <= 0:
|
|
1133
|
+
return None
|
|
1134
|
+
v, b = v[:n], b[:n]
|
|
1135
|
+
e = voice_env(v, sr=sr, thresh_db=thresh_db, floor_db=floor_db)
|
|
1136
|
+
act = e >= 0.5
|
|
1137
|
+
if int(np.sum(act)) < int(0.05 * sr):
|
|
1138
|
+
return None
|
|
1139
|
+
vr = float(np.sqrt(np.mean(v[act] ** 2)))
|
|
1140
|
+
br = float(np.sqrt(np.mean(b[act] ** 2)))
|
|
1141
|
+
return 20.0 * np.log10((vr + 1e-12) / (br + 1e-12))
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def guard_snr(voice, bg, target_db=12.0, cap_db=12.0, sr=SR,
|
|
1145
|
+
thresh_db=-42.0, floor_db=-60.0):
|
|
1146
|
+
"""旁白清晰度兜底:人声/背景不足 target_db 时,返回背景应再压多少 dB。
|
|
1147
|
+
|
|
1148
|
+
返回 (trim_db, snr_db):trim_db=0 表示已达标;cap_db 防止把背景压到无声
|
|
1149
|
+
(压过头会变成"配乐忽有忽无",比听不清更假)。判定口径同 `voice_snr()`。
|
|
1150
|
+
"""
|
|
1151
|
+
snr = voice_snr(voice, bg, sr=sr, thresh_db=thresh_db, floor_db=floor_db)
|
|
1152
|
+
if snr is None or snr >= float(target_db):
|
|
1153
|
+
return 0.0, (snr if snr is not None else float("nan"))
|
|
1154
|
+
return float(min(max(0.0, float(cap_db)), float(target_db) - snr)), snr
|
|
1155
|
+
|
|
1156
|
+
|
|
977
1157
|
def duck(music, sfx, depth_db=4.0, attack=0.03, release=0.35, sr=SR):
|
|
978
1158
|
"""音效响起时配乐自动让位(最多 depth_db),避免重击被配乐吃掉。
|
|
979
1159
|
|
|
@@ -991,13 +1171,18 @@ def duck(music, sfx, depth_db=4.0, attack=0.03, release=0.35, sr=SR):
|
|
|
991
1171
|
if env.size == 0 or float(env.max()) <= 1e-9:
|
|
992
1172
|
return m2
|
|
993
1173
|
block = max(1, int(sr * 0.01))
|
|
994
|
-
nb =
|
|
1174
|
+
nb = n // block
|
|
1175
|
+
if nb == 0: # 极短音效(< 10 ms):整段当一块并补零
|
|
1176
|
+
nb = 1
|
|
1177
|
+
env = np.concatenate([env, np.zeros(block - n, dtype=np.float32)])
|
|
995
1178
|
e = env[: nb * block].reshape(nb, block).max(axis=1)
|
|
996
1179
|
e = e / (float(np.percentile(e, 99.5)) + 1e-9)
|
|
997
1180
|
e = np.clip(e, 0.0, 1.0)
|
|
998
1181
|
# 单向平滑:起音快、释放慢(对包络做两次不同时间常数的指数平滑,仍保持向量化)
|
|
999
1182
|
def _smooth(x, tau_s):
|
|
1000
|
-
|
|
1183
|
+
# 与 voice_env 同一处理:短片段(< release 0.35 s)时核不能比信号长,
|
|
1184
|
+
# 否则 np.convolve(mode="same") 返回核长、np.maximum 广播失败(短片混音直接崩)。
|
|
1185
|
+
k = min(max(1, int(sr * tau_s / block)), len(x))
|
|
1001
1186
|
w = np.ones(k, dtype=np.float32) / k
|
|
1002
1187
|
return np.convolve(x, w, mode="same")
|
|
1003
1188
|
e = np.maximum(_smooth(e, attack), _smooth(e, release) * 0.7)
|
|
@@ -1008,17 +1193,35 @@ def duck(music, sfx, depth_db=4.0, attack=0.03, release=0.35, sr=SR):
|
|
|
1008
1193
|
|
|
1009
1194
|
def mix_film(voice, music=None, sfx=None, total=None, sr=SR,
|
|
1010
1195
|
voice_gain=1.0, music_gain_db=0.0, sfx_gain_db=0.0,
|
|
1011
|
-
duck_db=3.0, peak_target=0.89, fade=0.4, verbose=True
|
|
1196
|
+
duck_db=3.0, peak_target=0.89, fade=0.4, verbose=True,
|
|
1197
|
+
voice_duck_db=6.0, voice_duck_sfx_db=3.0,
|
|
1198
|
+
voice_thresh_db=-42.0, voice_floor_db=-60.0,
|
|
1199
|
+
guard_snr_db=12.0, guard_cap_db=30.0):
|
|
1012
1200
|
"""三总线混音:旁白 + 配乐 + 音效 -> 立体声 float32。
|
|
1013
1201
|
|
|
1014
|
-
|
|
1015
|
-
|
|
1202
|
+
**旁白优先**(v0.7.1 起是硬行为,不是文档口号):
|
|
1203
|
+
1. 旁白电平固定 `voice_gain=1.0`,不被任何侧链影响;
|
|
1204
|
+
2. 有人声的地方,配乐再压 `voice_duck_db`(默认 6 dB)、音效再压 `voice_duck_sfx_db`(默认 3 dB)
|
|
1205
|
+
—— 保证"人声比背景响 ≥12 dB"这条可听清线;没人声的间隙自动恢复;
|
|
1206
|
+
3. 音效响时配乐额外让位(`duck_db`,v0.7 默认 4 dB);
|
|
1207
|
+
4. 最后统一做峰值保护(真正的响度标准化交给 ffmpeg 的 loudnorm)。
|
|
1208
|
+
|
|
1209
|
+
`voice_duck_db=0` / `voice_duck_sfx_db=0` 可单独关掉旁白优先让位(旧片对比用)。
|
|
1210
|
+
|
|
1211
|
+
**SNR 兜底(guard_snr_db=12,0 = 关)**:按台词活跃区实测"人声/背景"响度差,
|
|
1212
|
+
不足 12 dB 就整体再压背景(最多 guard_cap_db),保证"听清旁白"不依赖配乐素材的天然电平——
|
|
1213
|
+
这是本仓库唯一一条会在混音后自己改电平的门禁。触发时会打印实际 SNR 与补压量。
|
|
1016
1214
|
"""
|
|
1017
1215
|
def _to_stereo(x):
|
|
1018
1216
|
a = np.asarray(x, dtype=np.float32)
|
|
1019
1217
|
if a.ndim == 1:
|
|
1020
1218
|
return np.stack([a, a], axis=1)
|
|
1021
|
-
|
|
1219
|
+
if a.shape[1] == 1:
|
|
1220
|
+
return np.concatenate([a, a], axis=1)
|
|
1221
|
+
# 多于 2 声道(例如把 5.1 的 music.wav 塞进 work/):只取前两声道。
|
|
1222
|
+
# 旧实现原样返回 (n,3+),back += mm 直接广播崩(mix_film 是公共 API,
|
|
1223
|
+
# 而 wav_read 不限制声道数,用户完全可能踩到)。口径与 AudioBus._fit 一致。
|
|
1224
|
+
return a[:, :2] if a.shape[1] > 2 else a
|
|
1022
1225
|
|
|
1023
1226
|
n = None
|
|
1024
1227
|
v = None if voice is None else _to_stereo(voice)
|
|
@@ -1033,18 +1236,57 @@ def mix_film(voice, music=None, sfx=None, total=None, sr=SR,
|
|
|
1033
1236
|
if not n:
|
|
1034
1237
|
return np.zeros((0, 2), dtype=np.float32)
|
|
1035
1238
|
|
|
1036
|
-
|
|
1239
|
+
# v0.7.1 旁白优先:先算背景(配乐 + 音效各自让位),再用 SNR 兜底确认"人声压得住"。
|
|
1240
|
+
# 侧链与 SNR 都用"实际交付的人声"(乘过 voice_gain):voice_gain=0.5 时旧实现用未缩放
|
|
1241
|
+
# 人声算 SNR,日志写"人声/背景=12 dB 达标"、交付出去只有 6 dB(兜底承诺失效)。
|
|
1242
|
+
venv_kw = dict(sr=sr, thresh_db=float(voice_thresh_db), floor_db=float(voice_floor_db))
|
|
1243
|
+
vg = None
|
|
1244
|
+
if v is not None:
|
|
1245
|
+
vg = v[:n] * float(voice_gain)
|
|
1246
|
+
back = np.zeros((n, 2), dtype=np.float32)
|
|
1037
1247
|
if s is not None:
|
|
1038
|
-
|
|
1248
|
+
ss = s[:n] * (10.0 ** (float(sfx_gain_db) / 20.0))
|
|
1249
|
+
if vg is not None and float(voice_duck_sfx_db) > 0:
|
|
1250
|
+
ss = duck_voice(ss, vg, depth_db=float(voice_duck_sfx_db), **venv_kw)
|
|
1251
|
+
back += ss
|
|
1039
1252
|
if m is not None:
|
|
1040
1253
|
mm = m[:n] * (10.0 ** (float(music_gain_db) / 20.0))
|
|
1041
1254
|
if s is not None:
|
|
1042
1255
|
mm = duck(mm, s[:n], depth_db=duck_db, sr=sr)
|
|
1043
1256
|
else:
|
|
1044
1257
|
mm = _to_stereo(mm)
|
|
1045
|
-
|
|
1258
|
+
if vg is not None and float(voice_duck_db) > 0:
|
|
1259
|
+
mm = duck_voice(mm, vg, depth_db=float(voice_duck_db), **venv_kw)
|
|
1260
|
+
back += mm
|
|
1261
|
+
# SNR 兜底:实测人声活跃区的人声/背景比;不够就整体再压背景(不碰人声)。
|
|
1262
|
+
if vg is not None and back.any() and float(guard_snr_db) > 0:
|
|
1263
|
+
trim_db, snr_db = guard_snr(vg, back, target_db=float(guard_snr_db),
|
|
1264
|
+
cap_db=float(guard_cap_db), **venv_kw)
|
|
1265
|
+
if trim_db > 0.0:
|
|
1266
|
+
back *= 10.0 ** (-trim_db / 20.0)
|
|
1267
|
+
if verbose:
|
|
1268
|
+
if snr_db == snr_db: # 非 NaN = 有可判定的人声段
|
|
1269
|
+
note = ",达标"
|
|
1270
|
+
if trim_db > 0:
|
|
1271
|
+
note = ",背景再压 %.1f dB" % trim_db
|
|
1272
|
+
if trim_db >= float(guard_cap_db) - 1e-6:
|
|
1273
|
+
note += "(已到兜底上限:检查配乐/音效素材电平)"
|
|
1274
|
+
elif trim_db > 18.0:
|
|
1275
|
+
note += "(补压很大:建议调低 music.gain_db)"
|
|
1276
|
+
print(" 旁白优先: 人声/背景 = %.1f dB(目标 %.1f)%s"
|
|
1277
|
+
% (snr_db, float(guard_snr_db), note))
|
|
1278
|
+
else:
|
|
1279
|
+
# snr_db 是 NaN = 活跃区为空:旁白电平低于 voice_thresh_db 时,旁白优先
|
|
1280
|
+
# 与 SNR 兜底会**整体静默失效**(背景一点不压)。纯静音片是有意的,
|
|
1281
|
+
# 但"目的是说话却录得很轻"必须在日志里喊出来,否则等于没有门禁。
|
|
1282
|
+
vpk = float(np.max(np.abs(vg))) if vg.size else 0.0
|
|
1283
|
+
if vpk > 1e-4: # 有声音但到不了活跃区
|
|
1284
|
+
print(" [WARN] 旁白电平过低(峰值 %.1f dBFS,低于活跃度门限 %.0f dB)——"
|
|
1285
|
+
"旁白优先/SNR 兜底本次无法判定;检查 voice_gain / TTS 音量"
|
|
1286
|
+
% (20.0 * math.log10(vpk), float(voice_thresh_db)))
|
|
1287
|
+
out = back
|
|
1046
1288
|
if v is not None:
|
|
1047
|
-
out
|
|
1289
|
+
out = out + v[:n] * float(voice_gain)
|
|
1048
1290
|
|
|
1049
1291
|
raw = float(np.max(np.abs(out))) if out.size else 0.0
|
|
1050
1292
|
if fade > 0:
|
|
@@ -1619,8 +1861,8 @@ def mux(video, audio, out, crf=18, preset="medium", audio_bitrate="192k",
|
|
|
1619
1861
|
af = loudnorm_two_pass(audio, target_i=target_i, target_tp=target_tp) if loudnorm else None
|
|
1620
1862
|
if loudnorm and not af:
|
|
1621
1863
|
# 不静默降级:量不到就喊出来(qa 的响度门禁还在,但这里先说清楚)
|
|
1622
|
-
print(" [WARN] loudnorm
|
|
1623
|
-
flush=True)
|
|
1864
|
+
print(" [WARN] loudnorm 两遍测量失败(拿不到可用测量值:全静音/极短音频会给出 -inf),"
|
|
1865
|
+
"本次不做响度标准化", flush=True)
|
|
1624
1866
|
|
|
1625
1867
|
vd = media_dur(video)
|
|
1626
1868
|
ad = media_dur(audio)
|
|
@@ -1658,6 +1900,17 @@ def loudnorm_two_pass(src, target_i=-16.0, target_tp=-1.5, target_lra=11.0):
|
|
|
1658
1900
|
j = json.loads(m.group(0))
|
|
1659
1901
|
except Exception:
|
|
1660
1902
|
return None
|
|
1903
|
+
# 全静音 / 极短音频时 ffmpeg 会报 input_i=-inf、target_offset=inf:这些值塞进
|
|
1904
|
+
# measured_* 会被 ffmpeg 拒绝("Value -inf for parameter 'measured_I' out of range"),
|
|
1905
|
+
# mux 直接失败(旧实现只判"解析到 JSON",于是拿着 -inf 的滤镜串去编码)。
|
|
1906
|
+
# 量不到可用测量值就返回 None:mux 会打 WARN 跳过响度标准化,qa 的响度门禁照旧兜底。
|
|
1907
|
+
need = ("input_i", "input_tp", "input_lra", "input_thresh", "target_offset")
|
|
1908
|
+
try:
|
|
1909
|
+
bad = [k for k in need if not math.isfinite(float(j.get(k, 0.0)))]
|
|
1910
|
+
except (TypeError, ValueError):
|
|
1911
|
+
return None
|
|
1912
|
+
if bad:
|
|
1913
|
+
return None
|
|
1661
1914
|
return ("loudnorm=I=%.1f:TP=%.1f:LRA=%.1f:measured_I=%s:measured_TP=%s:"
|
|
1662
1915
|
"measured_LRA=%s:measured_thresh=%s:offset=%s:print_format=summary"
|
|
1663
1916
|
% (target_i, target_tp, target_lra, j.get("input_i"), j.get("input_tp"),
|