@mthines/reaper-mcp 0.19.0 → 0.70.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -570
- package/bridge-ui/mcp_bridge.html +129 -0
- package/bridge-ui/mcp_bridge.webmanifest +16 -0
- package/bridge-ui/mcp_bridge_icon_192.png +0 -0
- package/bridge-ui/mcp_bridge_icon_512.png +0 -0
- package/bridge-ui/mcp_bridge_sw.js +35 -0
- package/claude/agents/plugin-researcher.md +188 -0
- package/claude/knowledge/reference/aesthetics-scoring.md +48 -0
- package/{knowledge → claude/knowledge}/reference/common-mistakes.md +1 -1
- package/claude/knowledge/reference/lorekit-setup.md +36 -0
- package/claude/knowledge/reference/memory-protocol.md +377 -0
- package/{knowledge → claude/knowledge}/reference/metering.md +5 -5
- package/claude/knowledge/reference/perceived-loudness.md +182 -0
- package/{knowledge → claude/knowledge}/workflows/editing.md +4 -0
- package/{knowledge → claude/knowledge}/workflows/gain-staging.md +21 -23
- package/{knowledge → claude/knowledge}/workflows/learn-plugin.md +15 -48
- package/{knowledge → claude/knowledge}/workflows/master-bus.md +13 -1
- package/claude/knowledge/workflows/section-levels.md +138 -0
- package/{knowledge → claude/knowledge}/workflows/session-prep.md +4 -0
- package/claude/knowledge/workflows/transient-lighting.md +52 -0
- package/claude/skills/critique/SKILL.md +131 -0
- package/claude/skills/editing/SKILL.md +281 -0
- package/claude/skills/learn-plugin/SKILL.md +210 -0
- package/claude/skills/mastering/SKILL.md +151 -0
- package/claude/skills/mix-memory/SKILL.md +140 -0
- package/claude/skills/mixer/SKILL.md +253 -0
- package/claude/skills/new-extension/SKILL.md +324 -0
- package/main.js +7843 -1940
- package/package.json +17 -15
- package/reaper/CLAUDE.md +22 -4
- package/reaper/mcp_actions_lib.lua +375 -0
- package/reaper/mcp_analyzer.jsfx +27 -12
- package/reaper/mcp_arrange_lib.lua +949 -0
- package/reaper/mcp_bridge.lua +2321 -758
- package/reaper/mcp_edit_views_lib.lua +1378 -0
- package/reaper/mcp_envelopes_lib.lua +748 -0
- package/reaper/mcp_extensions_lib.lua +55 -0
- package/reaper/mcp_fxstate_lib.lua +144 -0
- package/reaper/mcp_gain_trace.jsfx +126 -0
- package/reaper/mcp_items_lib.lua +877 -0
- package/reaper/mcp_lufs_meter.jsfx +16 -13
- package/reaper/mcp_midi_lib.lua +555 -0
- package/reaper/mcp_mixing_lib.lua +1021 -0
- package/reaper/mcp_plan_lib.lua +801 -0
- package/reaper/mcp_prefs_lib.lua +127 -0
- package/reaper/mcp_project_lib.lua +1088 -0
- package/reaper/mcp_recording_lib.lua +595 -0
- package/reaper/mcp_render_lib.lua +492 -0
- package/reaper/mcp_snapshot_lib.lua +547 -120
- package/reaper/mcp_snapshot_next.lua +10 -4
- package/reaper/mcp_snapshot_prev.lua +11 -4
- package/reaper/mcp_snapshot_quick_save.lua +7 -32
- package/reaper/mcp_snapshots_ui_lib.lua +258 -0
- package/reaper/mcp_tracks_lib.lua +409 -0
- package/sidecar/analysis.py +661 -0
- package/sidecar/chords.py +96 -0
- package/sidecar/mixcheck.py +295 -0
- package/sidecar/requirements-chords.txt +6 -0
- package/sidecar/requirements-separation.txt +3 -0
- package/sidecar/requirements.txt +17 -0
- package/sidecar/separation.py +142 -0
- package/sidecar/server.py +622 -0
- package/sidecar/tests/test_analysis.py +249 -0
- package/sidecar/tests/test_chords.py +117 -0
- package/sidecar/tests/test_mixcheck.py +163 -0
- package/sidecar/tests/test_separation.py +73 -0
- package/sidecar/tests/test_server.py +464 -0
- package/sidecar/tests/test_verify.py +243 -0
- package/sidecar/tests/test_voice_render.py +268 -0
- package/sidecar/tests/test_voice_sketch.py +140 -0
- package/sidecar/verify.py +224 -0
- package/sidecar/voice_render.py +295 -0
- package/sidecar/voice_sketch.py +280 -0
- package/claude-agents/critique.md +0 -147
- package/claude-agents/editor.md +0 -184
- package/claude-agents/levels.md +0 -107
- package/claude-agents/mastering.md +0 -129
- package/claude-agents/mixer.md +0 -248
- package/claude-agents/preflight.md +0 -149
- package/claude-agents/producer.md +0 -159
- package/claude-agents/setup.md +0 -150
- package/claude-agents/stems.md +0 -135
- package/claude-rules/architecture.md +0 -39
- package/claude-rules/development.md +0 -54
- package/claude-rules/lua-bridge.md +0 -50
- package/claude-rules/testing.md +0 -42
- package/claude-skills/learn-plugin.md +0 -123
- package/knowledge/CLAUDE.md +0 -71
- package/knowledge/plugins/_template.md +0 -82
- package/knowledge/plugins/fabfilter/pro-c-2.md +0 -117
- package/knowledge/plugins/fabfilter/pro-l-2.md +0 -95
- package/knowledge/plugins/fabfilter/pro-q-3.md +0 -112
- package/knowledge/plugins/neural-dsp/helix-native.md +0 -104
- package/knowledge/plugins/stock-reaper/js-1175-compressor.md +0 -94
- package/knowledge/plugins/stock-reaper/rea-comp.md +0 -100
- package/knowledge/plugins/stock-reaper/rea-delay.md +0 -95
- package/knowledge/plugins/stock-reaper/rea-eq.md +0 -103
- package/knowledge/plugins/stock-reaper/rea-gate.md +0 -99
- package/knowledge/plugins/stock-reaper/rea-limit.md +0 -75
- package/knowledge/plugins/stock-reaper/rea-verb.md +0 -76
- package/knowledge/reference/perceived-loudness.md +0 -122
- package/reaper/mcp_snapshot_manager.lua +0 -1001
- /package/{knowledge → claude/knowledge}/genres/_template.md +0 -0
- /package/{knowledge → claude/knowledge}/genres/electronic.md +0 -0
- /package/{knowledge → claude/knowledge}/genres/hip-hop.md +0 -0
- /package/{knowledge → claude/knowledge}/genres/metal.md +0 -0
- /package/{knowledge → claude/knowledge}/genres/orchestral.md +0 -0
- /package/{knowledge → claude/knowledge}/genres/pop.md +0 -0
- /package/{knowledge → claude/knowledge}/genres/rock.md +0 -0
- /package/{knowledge → claude/knowledge}/reference/compression.md +0 -0
- /package/{knowledge → claude/knowledge}/reference/frequencies.md +0 -0
- /package/{knowledge → claude/knowledge}/workflows/delivery.md +0 -0
- /package/{knowledge → claude/knowledge}/workflows/drum-bus.md +0 -0
- /package/{knowledge → claude/knowledge}/workflows/low-end.md +0 -0
- /package/{knowledge → claude/knowledge}/workflows/stem-prep.md +0 -0
- /package/{knowledge → claude/knowledge}/workflows/vocal-chain.md +0 -0
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Checks an agent can run on the result of an edit (verify_edit).
|
|
3
|
+
|
|
4
|
+
The agent cannot listen, so after an edit it compares the edited audio with the
|
|
5
|
+
audio from before the edit. Two checks held up across a vocal and a bass in the
|
|
6
|
+
experiment behind the tool (github.com/mthines/reaper-mcp issue 69):
|
|
7
|
+
|
|
8
|
+
* Compensated null: estimate the delay (sub-sample), gain and polarity between the
|
|
9
|
+
two, remove them, and measure what is left. A plain null test calls a pure 4 ms
|
|
10
|
+
shift or a -3 dB gain "damage"; the compensated one reports them as numbers and
|
|
11
|
+
leaves only what changed besides them (noise, a splice, a different waveform).
|
|
12
|
+
* Discontinuity: the largest sample-to-sample jump against a high percentile. A hard
|
|
13
|
+
splice or a click scores far above anything a clean edit produces. Judged against
|
|
14
|
+
the reference's own score, so a source that is naturally spiky is not flagged.
|
|
15
|
+
|
|
16
|
+
Pitch, formant and transient checks were source-dependent in the experiment (they
|
|
17
|
+
need a pitch range and a baseline per source), so they are not part of this module.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
# Compensated null: residual (dB below the reference) for an edit that should be transparent
|
|
27
|
+
NULL_OK_DB = -50.0
|
|
28
|
+
NULL_WARN_DB = -35.0
|
|
29
|
+
# Discontinuity: a score is a clear click from this value up, and relative to the reference's own score
|
|
30
|
+
CLICK_FAIL_SCORE = 8.0
|
|
31
|
+
CLICK_FAIL_RATIO = 2.5
|
|
32
|
+
CLICK_WARN_SCORE = 6.0
|
|
33
|
+
CLICK_WARN_RATIO = 2.0
|
|
34
|
+
|
|
35
|
+
SILENCE = 1e-6
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _mono(audio: np.ndarray) -> np.ndarray:
|
|
39
|
+
"""(channels, samples) or (samples,) to float64 mono."""
|
|
40
|
+
a = np.asarray(audio, dtype=np.float64)
|
|
41
|
+
return a if a.ndim == 1 else a.mean(axis=0)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _channels(audio: np.ndarray) -> np.ndarray:
|
|
45
|
+
"""(channels, samples) float64, whatever the input shape."""
|
|
46
|
+
a = np.asarray(audio, dtype=np.float64)
|
|
47
|
+
return a[None, :] if a.ndim == 1 else a
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _db(x: float, floor: float = -150.0) -> float:
|
|
51
|
+
return float(max(floor, 10.0 * np.log10(max(x, 1e-15))))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _fractional_delay(x: np.ndarray, lag: float) -> np.ndarray:
|
|
55
|
+
"""Delay x by `lag` samples (negative = advance) with an FFT phase ramp."""
|
|
56
|
+
spectrum = np.fft.rfft(x)
|
|
57
|
+
freqs = np.fft.rfftfreq(len(x))
|
|
58
|
+
return np.fft.irfft(spectrum * np.exp(-2j * np.pi * freqs * lag), len(x))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def compensated_null(reference: np.ndarray, edited: np.ndarray, sample_rate: int,
|
|
62
|
+
max_lag_ms: float = 20.0) -> dict[str, Any]:
|
|
63
|
+
"""
|
|
64
|
+
Delay (ms; positive = the edit is later), gain (dB), polarity and the residual
|
|
65
|
+
(dB relative to the reference) once those are removed. The inputs are cut to the
|
|
66
|
+
shorter one's length; the lag margin at both ends is left out of the residual.
|
|
67
|
+
"""
|
|
68
|
+
ref_ch, edit_ch = _channels(reference), _channels(edited)
|
|
69
|
+
ref, edit = _mono(reference), _mono(edited)
|
|
70
|
+
n = min(len(ref), len(edit))
|
|
71
|
+
ref, edit = ref[:n], edit[:n]
|
|
72
|
+
# Delay and gain come from the mono sum. When both have the same channels the residual is measured on each,
|
|
73
|
+
# so swapping left and right (or any change only to the stereo image) is not hidden by the sum.
|
|
74
|
+
per_channel = ref_ch.shape[0] == edit_ch.shape[0] and ref_ch.shape[0] > 1
|
|
75
|
+
if n < int(0.1 * sample_rate):
|
|
76
|
+
return {"error": "Audio too short: need at least 0.1 s"}
|
|
77
|
+
if np.max(np.abs(ref_ch)) < SILENCE:
|
|
78
|
+
return {"error": "The reference is silent over this range"}
|
|
79
|
+
if np.max(np.abs(edit_ch)) < SILENCE:
|
|
80
|
+
return {"error": "The edited audio is silent over this range"}
|
|
81
|
+
|
|
82
|
+
lag_max = max(1, int(sample_rate * max_lag_ms / 1000.0))
|
|
83
|
+
size = 1 << (2 * n - 1).bit_length()
|
|
84
|
+
corr = np.fft.irfft(np.fft.rfft(edit, size) * np.conj(np.fft.rfft(ref, size)), size)
|
|
85
|
+
window = np.concatenate([corr[-lag_max:], corr[: lag_max + 1]]) # lags -lag_max..+lag_max
|
|
86
|
+
peak = int(np.argmax(np.abs(window)))
|
|
87
|
+
lag = float(peak - lag_max)
|
|
88
|
+
if 0 < peak < len(window) - 1: # parabolic refinement of the correlation peak
|
|
89
|
+
a, b, c = np.abs(window[peak - 1]), np.abs(window[peak]), np.abs(window[peak + 1])
|
|
90
|
+
denom = a - 2 * b + c
|
|
91
|
+
if abs(denom) > 1e-12:
|
|
92
|
+
lag += 0.5 * (a - c) / denom
|
|
93
|
+
|
|
94
|
+
aligned = _fractional_delay(edit, -lag) # undo the delay: bring the edit back onto the reference
|
|
95
|
+
gain = float(np.dot(ref, aligned) / (np.dot(aligned, aligned) + 1e-15))
|
|
96
|
+
margin = lag_max * 2
|
|
97
|
+
span = slice(margin, n - margin) if n > 4 * margin else slice(0, n)
|
|
98
|
+
if per_channel:
|
|
99
|
+
pairs = [(r[:n], _fractional_delay(e[:n], -lag)) for r, e in zip(ref_ch, edit_ch)]
|
|
100
|
+
residual_power = float(np.mean([np.mean((r[span] - gain * a[span]) ** 2) for r, a in pairs]))
|
|
101
|
+
ref_power = float(np.mean([np.mean(r[span] ** 2) for r, _ in pairs])) + 1e-15
|
|
102
|
+
else:
|
|
103
|
+
residual_power = float(np.mean((ref - gain * aligned)[span] ** 2))
|
|
104
|
+
ref_power = float(np.mean(ref[span] ** 2)) + 1e-15
|
|
105
|
+
return {
|
|
106
|
+
"delayMs": round(1000.0 * lag / sample_rate, 3),
|
|
107
|
+
# gain that maps the edit back to the reference; the edit is 1/gain times as loud
|
|
108
|
+
"gainDb": round(-20.0 * np.log10(abs(gain) + 1e-15), 2),
|
|
109
|
+
"polarityInverted": bool(gain < 0),
|
|
110
|
+
"residualDb": round(_db(residual_power / ref_power), 1),
|
|
111
|
+
# The residual is relative to this: with a quiet reference, rounding noise of a 16-bit file can set it
|
|
112
|
+
"referenceRmsDb": round(_db(ref_power), 1),
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def discontinuity_score(audio: np.ndarray, edge_samples: int = 0) -> dict[str, Any]:
|
|
117
|
+
"""
|
|
118
|
+
Largest second difference against its 99.5th percentile, and where it happens.
|
|
119
|
+
The first and last `edge_samples` are left out: the ends of a rendered window are
|
|
120
|
+
not edit points (a delayed copy starts with a step there that is not a click).
|
|
121
|
+
"""
|
|
122
|
+
x = _mono(audio)
|
|
123
|
+
if len(x) < 16 or np.max(np.abs(x)) < SILENCE:
|
|
124
|
+
return {"score": None, "atSample": None}
|
|
125
|
+
d2 = np.abs(np.diff(x, 2))
|
|
126
|
+
offset = edge_samples if len(d2) > 4 * edge_samples + 16 else 0
|
|
127
|
+
inner = d2[offset: len(d2) - offset]
|
|
128
|
+
scale = float(np.percentile(inner, 99.5))
|
|
129
|
+
if scale < 1e-12: # a pure DC or a perfectly smooth signal: any jump is an outlier
|
|
130
|
+
scale = 1e-12
|
|
131
|
+
k = int(np.argmax(inner))
|
|
132
|
+
return {"score": round(float(inner[k] / scale), 2), "atSample": offset + k + 1}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
_VERDICT_RANK = {"unknown": 0, "ok": 1, "warn": 2, "fail": 3}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _worst_channel(reference: np.ndarray, edited: np.ndarray, edge_samples: int) -> tuple[dict, dict, int | None]:
|
|
139
|
+
"""
|
|
140
|
+
Discontinuity of the channel that scores highest in the edit, with the reference's score for the same channel.
|
|
141
|
+
A click on one channel, or opposite clicks on left and right, would shrink or cancel in a mono sum.
|
|
142
|
+
Channel counts that differ are compared as mono sums.
|
|
143
|
+
"""
|
|
144
|
+
ref_ch, edit_ch = _channels(reference), _channels(edited)
|
|
145
|
+
if ref_ch.shape[0] != edit_ch.shape[0]:
|
|
146
|
+
ref_ch, edit_ch = _mono(reference)[None, :], _mono(edited)[None, :]
|
|
147
|
+
best: tuple[dict, dict, int | None] | None = None
|
|
148
|
+
best_key: tuple[int, float] | None = None
|
|
149
|
+
for index, (ref, edit) in enumerate(zip(ref_ch, edit_ch)):
|
|
150
|
+
edit_disc, ref_disc = discontinuity_score(edit, edge_samples), discontinuity_score(ref, edge_samples)
|
|
151
|
+
if edit_disc["score"] is None:
|
|
152
|
+
continue
|
|
153
|
+
# Worst verdict first, then the biggest jump over the channel's own reference: a naturally spiky,
|
|
154
|
+
# unchanged channel must not hide a click on the other one.
|
|
155
|
+
rank = _VERDICT_RANK[_click_verdict(edit_disc["score"], ref_disc["score"])]
|
|
156
|
+
ratio = edit_disc["score"] / max(ref_disc["score"] or 1.0, 1.0)
|
|
157
|
+
if best_key is None or (rank, ratio) > best_key:
|
|
158
|
+
best, best_key = (edit_disc, ref_disc, index if len(edit_ch) > 1 else None), (rank, ratio)
|
|
159
|
+
return best or (discontinuity_score(edit_ch[0]), discontinuity_score(ref_ch[0]), None)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _click_verdict(edit_score: float | None, ref_score: float | None) -> str:
|
|
163
|
+
if edit_score is None:
|
|
164
|
+
return "unknown"
|
|
165
|
+
ratio = edit_score / max(ref_score or 1.0, 1.0)
|
|
166
|
+
if edit_score >= CLICK_FAIL_SCORE and ratio >= CLICK_FAIL_RATIO:
|
|
167
|
+
return "fail"
|
|
168
|
+
if edit_score >= CLICK_WARN_SCORE and ratio >= CLICK_WARN_RATIO:
|
|
169
|
+
return "warn"
|
|
170
|
+
return "ok"
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _null_verdict(residual_db: float, expect: str) -> str:
|
|
174
|
+
if expect != "identical":
|
|
175
|
+
return "info"
|
|
176
|
+
if residual_db <= NULL_OK_DB:
|
|
177
|
+
return "ok"
|
|
178
|
+
return "warn" if residual_db <= NULL_WARN_DB else "fail"
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def verify_edit(reference: np.ndarray, edited: np.ndarray, sample_rate: int, *,
|
|
182
|
+
expect: str = "identical", time_scale: float = 1.0,
|
|
183
|
+
max_lag_ms: float = 20.0) -> dict[str, Any]:
|
|
184
|
+
"""
|
|
185
|
+
Run the checks. `expect` is "identical" (the edit should leave the sound alone apart
|
|
186
|
+
from level, delay and polarity: alignment, gain, a split, a crossfade) or "changed"
|
|
187
|
+
(EQ, compression, stretch: the null residual is reported without a verdict).
|
|
188
|
+
`time_scale` != 1 means the edit changed the timing (a stretch), so the two cannot
|
|
189
|
+
be nulled against each other and only the discontinuity check runs.
|
|
190
|
+
"""
|
|
191
|
+
if expect not in ("identical", "changed"):
|
|
192
|
+
return {"error": f"expect must be 'identical' or 'changed', got {expect!r}"}
|
|
193
|
+
ref_len, edit_len = np.asarray(reference).shape[-1], np.asarray(edited).shape[-1]
|
|
194
|
+
if min(ref_len, edit_len) < int(0.1 * sample_rate):
|
|
195
|
+
return {"error": "Audio too short: need at least 0.1 s"}
|
|
196
|
+
for name, audio in (("reference", reference), ("edited audio", edited)):
|
|
197
|
+
if np.max(np.abs(audio)) < SILENCE:
|
|
198
|
+
return {"error": f"The {name} is silent over this range"}
|
|
199
|
+
if abs(time_scale - 1.0) <= 1e-6 and abs(ref_len - edit_len) > sample_rate * max_lag_ms / 1000.0:
|
|
200
|
+
return {"error": f"The reference is {ref_len / sample_rate:.2f} s and the edited audio {edit_len / sample_rate:.2f} s: "
|
|
201
|
+
"they must cover the same range (cut the reference file to the window, or use referenceTrack)"}
|
|
202
|
+
checks: dict[str, Any] = {}
|
|
203
|
+
|
|
204
|
+
edge = int(sample_rate * max_lag_ms / 1000.0)
|
|
205
|
+
edit_disc, ref_disc, channel = _worst_channel(reference, edited, edge)
|
|
206
|
+
checks["discontinuity"] = {
|
|
207
|
+
"verdict": _click_verdict(edit_disc["score"], ref_disc["score"]),
|
|
208
|
+
"score": edit_disc["score"],
|
|
209
|
+
"referenceScore": ref_disc["score"],
|
|
210
|
+
"atSeconds": None if edit_disc["atSample"] is None else round(edit_disc["atSample"] / sample_rate, 3),
|
|
211
|
+
**({"channel": channel} if channel is not None else {}),
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
if abs(time_scale - 1.0) > 1e-6:
|
|
215
|
+
checks["null"] = {"verdict": "skipped", "reason": "timing changed (timeScale != 1): a waveform null needs the same timing"}
|
|
216
|
+
else:
|
|
217
|
+
null = compensated_null(reference, edited, sample_rate, max_lag_ms)
|
|
218
|
+
if "error" in null:
|
|
219
|
+
return null
|
|
220
|
+
checks["null"] = {"verdict": _null_verdict(null["residualDb"], expect), **null}
|
|
221
|
+
|
|
222
|
+
verdicts = [c["verdict"] for c in checks.values()]
|
|
223
|
+
overall = "fail" if "fail" in verdicts else "warn" if "warn" in verdicts else "ok"
|
|
224
|
+
return {"verdict": overall, "expect": expect, "sampleRate": sample_rate, "checks": checks}
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
"""Offline GM SoundFont rendering through an optional native FluidSynth library."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ctypes
|
|
6
|
+
import ctypes.util
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import shutil
|
|
11
|
+
from contextlib import ExitStack, contextmanager
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Iterator
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import soundfile as sf
|
|
17
|
+
|
|
18
|
+
SAMPLE_RATE = 22050
|
|
19
|
+
BLOCK_SIZE = 4096
|
|
20
|
+
NOT_INSTALLED = (
|
|
21
|
+
"FluidSynth native library is not installed or could not be loaded. "
|
|
22
|
+
"Install it with 'brew install fluid-synth' on macOS or "
|
|
23
|
+
"'sudo apt install libfluidsynth3' on Debian/Ubuntu. "
|
|
24
|
+
"On Windows, install FluidSynth and add its DLL directory to PATH. "
|
|
25
|
+
"Supply your own local GM .sf2 via soundfontPath; no SoundFont is downloaded."
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _number(value: Any, field: str, low: float, high: float, *, integer: bool = False) -> float:
|
|
30
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
31
|
+
raise ValueError(f"{field} must be a finite number")
|
|
32
|
+
if not low <= value <= high or not math.isfinite(value):
|
|
33
|
+
raise ValueError(f"{field} must be finite and between {low} and {high}")
|
|
34
|
+
if integer and value != int(value):
|
|
35
|
+
raise ValueError(f"{field} must be an integer")
|
|
36
|
+
return value
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _validate_score(params: dict[str, Any]) -> None:
|
|
40
|
+
duration = _number(params["durationSeconds"], "durationSeconds", 0, 240)
|
|
41
|
+
if duration <= 0:
|
|
42
|
+
raise ValueError("durationSeconds must be greater than zero")
|
|
43
|
+
tracks = params["tracks"]
|
|
44
|
+
if not isinstance(tracks, list) or not 1 <= len(tracks) <= 8:
|
|
45
|
+
raise ValueError("tracks must contain 1 to 8 tracks")
|
|
46
|
+
names: set[str] = set()
|
|
47
|
+
total_notes = 0
|
|
48
|
+
for track in tracks:
|
|
49
|
+
if not isinstance(track, dict):
|
|
50
|
+
raise ValueError("tracks must contain objects")
|
|
51
|
+
name = track.get("name")
|
|
52
|
+
if not isinstance(name, str) or not re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", name) or name == "mix":
|
|
53
|
+
raise ValueError("track name must be a lowercase filename slug, not 'mix'")
|
|
54
|
+
if name in names:
|
|
55
|
+
raise ValueError("track name must be unique")
|
|
56
|
+
names.add(name)
|
|
57
|
+
_number(track.get("channel"), "channel", 0, 15, integer=True)
|
|
58
|
+
_number(track.get("program"), "program", 0, 127, integer=True)
|
|
59
|
+
_number(track.get("gain"), "gain", 0, 1)
|
|
60
|
+
notes = track.get("notes")
|
|
61
|
+
if not isinstance(notes, list):
|
|
62
|
+
raise ValueError("notes must be an array")
|
|
63
|
+
total_notes += len(notes)
|
|
64
|
+
if total_notes > 20000:
|
|
65
|
+
raise ValueError("notes must total at most 20000 across all tracks")
|
|
66
|
+
for note in notes:
|
|
67
|
+
_validate_note(note, duration)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _validate_note(note: Any, duration: float) -> None:
|
|
71
|
+
if not isinstance(note, dict):
|
|
72
|
+
raise ValueError("notes must contain objects")
|
|
73
|
+
start = _number(note.get("time"), "time", 0, duration)
|
|
74
|
+
length = _number(note.get("duration"), "duration", 0, duration)
|
|
75
|
+
if start >= duration:
|
|
76
|
+
raise ValueError("note time must be before durationSeconds")
|
|
77
|
+
if length <= 0 or start + length > duration + 1e-9:
|
|
78
|
+
raise ValueError("note duration must be positive and end within durationSeconds")
|
|
79
|
+
_number(note.get("pitch"), "pitch", 0, 127, integer=True)
|
|
80
|
+
_number(note.get("velocity"), "velocity", 1, 127, integer=True)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _absolute_path(value: Any, field: str) -> Path:
|
|
84
|
+
if not isinstance(value, str) or not value or "\0" in value:
|
|
85
|
+
raise ValueError(f"{field} must be an absolute path without NUL bytes")
|
|
86
|
+
path = Path(value)
|
|
87
|
+
if not path.is_absolute() or ".." in path.parts:
|
|
88
|
+
raise ValueError(f"{field} must be an absolute path without '..'")
|
|
89
|
+
return path
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _validate_paths(params: dict[str, Any]) -> tuple[Path, Path]:
|
|
93
|
+
font = _absolute_path(params["soundfontPath"], "soundfontPath")
|
|
94
|
+
if font.suffix.lower() != ".sf2" or not font.is_file():
|
|
95
|
+
raise ValueError("soundfontPath must point to an existing .sf2 file")
|
|
96
|
+
out = _absolute_path(params["outDir"], "outDir")
|
|
97
|
+
if os.path.lexists(out):
|
|
98
|
+
raise ValueError("outDir already exists; use a new directory")
|
|
99
|
+
if not out.parent.is_dir():
|
|
100
|
+
raise ValueError("outDir parent directory must already exist")
|
|
101
|
+
return font, out
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _native() -> Any:
|
|
105
|
+
candidates = [ctypes.util.find_library("fluidsynth"), "libfluidsynth.so.3", "libfluidsynth.dylib",
|
|
106
|
+
"/opt/homebrew/lib/libfluidsynth.dylib", "/usr/local/lib/libfluidsynth.dylib",
|
|
107
|
+
"libfluidsynth-3.dll", "libfluidsynth.dll"]
|
|
108
|
+
for candidate in candidates:
|
|
109
|
+
if not candidate:
|
|
110
|
+
continue
|
|
111
|
+
try:
|
|
112
|
+
lib = ctypes.CDLL(candidate)
|
|
113
|
+
break
|
|
114
|
+
except OSError:
|
|
115
|
+
continue
|
|
116
|
+
else:
|
|
117
|
+
raise RuntimeError(NOT_INSTALLED)
|
|
118
|
+
ptr, integer, string = ctypes.c_void_p, ctypes.c_int, ctypes.c_char_p
|
|
119
|
+
signatures = {
|
|
120
|
+
"new_fluid_settings": (ptr, []),
|
|
121
|
+
"delete_fluid_settings": (None, [ptr]),
|
|
122
|
+
"fluid_settings_setnum": (integer, [ptr, string, ctypes.c_double]),
|
|
123
|
+
"fluid_settings_setint": (integer, [ptr, string, integer]),
|
|
124
|
+
"new_fluid_synth": (ptr, [ptr]),
|
|
125
|
+
"delete_fluid_synth": (None, [ptr]),
|
|
126
|
+
"fluid_synth_sfload": (integer, [ptr, string, integer]),
|
|
127
|
+
"fluid_synth_program_select": (integer, [ptr, integer, integer, integer, integer]),
|
|
128
|
+
"fluid_synth_noteon": (integer, [ptr, integer, integer, integer]),
|
|
129
|
+
"fluid_synth_noteoff": (integer, [ptr, integer, integer]),
|
|
130
|
+
"fluid_synth_write_float": (integer, [ptr, integer, ptr, integer, integer, ptr, integer, integer]),
|
|
131
|
+
}
|
|
132
|
+
try:
|
|
133
|
+
for name, (result, args) in signatures.items():
|
|
134
|
+
function = getattr(lib, name)
|
|
135
|
+
function.restype = result
|
|
136
|
+
function.argtypes = args
|
|
137
|
+
except AttributeError as exc:
|
|
138
|
+
raise RuntimeError(f"{NOT_INSTALLED} The library lacks a required FluidSynth function.") from exc
|
|
139
|
+
return lib
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@contextmanager
|
|
143
|
+
def _synth(lib: Any, font: Path, track: dict[str, Any]) -> Iterator[Any]:
|
|
144
|
+
settings = lib.new_fluid_settings()
|
|
145
|
+
if not settings:
|
|
146
|
+
raise RuntimeError("FluidSynth could not allocate settings")
|
|
147
|
+
synth = None
|
|
148
|
+
try:
|
|
149
|
+
if lib.fluid_settings_setnum(settings, b"synth.sample-rate", SAMPLE_RATE) != 0:
|
|
150
|
+
raise RuntimeError("FluidSynth could not set its sample rate")
|
|
151
|
+
lib.fluid_settings_setnum(settings, b"synth.gain", 0.2)
|
|
152
|
+
lib.fluid_settings_setint(settings, b"synth.reverb.active", 0)
|
|
153
|
+
lib.fluid_settings_setint(settings, b"synth.chorus.active", 0)
|
|
154
|
+
synth = lib.new_fluid_synth(settings)
|
|
155
|
+
if not synth:
|
|
156
|
+
raise RuntimeError("FluidSynth could not allocate an offline synthesizer")
|
|
157
|
+
font_id = lib.fluid_synth_sfload(synth, os.fsencode(font), 1)
|
|
158
|
+
if font_id < 0:
|
|
159
|
+
raise RuntimeError("FluidSynth could not load soundfontPath as a SoundFont")
|
|
160
|
+
channel, program = int(track["channel"]), int(track["program"])
|
|
161
|
+
bank = 128 if channel == 9 else 0
|
|
162
|
+
if lib.fluid_synth_program_select(synth, channel, font_id, bank, program) != 0:
|
|
163
|
+
raise RuntimeError(f"SoundFont has no bank {bank} program {program} for track '{track['name']}'")
|
|
164
|
+
# No audio or MIDI driver is created. Samples only leave through write_float.
|
|
165
|
+
yield synth
|
|
166
|
+
finally:
|
|
167
|
+
if synth:
|
|
168
|
+
lib.delete_fluid_synth(synth)
|
|
169
|
+
lib.delete_fluid_settings(settings)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _events(notes: list[dict[str, Any]], frames: int) -> list[tuple[int, int, int, int, int]]:
|
|
173
|
+
events = []
|
|
174
|
+
for index, note in enumerate(notes):
|
|
175
|
+
start = min(frames - 1, round(note["time"] * SAMPLE_RATE))
|
|
176
|
+
end = min(frames, max(start + 1, round((note["time"] + note["duration"]) * SAMPLE_RATE)))
|
|
177
|
+
pitch = int(note["pitch"])
|
|
178
|
+
# Offs sort before ons at an identical sample, including adjacent repeats.
|
|
179
|
+
events.append((start, 1, index, pitch, int(note["velocity"])))
|
|
180
|
+
events.append((end, 0, index, pitch, 0))
|
|
181
|
+
return sorted(events)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _write_samples(lib: Any, synth: Any, output: sf.SoundFile, frames: int, gain: float) -> None:
|
|
185
|
+
while frames:
|
|
186
|
+
count = min(BLOCK_SIZE, frames)
|
|
187
|
+
block = np.empty((count, 2), dtype=np.float32)
|
|
188
|
+
if lib.fluid_synth_write_float(synth, count, block.ctypes.data, 0, 2, block.ctypes.data, 1, 2) != 0:
|
|
189
|
+
raise RuntimeError("FluidSynth failed while rendering audio samples")
|
|
190
|
+
block *= gain
|
|
191
|
+
if not np.isfinite(block).all():
|
|
192
|
+
raise RuntimeError("FluidSynth rendered non-finite audio samples")
|
|
193
|
+
output.write(block)
|
|
194
|
+
frames -= count
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _write_track(lib: Any, font: Path, path: Path, track: dict[str, Any], frames: int) -> None:
|
|
198
|
+
channel = int(track["channel"])
|
|
199
|
+
with _synth(lib, font, track) as synth, sf.SoundFile(
|
|
200
|
+
path, "w", samplerate=SAMPLE_RATE, channels=2, format="WAV", subtype="FLOAT",
|
|
201
|
+
) as output:
|
|
202
|
+
position = 0
|
|
203
|
+
active: dict[int, int] = {}
|
|
204
|
+
for frame, on, index, pitch, velocity in _events(track["notes"], frames):
|
|
205
|
+
_write_samples(lib, synth, output, frame - position, track["gain"])
|
|
206
|
+
position = frame
|
|
207
|
+
if on:
|
|
208
|
+
if lib.fluid_synth_noteon(synth, channel, pitch, velocity) != 0:
|
|
209
|
+
raise RuntimeError(f"FluidSynth could not play pitch {pitch} on track '{track['name']}'")
|
|
210
|
+
active[pitch] = index
|
|
211
|
+
elif active.get(pitch) == index:
|
|
212
|
+
# An older, overlapping repeat must not turn off the newer note.
|
|
213
|
+
lib.fluid_synth_noteoff(synth, channel, pitch)
|
|
214
|
+
del active[pitch]
|
|
215
|
+
_write_samples(lib, synth, output, frames - position, track["gain"])
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _mixed_blocks(paths: list[Path]) -> Iterator[tuple[np.ndarray, float]]:
|
|
219
|
+
with ExitStack() as stack:
|
|
220
|
+
sources = [stack.enter_context(sf.SoundFile(path)) for path in paths]
|
|
221
|
+
frames = len(sources[0])
|
|
222
|
+
for start in range(0, frames, BLOCK_SIZE):
|
|
223
|
+
mixed = np.zeros((min(BLOCK_SIZE, frames - start), 2), dtype=np.float32)
|
|
224
|
+
stem_peak = 0.0
|
|
225
|
+
for source in sources:
|
|
226
|
+
block = source.read(len(mixed), dtype="float32", always_2d=True)
|
|
227
|
+
stem_peak = max(stem_peak, float(np.max(np.abs(block))))
|
|
228
|
+
mixed += block
|
|
229
|
+
yield mixed, stem_peak
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _apply_gain(path: Path, gain: float) -> None:
|
|
233
|
+
if gain == 1:
|
|
234
|
+
return
|
|
235
|
+
with sf.SoundFile(path, "r+") as source:
|
|
236
|
+
for start in range(0, len(source), BLOCK_SIZE):
|
|
237
|
+
source.seek(start)
|
|
238
|
+
block = source.read(BLOCK_SIZE, dtype="float32", always_2d=True)
|
|
239
|
+
block *= gain
|
|
240
|
+
source.seek(start)
|
|
241
|
+
source.write(block)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _finish_mix(paths: list[Path], mix_path: Path) -> tuple[float, float]:
|
|
245
|
+
peak = 0.0
|
|
246
|
+
for block, stem_peak in _mixed_blocks(paths):
|
|
247
|
+
peak = max(peak, stem_peak, float(np.max(np.abs(block))))
|
|
248
|
+
# Cover float32 rounding across all eight stems, including cancellation.
|
|
249
|
+
gain = min(1.0, 0.9 * (1 - 1e-5) / peak) if peak else 1.0
|
|
250
|
+
for path in paths:
|
|
251
|
+
_apply_gain(path, gain)
|
|
252
|
+
mix_peak = 0.0
|
|
253
|
+
with sf.SoundFile(mix_path, "w", samplerate=SAMPLE_RATE, channels=2, format="WAV", subtype="FLOAT") as output:
|
|
254
|
+
for block, _ in _mixed_blocks(paths):
|
|
255
|
+
output.write(block)
|
|
256
|
+
mix_peak = max(mix_peak, float(np.max(np.abs(block))))
|
|
257
|
+
return gain, mix_peak
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def render(params: dict[str, Any]) -> dict[str, Any]:
|
|
261
|
+
"""Render isolated GM stems with common attenuation and a sample-exact summed mix.
|
|
262
|
+
|
|
263
|
+
Memory is independent of duration and stem count, apart from the bounded note
|
|
264
|
+
events and one native synth/SoundFont. Audio is streamed in 4096-frame blocks.
|
|
265
|
+
``durationSeconds`` includes release tails; all files have the same length,
|
|
266
|
+
rounded up to the next sample. Silence has ``peakDb: None`` for JSON safety.
|
|
267
|
+
"""
|
|
268
|
+
if not isinstance(params, dict):
|
|
269
|
+
raise ValueError("render params must be an object")
|
|
270
|
+
if not {"soundfontPath", "outDir", "durationSeconds", "tracks"} <= params.keys():
|
|
271
|
+
raise ValueError("soundfontPath, outDir, durationSeconds and tracks are required")
|
|
272
|
+
_validate_score(params)
|
|
273
|
+
font, out = _validate_paths(params)
|
|
274
|
+
lib = _native()
|
|
275
|
+
frames = math.ceil(params["durationSeconds"] * SAMPLE_RATE)
|
|
276
|
+
# mkdir is atomic. If another caller wins the race, never clean up its folder.
|
|
277
|
+
try:
|
|
278
|
+
out.mkdir()
|
|
279
|
+
except FileExistsError as exc:
|
|
280
|
+
raise ValueError("outDir already exists; use a new directory") from exc
|
|
281
|
+
try:
|
|
282
|
+
paths = [out / f"{track['name']}.wav" for track in params["tracks"]]
|
|
283
|
+
for path, track in zip(paths, params["tracks"]):
|
|
284
|
+
_write_track(lib, font, path, track, frames)
|
|
285
|
+
gain, peak = _finish_mix(paths, out / "mix.wav")
|
|
286
|
+
return {
|
|
287
|
+
"sampleRate": SAMPLE_RATE, "durationSeconds": frames / SAMPLE_RATE,
|
|
288
|
+
"gain": gain, "peakDb": 20 * math.log10(peak) if peak else None,
|
|
289
|
+
"mixPath": str(out / "mix.wav"),
|
|
290
|
+
"stems": [{"name": track["name"], "path": str(path)} for track, path in zip(params["tracks"], paths)],
|
|
291
|
+
"engine": "FluidSynth", "soundfontPath": str(font),
|
|
292
|
+
}
|
|
293
|
+
except BaseException:
|
|
294
|
+
shutil.rmtree(out)
|
|
295
|
+
raise
|