davinci-resolve-mcp 2.60.0 → 2.61.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +125 -0
- package/README.md +2 -2
- package/docs/SKILL.md +38 -2
- package/docs/install.md +8 -0
- package/docs/reference/api-limitations.md +9 -1
- package/install.py +1 -1
- package/package.json +1 -1
- package/src/granular/common.py +1 -1
- package/src/server.py +179 -1
- package/src/utils/analysis_store.py +75 -1
- package/src/utils/api_truth.py +32 -0
- package/src/utils/deep_vision.py +1 -30
- package/src/utils/strata.py +632 -0
- package/src/utils/strata_analyzers.py +728 -0
- package/src/utils/strata_faces.py +347 -0
- package/src/utils/strata_queries.py +663 -0
- package/src/utils/strata_story.py +305 -0
- package/src/utils/timeline_brain_db.py +138 -1
- package/src/utils/timeline_versioning.py +19 -2
|
@@ -0,0 +1,728 @@
|
|
|
1
|
+
"""Perception-strata analyzers — local track writers (no cloud, no Resolve).
|
|
2
|
+
|
|
3
|
+
Each analyzer reads a clip's media file (via ffmpeg) and/or its existing
|
|
4
|
+
strata rows, computes timecoded tracks, and writes them through
|
|
5
|
+
src/utils/strata. All are optional-capability: they self-describe what they
|
|
6
|
+
need (ffmpeg, numpy, a transcript) and refuse honestly instead of degrading
|
|
7
|
+
silently.
|
|
8
|
+
|
|
9
|
+
Analyzers here are deliberately dependency-light: numpy + ffmpeg only.
|
|
10
|
+
Nothing imports librosa/mediapipe — heavier analyzers (face/blink) live
|
|
11
|
+
behind their own capability gate in strata_faces.py.
|
|
12
|
+
|
|
13
|
+
Compute functions are pure (arrays in, tracks out) so they are unit-testable
|
|
14
|
+
on synthetic signals; the run_* wrappers handle clip resolution, decoding,
|
|
15
|
+
and DB writes.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import logging
|
|
21
|
+
import math
|
|
22
|
+
import shutil
|
|
23
|
+
import subprocess
|
|
24
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
25
|
+
|
|
26
|
+
from src.utils import strata, timeline_brain_db
|
|
27
|
+
from src.utils.proc import safe_run
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger("resolve-mcp.strata-analyzers")
|
|
30
|
+
|
|
31
|
+
try: # numpy is present in most installs but is not a hard requirement.
|
|
32
|
+
import numpy as _np
|
|
33
|
+
except ImportError: # pragma: no cover - exercised on minimal installs
|
|
34
|
+
_np = None
|
|
35
|
+
|
|
36
|
+
PROSODY_SOURCE = "prosody_v1"
|
|
37
|
+
PROSODY_VERSION = "1.0.0"
|
|
38
|
+
BEATGRID_SOURCE = "beatgrid_v1"
|
|
39
|
+
BEATGRID_VERSION = "1.0.0"
|
|
40
|
+
MOTION_SOURCE = "motion_v1"
|
|
41
|
+
MOTION_VERSION = "1.0.0"
|
|
42
|
+
|
|
43
|
+
AUDIO_SAMPLE_RATE = 16000
|
|
44
|
+
FRAME_SECONDS = 0.040 # analysis window
|
|
45
|
+
HOP_SECONDS = 0.010 # 100 Hz curve rate
|
|
46
|
+
CURVE_RATE = 1.0 / HOP_SECONDS
|
|
47
|
+
|
|
48
|
+
# Word-gap thresholds (seconds). A gap shorter than PAUSE_MIN is articulation,
|
|
49
|
+
# not a pause; longer than PAUSE_MAX it is silence/room, not a beat the editor
|
|
50
|
+
# cuts on — still recorded, capped payload marks it.
|
|
51
|
+
PAUSE_MIN_SECONDS = 0.35
|
|
52
|
+
HESITATION_WORDS = {"uh", "um", "er", "erm", "uhm", "hmm", "mm", "mhm", "ah", "eh"}
|
|
53
|
+
|
|
54
|
+
PITCH_MIN_HZ = 60.0
|
|
55
|
+
PITCH_MAX_HZ = 400.0
|
|
56
|
+
|
|
57
|
+
MOTION_CURVE_RATE = 10.0 # Hz
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _ensure_tool_path() -> None:
|
|
61
|
+
"""GUI-launched processes get launchd's bare PATH; reuse media_analysis's
|
|
62
|
+
augmentation so shutil.which finds Homebrew/MacPorts ffmpeg."""
|
|
63
|
+
try:
|
|
64
|
+
from src.utils.media_analysis import _ensure_path_includes_standard_tool_dirs
|
|
65
|
+
|
|
66
|
+
_ensure_path_includes_standard_tool_dirs()
|
|
67
|
+
except Exception: # pragma: no cover - best effort
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def capabilities() -> Dict[str, Any]:
|
|
72
|
+
"""What the local analyzers can run on this machine.
|
|
73
|
+
|
|
74
|
+
Derived from the ANALYZERS registry — adding an analyzer there is the
|
|
75
|
+
single edit; this report and run_analyzers' default set follow.
|
|
76
|
+
"""
|
|
77
|
+
_ensure_tool_path()
|
|
78
|
+
ffmpeg = shutil.which("ffmpeg")
|
|
79
|
+
have = {"ffmpeg": bool(ffmpeg), "numpy": _np is not None}
|
|
80
|
+
analyzers: Dict[str, Any] = {}
|
|
81
|
+
for name, spec in ANALYZERS.items():
|
|
82
|
+
probe = spec.get("capability")
|
|
83
|
+
if probe is not None:
|
|
84
|
+
analyzers[name] = probe()
|
|
85
|
+
continue
|
|
86
|
+
requires = spec["requires"]
|
|
87
|
+
analyzers[name] = {
|
|
88
|
+
"available": all(have.get(req, False) for req in requires),
|
|
89
|
+
"requires": list(requires),
|
|
90
|
+
"writes": spec["writes"],
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
"ffmpeg": {"available": bool(ffmpeg), "path": ffmpeg},
|
|
94
|
+
"numpy": {"available": _np is not None},
|
|
95
|
+
"analyzers": analyzers,
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _face_capability() -> Dict[str, Any]:
|
|
100
|
+
# A broken optional face stack must never take strata_status/strata_run
|
|
101
|
+
# down with it — report unavailable instead.
|
|
102
|
+
try:
|
|
103
|
+
from src.utils import strata_faces
|
|
104
|
+
|
|
105
|
+
return strata_faces.capabilities()
|
|
106
|
+
except Exception as exc: # pragma: no cover - environment-dependent
|
|
107
|
+
return {"available": False, "error": f"face stack failed to load: {exc}"}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _require(*needs: str) -> Optional[Dict[str, Any]]:
|
|
111
|
+
if "numpy" in needs and _np is None:
|
|
112
|
+
return {"success": False, "error": "numpy is required for this analyzer", "missing": "numpy"}
|
|
113
|
+
if "ffmpeg" in needs:
|
|
114
|
+
_ensure_tool_path()
|
|
115
|
+
if not shutil.which("ffmpeg"):
|
|
116
|
+
return {"success": False, "error": "ffmpeg not found on PATH", "missing": "ffmpeg"}
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
# ── clip resolution ──────────────────────────────────────────────────────────
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _clip_row(project_root: str, clip_ref: Any) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
|
|
124
|
+
"""Resolve a clip ref to {clip_uuid, file_path, duration_seconds, fps};
|
|
125
|
+
analyzers read the media file, so it must exist."""
|
|
126
|
+
_conn, clip, err = strata.resolve_clip(project_root, clip_ref, require_media=True)
|
|
127
|
+
return clip, err
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# ── decoding ─────────────────────────────────────────────────────────────────
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def decode_audio(path: str, sample_rate: int = AUDIO_SAMPLE_RATE, timeout: int = 600) -> "Any":
|
|
134
|
+
"""Decode a media file's audio to mono float32 PCM via ffmpeg."""
|
|
135
|
+
cmd = [
|
|
136
|
+
shutil.which("ffmpeg") or "ffmpeg",
|
|
137
|
+
"-v", "error",
|
|
138
|
+
"-i", path,
|
|
139
|
+
"-map", "0:a:0",
|
|
140
|
+
"-ac", "1",
|
|
141
|
+
"-ar", str(sample_rate),
|
|
142
|
+
"-f", "f32le",
|
|
143
|
+
"-",
|
|
144
|
+
]
|
|
145
|
+
proc = safe_run(cmd, capture_output=True, timeout=timeout, check=False)
|
|
146
|
+
if proc.returncode != 0:
|
|
147
|
+
raise RuntimeError(f"ffmpeg audio decode failed: {proc.stderr.decode('utf-8', 'replace')[:500]}")
|
|
148
|
+
return _np.frombuffer(proc.stdout, dtype=_np.float32)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _audio_context(
|
|
152
|
+
project_root: str,
|
|
153
|
+
clip_ref: Any,
|
|
154
|
+
*,
|
|
155
|
+
clip: Optional[Dict[str, Any]] = None,
|
|
156
|
+
samples: "Any" = None,
|
|
157
|
+
) -> Tuple[Optional[Dict[str, Any]], "Any", Optional[Dict[str, Any]]]:
|
|
158
|
+
"""The shared audio-analyzer preamble: deps → clip row → decoded samples.
|
|
159
|
+
|
|
160
|
+
Returns (clip, samples, error). Pass a previously resolved ``clip`` and
|
|
161
|
+
decoded ``samples`` to skip the ffmpeg decode — run_analyzers uses this
|
|
162
|
+
so prosody and beat_grid share one decode of the same media file.
|
|
163
|
+
"""
|
|
164
|
+
missing = _require("ffmpeg", "numpy")
|
|
165
|
+
if missing:
|
|
166
|
+
return None, None, missing
|
|
167
|
+
if clip is None:
|
|
168
|
+
clip, err = _clip_row(project_root, clip_ref)
|
|
169
|
+
if err:
|
|
170
|
+
return None, None, err
|
|
171
|
+
if samples is None:
|
|
172
|
+
try:
|
|
173
|
+
samples = decode_audio(clip["file_path"])
|
|
174
|
+
except (RuntimeError, subprocess.TimeoutExpired) as exc:
|
|
175
|
+
return None, None, {"success": False, "error": str(exc), "clip_uuid": clip["clip_uuid"]}
|
|
176
|
+
if samples.size == 0:
|
|
177
|
+
return None, None, {
|
|
178
|
+
"success": False,
|
|
179
|
+
"error": "clip has no decodable audio",
|
|
180
|
+
"clip_uuid": clip["clip_uuid"],
|
|
181
|
+
}
|
|
182
|
+
return clip, samples, None
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ── prosody: pure compute ────────────────────────────────────────────────────
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _frame_signal(samples: "Any", sample_rate: int) -> Tuple["Any", int, int]:
|
|
189
|
+
frame_len = int(sample_rate * FRAME_SECONDS)
|
|
190
|
+
hop_len = int(sample_rate * HOP_SECONDS)
|
|
191
|
+
if len(samples) < frame_len:
|
|
192
|
+
return _np.empty((0, frame_len), dtype=_np.float32), frame_len, hop_len
|
|
193
|
+
n_frames = 1 + (len(samples) - frame_len) // hop_len
|
|
194
|
+
idx = _np.arange(frame_len)[None, :] + hop_len * _np.arange(n_frames)[:, None]
|
|
195
|
+
return samples[idx], frame_len, hop_len
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def compute_energy_curve(samples: "Any", sample_rate: int = AUDIO_SAMPLE_RATE) -> List[float]:
|
|
199
|
+
"""Per-frame RMS, normalized to the clip's 95th percentile (0..~1)."""
|
|
200
|
+
frames, _, _ = _frame_signal(samples, sample_rate)
|
|
201
|
+
if frames.shape[0] == 0:
|
|
202
|
+
return []
|
|
203
|
+
rms = _np.sqrt(_np.mean(frames.astype(_np.float64) ** 2, axis=1))
|
|
204
|
+
scale = float(_np.percentile(rms, 95)) or 1.0
|
|
205
|
+
if scale <= 0:
|
|
206
|
+
scale = 1.0
|
|
207
|
+
return [float(v) for v in _np.clip(rms / scale, 0.0, 4.0)]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def compute_pitch_curve(
|
|
211
|
+
samples: "Any",
|
|
212
|
+
sample_rate: int = AUDIO_SAMPLE_RATE,
|
|
213
|
+
energy: Optional[Sequence[float]] = None,
|
|
214
|
+
voiced_threshold: float = 0.10,
|
|
215
|
+
) -> List[float]:
|
|
216
|
+
"""Autocorrelation pitch (Hz) per frame; NaN where unvoiced/silent.
|
|
217
|
+
|
|
218
|
+
Deliberately simple (no pYIN): good enough to carry contour direction,
|
|
219
|
+
range, and tremor for take comparison — not for music transcription.
|
|
220
|
+
"""
|
|
221
|
+
frames, frame_len, _ = _frame_signal(samples, sample_rate)
|
|
222
|
+
if frames.shape[0] == 0:
|
|
223
|
+
return []
|
|
224
|
+
if energy is None:
|
|
225
|
+
energy = compute_energy_curve(samples, sample_rate)
|
|
226
|
+
lag_min = int(sample_rate / PITCH_MAX_HZ)
|
|
227
|
+
lag_max = min(int(sample_rate / PITCH_MIN_HZ), frame_len - 1)
|
|
228
|
+
out: List[float] = []
|
|
229
|
+
hann = _np.hanning(frame_len)
|
|
230
|
+
for i in range(frames.shape[0]):
|
|
231
|
+
if i >= len(energy) or energy[i] < voiced_threshold:
|
|
232
|
+
out.append(float("nan"))
|
|
233
|
+
continue
|
|
234
|
+
frame = frames[i].astype(_np.float64) * hann
|
|
235
|
+
frame = frame - frame.mean()
|
|
236
|
+
# FFT autocorrelation
|
|
237
|
+
spec = _np.fft.rfft(frame, n=2 * frame_len)
|
|
238
|
+
ac = _np.fft.irfft(spec * _np.conj(spec))[:frame_len]
|
|
239
|
+
if ac[0] <= 0:
|
|
240
|
+
out.append(float("nan"))
|
|
241
|
+
continue
|
|
242
|
+
ac = ac / ac[0]
|
|
243
|
+
window = ac[lag_min:lag_max]
|
|
244
|
+
if window.size == 0:
|
|
245
|
+
out.append(float("nan"))
|
|
246
|
+
continue
|
|
247
|
+
peak = int(_np.argmax(window)) + lag_min
|
|
248
|
+
# Voicing gate: the autocorrelation peak must be strong.
|
|
249
|
+
if ac[peak] < 0.30:
|
|
250
|
+
out.append(float("nan"))
|
|
251
|
+
continue
|
|
252
|
+
out.append(float(sample_rate / peak))
|
|
253
|
+
return out
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def compute_speech_rate_curve(
|
|
257
|
+
words: Sequence[Dict[str, Any]],
|
|
258
|
+
duration_seconds: float,
|
|
259
|
+
window_seconds: float = 2.0,
|
|
260
|
+
rate_hz: float = 10.0,
|
|
261
|
+
) -> List[float]:
|
|
262
|
+
"""Words/second in a centered sliding window, sampled at rate_hz."""
|
|
263
|
+
n = max(1, int(math.ceil(duration_seconds * rate_hz)))
|
|
264
|
+
starts = [w.get("start_seconds") for w in words if isinstance(w.get("start_seconds"), (int, float))]
|
|
265
|
+
out = []
|
|
266
|
+
half = window_seconds / 2.0
|
|
267
|
+
for i in range(n):
|
|
268
|
+
t = i / rate_hz
|
|
269
|
+
lo, hi = t - half, t + half
|
|
270
|
+
count = sum(1 for s in starts if lo <= s < hi)
|
|
271
|
+
out.append(count / window_seconds)
|
|
272
|
+
return out
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def detect_pauses(
|
|
276
|
+
words: Sequence[Dict[str, Any]],
|
|
277
|
+
min_gap_seconds: float = PAUSE_MIN_SECONDS,
|
|
278
|
+
) -> List[Dict[str, Any]]:
|
|
279
|
+
"""Word-gap pauses inside the spoken region. Gap = prev.end → next.start."""
|
|
280
|
+
timed = [
|
|
281
|
+
w for w in words
|
|
282
|
+
if isinstance(w.get("start_seconds"), (int, float)) and isinstance(w.get("end_seconds"), (int, float))
|
|
283
|
+
]
|
|
284
|
+
timed.sort(key=lambda w: (w["start_seconds"], w["end_seconds"]))
|
|
285
|
+
pauses = []
|
|
286
|
+
for prev, nxt in zip(timed, timed[1:]):
|
|
287
|
+
gap_start = float(prev["end_seconds"])
|
|
288
|
+
gap = float(nxt["start_seconds"]) - gap_start
|
|
289
|
+
if gap >= min_gap_seconds:
|
|
290
|
+
pauses.append(
|
|
291
|
+
{
|
|
292
|
+
"time_seconds": gap_start,
|
|
293
|
+
"duration_seconds": gap,
|
|
294
|
+
"payload": {
|
|
295
|
+
"before_word": prev.get("word"),
|
|
296
|
+
"after_word": nxt.get("word"),
|
|
297
|
+
},
|
|
298
|
+
}
|
|
299
|
+
)
|
|
300
|
+
return pauses
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def detect_hesitations(words: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
304
|
+
out = []
|
|
305
|
+
for w in words:
|
|
306
|
+
token = str(w.get("word") or "").strip().lower().strip(".,!?;:")
|
|
307
|
+
if token in HESITATION_WORDS and isinstance(w.get("start_seconds"), (int, float)):
|
|
308
|
+
end = w.get("end_seconds")
|
|
309
|
+
out.append(
|
|
310
|
+
{
|
|
311
|
+
"time_seconds": float(w["start_seconds"]),
|
|
312
|
+
"duration_seconds": (float(end) - float(w["start_seconds"])) if isinstance(end, (int, float)) else None,
|
|
313
|
+
"payload": {"word": w.get("word")},
|
|
314
|
+
}
|
|
315
|
+
)
|
|
316
|
+
return out
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def detect_breaths(
|
|
320
|
+
energy: Sequence[float],
|
|
321
|
+
pauses: Sequence[Dict[str, Any]],
|
|
322
|
+
curve_rate: float = CURVE_RATE,
|
|
323
|
+
) -> List[Dict[str, Any]]:
|
|
324
|
+
"""Breath candidates: a sub-speech energy bump inside a word gap.
|
|
325
|
+
|
|
326
|
+
Honest heuristic (flagged low-confidence): within each pause, look for a
|
|
327
|
+
local energy peak that clears the pause's own floor but stays well under
|
|
328
|
+
speech level. Catches on-mic inhales; misses quiet ones.
|
|
329
|
+
"""
|
|
330
|
+
if not len(energy):
|
|
331
|
+
return []
|
|
332
|
+
# numpy assumed: every caller sits behind _require("numpy"), matching the
|
|
333
|
+
# sibling compute functions in this module.
|
|
334
|
+
arr = _np.asarray(energy, dtype=_np.float64)
|
|
335
|
+
breaths: List[Dict[str, Any]] = []
|
|
336
|
+
for pause in pauses:
|
|
337
|
+
start = float(pause["time_seconds"])
|
|
338
|
+
dur = float(pause.get("duration_seconds") or 0.0)
|
|
339
|
+
lo = int(start * curve_rate)
|
|
340
|
+
hi = min(int((start + dur) * curve_rate), len(energy))
|
|
341
|
+
if hi - lo < 3:
|
|
342
|
+
continue
|
|
343
|
+
seg = arr[lo:hi]
|
|
344
|
+
floor = float(_np.percentile(seg, 20))
|
|
345
|
+
peak_idx = int(_np.argmax(seg))
|
|
346
|
+
peak = float(seg[peak_idx])
|
|
347
|
+
# Bump: clearly above the gap floor, clearly below speech (~1.0 scale).
|
|
348
|
+
if peak >= floor + 0.05 and peak <= 0.5:
|
|
349
|
+
breaths.append(
|
|
350
|
+
{
|
|
351
|
+
"time_seconds": (lo + peak_idx) / curve_rate,
|
|
352
|
+
"duration_seconds": None,
|
|
353
|
+
"payload": {"confidence": "low", "peak_energy": round(peak, 4)},
|
|
354
|
+
}
|
|
355
|
+
)
|
|
356
|
+
return breaths
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
# ── prosody: runner ──────────────────────────────────────────────────────────
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def run_prosody(
|
|
363
|
+
project_root: str,
|
|
364
|
+
clip_ref: Any,
|
|
365
|
+
*,
|
|
366
|
+
clip: Optional[Dict[str, Any]] = None,
|
|
367
|
+
samples: "Any" = None,
|
|
368
|
+
) -> Dict[str, Any]:
|
|
369
|
+
"""Compute + persist prosody strata for one clip.
|
|
370
|
+
|
|
371
|
+
Writes curves pitch / vocal_energy / speech_rate and events pause /
|
|
372
|
+
breath / hesitation. Requires the clip's media file, ffmpeg, numpy, and
|
|
373
|
+
(for word-derived tracks) transcript_words rows. ``clip``/``samples``
|
|
374
|
+
accept a pre-resolved row and pre-decoded audio (see _audio_context).
|
|
375
|
+
"""
|
|
376
|
+
clip, samples, err = _audio_context(project_root, clip_ref, clip=clip, samples=samples)
|
|
377
|
+
if err:
|
|
378
|
+
return err
|
|
379
|
+
conn = timeline_brain_db.connect(project_root)
|
|
380
|
+
words = strata.read_words(conn, clip["clip_uuid"])
|
|
381
|
+
|
|
382
|
+
duration = clip["duration_seconds"] or (samples.size / AUDIO_SAMPLE_RATE)
|
|
383
|
+
energy = compute_energy_curve(samples)
|
|
384
|
+
pitch = compute_pitch_curve(samples, energy=energy)
|
|
385
|
+
pauses = detect_pauses(words)
|
|
386
|
+
hesitations = detect_hesitations(words)
|
|
387
|
+
breaths = detect_breaths(energy, pauses)
|
|
388
|
+
speech_rate = compute_speech_rate_curve(words, float(duration)) if words else []
|
|
389
|
+
|
|
390
|
+
with timeline_brain_db.transaction(project_root) as txn:
|
|
391
|
+
strata.write_curve(
|
|
392
|
+
txn, clip["clip_uuid"], "vocal_energy", energy,
|
|
393
|
+
sample_rate=CURVE_RATE, source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
394
|
+
)
|
|
395
|
+
strata.write_curve(
|
|
396
|
+
txn, clip["clip_uuid"], "pitch", pitch,
|
|
397
|
+
sample_rate=CURVE_RATE, source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
398
|
+
)
|
|
399
|
+
if speech_rate:
|
|
400
|
+
strata.write_curve(
|
|
401
|
+
txn, clip["clip_uuid"], "speech_rate", speech_rate,
|
|
402
|
+
sample_rate=10.0, source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
403
|
+
)
|
|
404
|
+
strata.replace_track_events(
|
|
405
|
+
txn, clip["clip_uuid"], "pause", pauses,
|
|
406
|
+
source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
407
|
+
)
|
|
408
|
+
strata.replace_track_events(
|
|
409
|
+
txn, clip["clip_uuid"], "hesitation", hesitations,
|
|
410
|
+
source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
411
|
+
)
|
|
412
|
+
strata.replace_track_events(
|
|
413
|
+
txn, clip["clip_uuid"], "breath", breaths,
|
|
414
|
+
source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
return {
|
|
418
|
+
"success": True,
|
|
419
|
+
"clip_uuid": clip["clip_uuid"],
|
|
420
|
+
"clip_name": clip["clip_name"],
|
|
421
|
+
"curves": {
|
|
422
|
+
"vocal_energy": len(energy),
|
|
423
|
+
"pitch": len(pitch),
|
|
424
|
+
"speech_rate": len(speech_rate),
|
|
425
|
+
},
|
|
426
|
+
"events": {
|
|
427
|
+
"pause": len(pauses),
|
|
428
|
+
"hesitation": len(hesitations),
|
|
429
|
+
"breath": len(breaths),
|
|
430
|
+
},
|
|
431
|
+
"had_words": bool(words),
|
|
432
|
+
"analyzer_version": PROSODY_VERSION,
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
# ── musical beat grid ────────────────────────────────────────────────────────
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def compute_beat_grid(
|
|
440
|
+
samples: "Any",
|
|
441
|
+
sample_rate: int = AUDIO_SAMPLE_RATE,
|
|
442
|
+
tempo_min: float = 60.0,
|
|
443
|
+
tempo_max: float = 200.0,
|
|
444
|
+
) -> Dict[str, Any]:
|
|
445
|
+
"""Onset-envelope beat tracking: spectral flux → tempo via autocorrelation
|
|
446
|
+
→ beat placement by phase search. numpy-only; no ML.
|
|
447
|
+
|
|
448
|
+
Returns {tempo_bpm, beats: [seconds], downbeats: [seconds], confidence}.
|
|
449
|
+
Downbeats are a phase-of-strongest-onset estimate every 4 beats —
|
|
450
|
+
explicitly low-confidence (no meter detection).
|
|
451
|
+
"""
|
|
452
|
+
win = int(sample_rate * FRAME_SECONDS)
|
|
453
|
+
frames, _, _ = _frame_signal(samples, sample_rate)
|
|
454
|
+
if frames.shape[0] < 16:
|
|
455
|
+
return {"tempo_bpm": None, "beats": [], "downbeats": [], "confidence": 0.0}
|
|
456
|
+
hann = _np.hanning(win)
|
|
457
|
+
mags = _np.abs(_np.fft.rfft(frames * hann, axis=1))
|
|
458
|
+
flux = _np.diff(mags, axis=0)
|
|
459
|
+
flux[flux < 0] = 0.0
|
|
460
|
+
onset = flux.sum(axis=1)
|
|
461
|
+
if onset.max() <= 0:
|
|
462
|
+
return {"tempo_bpm": None, "beats": [], "downbeats": [], "confidence": 0.0}
|
|
463
|
+
onset = onset / onset.max()
|
|
464
|
+
onset = onset - onset.mean()
|
|
465
|
+
|
|
466
|
+
# Tempo: autocorrelation peak in the plausible lag range.
|
|
467
|
+
ac = _np.correlate(onset, onset, mode="full")[len(onset) - 1:]
|
|
468
|
+
if ac[0] <= 0:
|
|
469
|
+
return {"tempo_bpm": None, "beats": [], "downbeats": [], "confidence": 0.0}
|
|
470
|
+
ac = ac / ac[0]
|
|
471
|
+
frame_rate = 1.0 / HOP_SECONDS
|
|
472
|
+
lag_min = max(1, int(frame_rate * 60.0 / tempo_max))
|
|
473
|
+
lag_max = min(len(ac) - 1, int(frame_rate * 60.0 / tempo_min))
|
|
474
|
+
if lag_max <= lag_min:
|
|
475
|
+
return {"tempo_bpm": None, "beats": [], "downbeats": [], "confidence": 0.0}
|
|
476
|
+
lag = int(_np.argmax(ac[lag_min:lag_max])) + lag_min
|
|
477
|
+
tempo_bpm = 60.0 * frame_rate / lag
|
|
478
|
+
confidence = float(ac[lag])
|
|
479
|
+
|
|
480
|
+
# Phase: shift the beat comb to maximize onset energy under it.
|
|
481
|
+
best_phase, best_score = 0, -1.0
|
|
482
|
+
for phase in range(lag):
|
|
483
|
+
score = float(onset[phase::lag].sum())
|
|
484
|
+
if score > best_score:
|
|
485
|
+
best_score, best_phase = score, phase
|
|
486
|
+
beat_frames = list(range(best_phase, len(onset), lag))
|
|
487
|
+
beats = [(f + 1) * HOP_SECONDS for f in beat_frames] # +1: flux is a diff
|
|
488
|
+
|
|
489
|
+
# Downbeat guess: strongest onset among the first 4 beats sets bar phase.
|
|
490
|
+
downbeats = []
|
|
491
|
+
if len(beat_frames) >= 4:
|
|
492
|
+
first_bar = beat_frames[:4]
|
|
493
|
+
strongest = max(range(len(first_bar)), key=lambda i: onset[first_bar[i]])
|
|
494
|
+
downbeats = beats[strongest::4]
|
|
495
|
+
|
|
496
|
+
return {
|
|
497
|
+
"tempo_bpm": round(tempo_bpm, 2),
|
|
498
|
+
"beats": beats,
|
|
499
|
+
"downbeats": downbeats,
|
|
500
|
+
"confidence": round(confidence, 3),
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def run_beat_grid(
|
|
505
|
+
project_root: str,
|
|
506
|
+
clip_ref: Any,
|
|
507
|
+
*,
|
|
508
|
+
clip: Optional[Dict[str, Any]] = None,
|
|
509
|
+
samples: "Any" = None,
|
|
510
|
+
) -> Dict[str, Any]:
|
|
511
|
+
"""Compute + persist the musical beat grid for one clip.
|
|
512
|
+
|
|
513
|
+
``clip``/``samples`` accept a pre-resolved row and pre-decoded audio
|
|
514
|
+
(see _audio_context)."""
|
|
515
|
+
clip, samples, err = _audio_context(project_root, clip_ref, clip=clip, samples=samples)
|
|
516
|
+
if err:
|
|
517
|
+
return err
|
|
518
|
+
|
|
519
|
+
grid = compute_beat_grid(samples)
|
|
520
|
+
beat_events = [
|
|
521
|
+
{"time_seconds": t, "payload": {"tempo_bpm": grid["tempo_bpm"], "confidence": grid["confidence"]}}
|
|
522
|
+
for t in grid["beats"]
|
|
523
|
+
]
|
|
524
|
+
downbeat_events = [
|
|
525
|
+
{"time_seconds": t, "payload": {"confidence": "low"}} for t in grid["downbeats"]
|
|
526
|
+
]
|
|
527
|
+
with timeline_brain_db.transaction(project_root) as txn:
|
|
528
|
+
strata.replace_track_events(
|
|
529
|
+
txn, clip["clip_uuid"], "beat", beat_events,
|
|
530
|
+
source=BEATGRID_SOURCE, analyzer_version=BEATGRID_VERSION,
|
|
531
|
+
)
|
|
532
|
+
strata.replace_track_events(
|
|
533
|
+
txn, clip["clip_uuid"], "downbeat", downbeat_events,
|
|
534
|
+
source=BEATGRID_SOURCE, analyzer_version=BEATGRID_VERSION,
|
|
535
|
+
)
|
|
536
|
+
return {
|
|
537
|
+
"success": True,
|
|
538
|
+
"clip_uuid": clip["clip_uuid"],
|
|
539
|
+
"clip_name": clip["clip_name"],
|
|
540
|
+
"tempo_bpm": grid["tempo_bpm"],
|
|
541
|
+
"beat_count": len(beat_events),
|
|
542
|
+
"downbeat_count": len(downbeat_events),
|
|
543
|
+
"confidence": grid["confidence"],
|
|
544
|
+
"analyzer_version": BEATGRID_VERSION,
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
# ── motion energy ────────────────────────────────────────────────────────────
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def run_motion_energy(project_root: str, clip_ref: Any, timeout: int = 1800) -> Dict[str, Any]:
|
|
552
|
+
"""Per-frame luma difference (ffmpeg signalstats YDIF) → motion_energy curve.
|
|
553
|
+
|
|
554
|
+
Downsampled to MOTION_CURVE_RATE by mean-pooling so an hour of footage is
|
|
555
|
+
~140 KB. 0 = static, higher = more inter-frame change; normalized 0..1
|
|
556
|
+
against the clip's own 98th percentile.
|
|
557
|
+
"""
|
|
558
|
+
missing = _require("ffmpeg")
|
|
559
|
+
if missing:
|
|
560
|
+
return missing
|
|
561
|
+
clip, err = _clip_row(project_root, clip_ref)
|
|
562
|
+
if err:
|
|
563
|
+
return err
|
|
564
|
+
|
|
565
|
+
cmd = [
|
|
566
|
+
shutil.which("ffmpeg") or "ffmpeg",
|
|
567
|
+
"-v", "error",
|
|
568
|
+
"-i", clip["file_path"],
|
|
569
|
+
"-map", "0:v:0",
|
|
570
|
+
"-vf", "signalstats,metadata=print:key=lavfi.signalstats.YDIF:file=-",
|
|
571
|
+
"-f", "null", "-",
|
|
572
|
+
]
|
|
573
|
+
try:
|
|
574
|
+
proc = safe_run(cmd, capture_output=True, timeout=timeout, check=False)
|
|
575
|
+
except subprocess.TimeoutExpired:
|
|
576
|
+
return {"success": False, "error": "motion analysis timed out", "clip_uuid": clip["clip_uuid"]}
|
|
577
|
+
if proc.returncode != 0:
|
|
578
|
+
return {
|
|
579
|
+
"success": False,
|
|
580
|
+
"error": f"ffmpeg signalstats failed: {proc.stderr.decode('utf-8', 'replace')[:500]}",
|
|
581
|
+
"clip_uuid": clip["clip_uuid"],
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
times: List[float] = []
|
|
585
|
+
ydifs: List[float] = []
|
|
586
|
+
pending_time: Optional[float] = None
|
|
587
|
+
for line in proc.stdout.decode("utf-8", "replace").splitlines():
|
|
588
|
+
line = line.strip()
|
|
589
|
+
if line.startswith("frame:"):
|
|
590
|
+
# e.g. "frame:12 pts:6006 pts_time:0.25025"
|
|
591
|
+
for token in line.split():
|
|
592
|
+
if token.startswith("pts_time:"):
|
|
593
|
+
try:
|
|
594
|
+
pending_time = float(token.split(":", 1)[1])
|
|
595
|
+
except ValueError:
|
|
596
|
+
pending_time = None
|
|
597
|
+
elif line.startswith("lavfi.signalstats.YDIF=") and pending_time is not None:
|
|
598
|
+
try:
|
|
599
|
+
ydifs.append(float(line.split("=", 1)[1]))
|
|
600
|
+
times.append(pending_time)
|
|
601
|
+
except ValueError:
|
|
602
|
+
pass
|
|
603
|
+
pending_time = None
|
|
604
|
+
|
|
605
|
+
if not ydifs:
|
|
606
|
+
return {"success": False, "error": "no video frames analyzed", "clip_uuid": clip["clip_uuid"]}
|
|
607
|
+
|
|
608
|
+
duration = times[-1] if times else (clip["duration_seconds"] or 0.0)
|
|
609
|
+
n_out = max(1, int(math.ceil((duration or 1.0) * MOTION_CURVE_RATE)))
|
|
610
|
+
buckets: List[List[float]] = [[] for _ in range(n_out)]
|
|
611
|
+
for t, v in zip(times, ydifs):
|
|
612
|
+
idx = min(int(t * MOTION_CURVE_RATE), n_out - 1)
|
|
613
|
+
buckets[idx].append(v)
|
|
614
|
+
pooled = [sum(b) / len(b) if b else float("nan") for b in buckets]
|
|
615
|
+
finite = sorted(v for v in pooled if not math.isnan(v))
|
|
616
|
+
if finite:
|
|
617
|
+
scale = finite[min(len(finite) - 1, int(len(finite) * 0.98))] or 1.0
|
|
618
|
+
if scale <= 0:
|
|
619
|
+
scale = 1.0
|
|
620
|
+
pooled = [min(v / scale, 4.0) if not math.isnan(v) else v for v in pooled]
|
|
621
|
+
|
|
622
|
+
with timeline_brain_db.transaction(project_root) as txn:
|
|
623
|
+
strata.write_curve(
|
|
624
|
+
txn, clip["clip_uuid"], "motion_energy", pooled,
|
|
625
|
+
sample_rate=MOTION_CURVE_RATE, source=MOTION_SOURCE, analyzer_version=MOTION_VERSION,
|
|
626
|
+
)
|
|
627
|
+
return {
|
|
628
|
+
"success": True,
|
|
629
|
+
"clip_uuid": clip["clip_uuid"],
|
|
630
|
+
"clip_name": clip["clip_name"],
|
|
631
|
+
"frames_analyzed": len(ydifs),
|
|
632
|
+
"samples": len(pooled),
|
|
633
|
+
"analyzer_version": MOTION_VERSION,
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
# ── dispatcher ───────────────────────────────────────────────────────────────
|
|
638
|
+
|
|
639
|
+
def _run_face(project_root: str, clip_ref: Any) -> Dict[str, Any]:
|
|
640
|
+
from src.utils import strata_faces
|
|
641
|
+
|
|
642
|
+
return strata_faces.run_face_strata(project_root, clip_ref)
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
# The single analyzer registry: run function + dependency/track metadata.
|
|
646
|
+
# capabilities() and run_analyzers' default set derive from it, so a new
|
|
647
|
+
# analyzer is one entry here (or a "capability" probe for stacks that
|
|
648
|
+
# self-describe, like face). "audio" marks analyzers that consume the shared
|
|
649
|
+
# mono PCM decode.
|
|
650
|
+
ANALYZERS: Dict[str, Dict[str, Any]] = {
|
|
651
|
+
"prosody": {
|
|
652
|
+
"run": run_prosody,
|
|
653
|
+
"requires": ("ffmpeg", "numpy"),
|
|
654
|
+
"writes": {
|
|
655
|
+
"curves": ["pitch", "vocal_energy", "speech_rate"],
|
|
656
|
+
"events": ["pause", "breath", "hesitation"],
|
|
657
|
+
},
|
|
658
|
+
"audio": True,
|
|
659
|
+
},
|
|
660
|
+
"beat_grid": {
|
|
661
|
+
"run": run_beat_grid,
|
|
662
|
+
"requires": ("ffmpeg", "numpy"),
|
|
663
|
+
"writes": {"events": ["beat", "downbeat"]},
|
|
664
|
+
"audio": True,
|
|
665
|
+
},
|
|
666
|
+
"motion_energy": {
|
|
667
|
+
"run": run_motion_energy,
|
|
668
|
+
"requires": ("ffmpeg",),
|
|
669
|
+
"writes": {"curves": ["motion_energy"]},
|
|
670
|
+
},
|
|
671
|
+
"face": {
|
|
672
|
+
"run": _run_face,
|
|
673
|
+
"capability": _face_capability,
|
|
674
|
+
},
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def run_analyzers(
|
|
679
|
+
project_root: str,
|
|
680
|
+
clip_ref: Any,
|
|
681
|
+
analyzers: Optional[Sequence[str]] = None,
|
|
682
|
+
) -> Dict[str, Any]:
|
|
683
|
+
"""Run analyzers on a clip; per-analyzer honest results.
|
|
684
|
+
|
|
685
|
+
Explicitly-requested analyzers run (and fail loudly if deps are
|
|
686
|
+
missing); the default set runs only what this machine can run, and
|
|
687
|
+
names what it skipped. When several audio analyzers run, the media
|
|
688
|
+
file is decoded once and the samples are shared.
|
|
689
|
+
"""
|
|
690
|
+
caps = capabilities()["analyzers"]
|
|
691
|
+
if analyzers:
|
|
692
|
+
names = list(analyzers)
|
|
693
|
+
skipped: List[str] = []
|
|
694
|
+
else:
|
|
695
|
+
names = [n for n in ANALYZERS if caps.get(n, {}).get("available")]
|
|
696
|
+
skipped = [n for n in ANALYZERS if n not in names]
|
|
697
|
+
unknown = [n for n in names if n not in ANALYZERS]
|
|
698
|
+
if unknown:
|
|
699
|
+
return {
|
|
700
|
+
"success": False,
|
|
701
|
+
"error": f"Unknown analyzer(s): {', '.join(unknown)}",
|
|
702
|
+
"available": sorted(ANALYZERS),
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
audio_names = [n for n in names if ANALYZERS[n].get("audio")]
|
|
706
|
+
shared_clip: Optional[Dict[str, Any]] = None
|
|
707
|
+
shared_samples: "Any" = None
|
|
708
|
+
shared_err: Optional[Dict[str, Any]] = None
|
|
709
|
+
if len(audio_names) > 1:
|
|
710
|
+
shared_clip, shared_samples, shared_err = _audio_context(project_root, clip_ref)
|
|
711
|
+
|
|
712
|
+
results: Dict[str, Any] = {}
|
|
713
|
+
for name in names:
|
|
714
|
+
if name in audio_names and shared_err is not None:
|
|
715
|
+
results[name] = shared_err
|
|
716
|
+
elif name in audio_names and shared_clip is not None:
|
|
717
|
+
results[name] = ANALYZERS[name]["run"](
|
|
718
|
+
project_root, clip_ref, clip=shared_clip, samples=shared_samples
|
|
719
|
+
)
|
|
720
|
+
else:
|
|
721
|
+
results[name] = ANALYZERS[name]["run"](project_root, clip_ref)
|
|
722
|
+
out: Dict[str, Any] = {
|
|
723
|
+
"success": all(r.get("success") for r in results.values()),
|
|
724
|
+
"results": results,
|
|
725
|
+
}
|
|
726
|
+
if skipped:
|
|
727
|
+
out["skipped_unavailable"] = skipped
|
|
728
|
+
return out
|