davinci-resolve-mcp 2.59.0 → 2.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +134 -0
- package/README.md +2 -2
- package/docs/SKILL.md +42 -4
- package/docs/install.md +8 -0
- package/docs/reference/api-limitations.md +9 -1
- package/install.py +1 -1
- package/package.json +1 -1
- package/src/granular/common.py +1 -1
- package/src/server.py +268 -6
- package/src/utils/analysis_store.py +36 -1
- package/src/utils/api_truth.py +56 -0
- package/src/utils/destructive_hook.py +4 -1
- package/src/utils/edit_engine.py +34 -3
- package/src/utils/strata.py +565 -0
- package/src/utils/strata_analyzers.py +684 -0
- package/src/utils/strata_faces.py +347 -0
- package/src/utils/strata_queries.py +643 -0
- package/src/utils/strata_story.py +311 -0
- package/src/utils/timeline_brain_db.py +121 -1
- package/src/utils/timeline_versioning.py +19 -2
|
@@ -0,0 +1,684 @@
|
|
|
1
|
+
"""Perception-strata analyzers — local track writers (no cloud, no Resolve).
|
|
2
|
+
|
|
3
|
+
Each analyzer reads a clip's media file (via ffmpeg) and/or its existing
|
|
4
|
+
strata rows, computes timecoded tracks, and writes them through
|
|
5
|
+
src/utils/strata. All are optional-capability: they self-describe what they
|
|
6
|
+
need (ffmpeg, numpy, a transcript) and refuse honestly instead of degrading
|
|
7
|
+
silently.
|
|
8
|
+
|
|
9
|
+
Analyzers here are deliberately dependency-light: numpy + ffmpeg only.
|
|
10
|
+
Nothing imports librosa/mediapipe — heavier analyzers (face/blink) live
|
|
11
|
+
behind their own capability gate in strata_faces.py.
|
|
12
|
+
|
|
13
|
+
Compute functions are pure (arrays in, tracks out) so they are unit-testable
|
|
14
|
+
on synthetic signals; the run_* wrappers handle clip resolution, decoding,
|
|
15
|
+
and DB writes.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import logging
|
|
22
|
+
import math
|
|
23
|
+
import os
|
|
24
|
+
import shutil
|
|
25
|
+
import subprocess
|
|
26
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
27
|
+
|
|
28
|
+
from src.utils import strata, timeline_brain_db
|
|
29
|
+
from src.utils.proc import safe_run
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger("resolve-mcp.strata-analyzers")
|
|
32
|
+
|
|
33
|
+
try: # numpy is present in most installs but is not a hard requirement.
|
|
34
|
+
import numpy as _np
|
|
35
|
+
except ImportError: # pragma: no cover - exercised on minimal installs
|
|
36
|
+
_np = None
|
|
37
|
+
|
|
38
|
+
PROSODY_SOURCE = "prosody_v1"
|
|
39
|
+
PROSODY_VERSION = "1.0.0"
|
|
40
|
+
BEATGRID_SOURCE = "beatgrid_v1"
|
|
41
|
+
BEATGRID_VERSION = "1.0.0"
|
|
42
|
+
MOTION_SOURCE = "motion_v1"
|
|
43
|
+
MOTION_VERSION = "1.0.0"
|
|
44
|
+
|
|
45
|
+
AUDIO_SAMPLE_RATE = 16000
|
|
46
|
+
FRAME_SECONDS = 0.040 # analysis window
|
|
47
|
+
HOP_SECONDS = 0.010 # 100 Hz curve rate
|
|
48
|
+
CURVE_RATE = 1.0 / HOP_SECONDS
|
|
49
|
+
|
|
50
|
+
# Word-gap thresholds (seconds). A gap shorter than PAUSE_MIN is articulation,
|
|
51
|
+
# not a pause; longer than PAUSE_MAX it is silence/room, not a beat the editor
|
|
52
|
+
# cuts on — still recorded, capped payload marks it.
|
|
53
|
+
PAUSE_MIN_SECONDS = 0.35
|
|
54
|
+
HESITATION_WORDS = {"uh", "um", "er", "erm", "uhm", "hmm", "mm", "mhm", "ah", "eh"}
|
|
55
|
+
|
|
56
|
+
PITCH_MIN_HZ = 60.0
|
|
57
|
+
PITCH_MAX_HZ = 400.0
|
|
58
|
+
|
|
59
|
+
MOTION_CURVE_RATE = 10.0 # Hz
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _ensure_tool_path() -> None:
|
|
63
|
+
"""GUI-launched processes get launchd's bare PATH; reuse media_analysis's
|
|
64
|
+
augmentation so shutil.which finds Homebrew/MacPorts ffmpeg."""
|
|
65
|
+
try:
|
|
66
|
+
from src.utils.media_analysis import _ensure_path_includes_standard_tool_dirs
|
|
67
|
+
|
|
68
|
+
_ensure_path_includes_standard_tool_dirs()
|
|
69
|
+
except Exception: # pragma: no cover - best effort
|
|
70
|
+
pass
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def capabilities() -> Dict[str, Any]:
|
|
74
|
+
"""What the local analyzers can run on this machine."""
|
|
75
|
+
_ensure_tool_path()
|
|
76
|
+
ffmpeg = shutil.which("ffmpeg")
|
|
77
|
+
return {
|
|
78
|
+
"ffmpeg": {"available": bool(ffmpeg), "path": ffmpeg},
|
|
79
|
+
"numpy": {"available": _np is not None},
|
|
80
|
+
"analyzers": {
|
|
81
|
+
"prosody": {
|
|
82
|
+
"available": bool(ffmpeg) and _np is not None,
|
|
83
|
+
"requires": ["ffmpeg", "numpy"],
|
|
84
|
+
"writes": {
|
|
85
|
+
"curves": ["pitch", "vocal_energy", "speech_rate"],
|
|
86
|
+
"events": ["pause", "breath", "hesitation"],
|
|
87
|
+
},
|
|
88
|
+
},
|
|
89
|
+
"beat_grid": {
|
|
90
|
+
"available": bool(ffmpeg) and _np is not None,
|
|
91
|
+
"requires": ["ffmpeg", "numpy"],
|
|
92
|
+
"writes": {"events": ["beat", "downbeat"]},
|
|
93
|
+
},
|
|
94
|
+
"motion_energy": {
|
|
95
|
+
"available": bool(ffmpeg),
|
|
96
|
+
"requires": ["ffmpeg"],
|
|
97
|
+
"writes": {"curves": ["motion_energy"]},
|
|
98
|
+
},
|
|
99
|
+
"face": _face_capability(),
|
|
100
|
+
},
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _face_capability() -> Dict[str, Any]:
|
|
105
|
+
# A broken optional face stack must never take strata_status/strata_run
|
|
106
|
+
# down with it — report unavailable instead.
|
|
107
|
+
try:
|
|
108
|
+
from src.utils import strata_faces
|
|
109
|
+
|
|
110
|
+
return strata_faces.capabilities()
|
|
111
|
+
except Exception as exc: # pragma: no cover - environment-dependent
|
|
112
|
+
return {"available": False, "error": f"face stack failed to load: {exc}"}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _require(*needs: str) -> Optional[Dict[str, Any]]:
|
|
116
|
+
if "numpy" in needs and _np is None:
|
|
117
|
+
return {"success": False, "error": "numpy is required for this analyzer", "missing": "numpy"}
|
|
118
|
+
if "ffmpeg" in needs:
|
|
119
|
+
_ensure_tool_path()
|
|
120
|
+
if not shutil.which("ffmpeg"):
|
|
121
|
+
return {"success": False, "error": "ffmpeg not found on PATH", "missing": "ffmpeg"}
|
|
122
|
+
return None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# ── clip resolution ──────────────────────────────────────────────────────────
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _clip_row(project_root: str, clip_ref: Any) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
|
|
129
|
+
"""Resolve a clip ref to {clip_uuid, file_path, duration_seconds, fps}."""
|
|
130
|
+
from src.utils import analysis_store
|
|
131
|
+
|
|
132
|
+
conn = timeline_brain_db.connect(project_root)
|
|
133
|
+
clip_uuid = analysis_store.resolve_clip_uuid(conn, clip_ref)
|
|
134
|
+
if not clip_uuid:
|
|
135
|
+
return None, {
|
|
136
|
+
"success": False,
|
|
137
|
+
"error": f"Unknown clip ref: {clip_ref!r} (older analysis root? run db_ingest first)",
|
|
138
|
+
}
|
|
139
|
+
row = conn.execute("SELECT * FROM clips WHERE clip_uuid = ?", (clip_uuid,)).fetchone()
|
|
140
|
+
if row is None:
|
|
141
|
+
return None, {"success": False, "error": f"No clips row for {clip_uuid}"}
|
|
142
|
+
file_path = row["file_path"]
|
|
143
|
+
if not file_path or not os.path.isfile(file_path):
|
|
144
|
+
return None, {
|
|
145
|
+
"success": False,
|
|
146
|
+
"error": f"Media file not accessible for clip {row['clip_name']!r}: {file_path!r}",
|
|
147
|
+
"clip_uuid": clip_uuid,
|
|
148
|
+
}
|
|
149
|
+
return (
|
|
150
|
+
{
|
|
151
|
+
"clip_uuid": clip_uuid,
|
|
152
|
+
"clip_name": row["clip_name"],
|
|
153
|
+
"file_path": file_path,
|
|
154
|
+
"duration_seconds": row["duration_seconds"],
|
|
155
|
+
"fps": row["fps"],
|
|
156
|
+
},
|
|
157
|
+
None,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# ── decoding ─────────────────────────────────────────────────────────────────
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def decode_audio(path: str, sample_rate: int = AUDIO_SAMPLE_RATE, timeout: int = 600) -> "Any":
|
|
165
|
+
"""Decode a media file's audio to mono float32 PCM via ffmpeg."""
|
|
166
|
+
cmd = [
|
|
167
|
+
shutil.which("ffmpeg") or "ffmpeg",
|
|
168
|
+
"-v", "error",
|
|
169
|
+
"-i", path,
|
|
170
|
+
"-map", "0:a:0",
|
|
171
|
+
"-ac", "1",
|
|
172
|
+
"-ar", str(sample_rate),
|
|
173
|
+
"-f", "f32le",
|
|
174
|
+
"-",
|
|
175
|
+
]
|
|
176
|
+
proc = safe_run(cmd, capture_output=True, timeout=timeout, check=False)
|
|
177
|
+
if proc.returncode != 0:
|
|
178
|
+
raise RuntimeError(f"ffmpeg audio decode failed: {proc.stderr.decode('utf-8', 'replace')[:500]}")
|
|
179
|
+
return _np.frombuffer(proc.stdout, dtype=_np.float32)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# ── prosody: pure compute ────────────────────────────────────────────────────
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _frame_signal(samples: "Any", sample_rate: int) -> Tuple["Any", int, int]:
|
|
186
|
+
frame_len = int(sample_rate * FRAME_SECONDS)
|
|
187
|
+
hop_len = int(sample_rate * HOP_SECONDS)
|
|
188
|
+
if len(samples) < frame_len:
|
|
189
|
+
return _np.empty((0, frame_len), dtype=_np.float32), frame_len, hop_len
|
|
190
|
+
n_frames = 1 + (len(samples) - frame_len) // hop_len
|
|
191
|
+
idx = _np.arange(frame_len)[None, :] + hop_len * _np.arange(n_frames)[:, None]
|
|
192
|
+
return samples[idx], frame_len, hop_len
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def compute_energy_curve(samples: "Any", sample_rate: int = AUDIO_SAMPLE_RATE) -> List[float]:
|
|
196
|
+
"""Per-frame RMS, normalized to the clip's 95th percentile (0..~1)."""
|
|
197
|
+
frames, _, _ = _frame_signal(samples, sample_rate)
|
|
198
|
+
if frames.shape[0] == 0:
|
|
199
|
+
return []
|
|
200
|
+
rms = _np.sqrt(_np.mean(frames.astype(_np.float64) ** 2, axis=1))
|
|
201
|
+
scale = float(_np.percentile(rms, 95)) or 1.0
|
|
202
|
+
if scale <= 0:
|
|
203
|
+
scale = 1.0
|
|
204
|
+
return [float(v) for v in _np.clip(rms / scale, 0.0, 4.0)]
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def compute_pitch_curve(
|
|
208
|
+
samples: "Any",
|
|
209
|
+
sample_rate: int = AUDIO_SAMPLE_RATE,
|
|
210
|
+
energy: Optional[Sequence[float]] = None,
|
|
211
|
+
voiced_threshold: float = 0.10,
|
|
212
|
+
) -> List[float]:
|
|
213
|
+
"""Autocorrelation pitch (Hz) per frame; NaN where unvoiced/silent.
|
|
214
|
+
|
|
215
|
+
Deliberately simple (no pYIN): good enough to carry contour direction,
|
|
216
|
+
range, and tremor for take comparison — not for music transcription.
|
|
217
|
+
"""
|
|
218
|
+
frames, frame_len, _ = _frame_signal(samples, sample_rate)
|
|
219
|
+
if frames.shape[0] == 0:
|
|
220
|
+
return []
|
|
221
|
+
if energy is None:
|
|
222
|
+
energy = compute_energy_curve(samples, sample_rate)
|
|
223
|
+
lag_min = int(sample_rate / PITCH_MAX_HZ)
|
|
224
|
+
lag_max = min(int(sample_rate / PITCH_MIN_HZ), frame_len - 1)
|
|
225
|
+
out: List[float] = []
|
|
226
|
+
hann = _np.hanning(frame_len)
|
|
227
|
+
for i in range(frames.shape[0]):
|
|
228
|
+
if i >= len(energy) or energy[i] < voiced_threshold:
|
|
229
|
+
out.append(float("nan"))
|
|
230
|
+
continue
|
|
231
|
+
frame = frames[i].astype(_np.float64) * hann
|
|
232
|
+
frame = frame - frame.mean()
|
|
233
|
+
# FFT autocorrelation
|
|
234
|
+
spec = _np.fft.rfft(frame, n=2 * frame_len)
|
|
235
|
+
ac = _np.fft.irfft(spec * _np.conj(spec))[:frame_len]
|
|
236
|
+
if ac[0] <= 0:
|
|
237
|
+
out.append(float("nan"))
|
|
238
|
+
continue
|
|
239
|
+
ac = ac / ac[0]
|
|
240
|
+
window = ac[lag_min:lag_max]
|
|
241
|
+
if window.size == 0:
|
|
242
|
+
out.append(float("nan"))
|
|
243
|
+
continue
|
|
244
|
+
peak = int(_np.argmax(window)) + lag_min
|
|
245
|
+
# Voicing gate: the autocorrelation peak must be strong.
|
|
246
|
+
if ac[peak] < 0.30:
|
|
247
|
+
out.append(float("nan"))
|
|
248
|
+
continue
|
|
249
|
+
out.append(float(sample_rate / peak))
|
|
250
|
+
return out
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def compute_speech_rate_curve(
|
|
254
|
+
words: Sequence[Dict[str, Any]],
|
|
255
|
+
duration_seconds: float,
|
|
256
|
+
window_seconds: float = 2.0,
|
|
257
|
+
rate_hz: float = 10.0,
|
|
258
|
+
) -> List[float]:
|
|
259
|
+
"""Words/second in a centered sliding window, sampled at rate_hz."""
|
|
260
|
+
n = max(1, int(math.ceil(duration_seconds * rate_hz)))
|
|
261
|
+
starts = [w.get("start_seconds") for w in words if isinstance(w.get("start_seconds"), (int, float))]
|
|
262
|
+
out = []
|
|
263
|
+
half = window_seconds / 2.0
|
|
264
|
+
for i in range(n):
|
|
265
|
+
t = i / rate_hz
|
|
266
|
+
lo, hi = t - half, t + half
|
|
267
|
+
count = sum(1 for s in starts if lo <= s < hi)
|
|
268
|
+
out.append(count / window_seconds)
|
|
269
|
+
return out
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def detect_pauses(
|
|
273
|
+
words: Sequence[Dict[str, Any]],
|
|
274
|
+
min_gap_seconds: float = PAUSE_MIN_SECONDS,
|
|
275
|
+
) -> List[Dict[str, Any]]:
|
|
276
|
+
"""Word-gap pauses inside the spoken region. Gap = prev.end → next.start."""
|
|
277
|
+
timed = [
|
|
278
|
+
w for w in words
|
|
279
|
+
if isinstance(w.get("start_seconds"), (int, float)) and isinstance(w.get("end_seconds"), (int, float))
|
|
280
|
+
]
|
|
281
|
+
timed.sort(key=lambda w: (w["start_seconds"], w["end_seconds"]))
|
|
282
|
+
pauses = []
|
|
283
|
+
for prev, nxt in zip(timed, timed[1:]):
|
|
284
|
+
gap_start = float(prev["end_seconds"])
|
|
285
|
+
gap = float(nxt["start_seconds"]) - gap_start
|
|
286
|
+
if gap >= min_gap_seconds:
|
|
287
|
+
pauses.append(
|
|
288
|
+
{
|
|
289
|
+
"time_seconds": gap_start,
|
|
290
|
+
"duration_seconds": gap,
|
|
291
|
+
"payload": {
|
|
292
|
+
"before_word": prev.get("word"),
|
|
293
|
+
"after_word": nxt.get("word"),
|
|
294
|
+
},
|
|
295
|
+
}
|
|
296
|
+
)
|
|
297
|
+
return pauses
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def detect_hesitations(words: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
301
|
+
out = []
|
|
302
|
+
for w in words:
|
|
303
|
+
token = str(w.get("word") or "").strip().lower().strip(".,!?;:")
|
|
304
|
+
if token in HESITATION_WORDS and isinstance(w.get("start_seconds"), (int, float)):
|
|
305
|
+
end = w.get("end_seconds")
|
|
306
|
+
out.append(
|
|
307
|
+
{
|
|
308
|
+
"time_seconds": float(w["start_seconds"]),
|
|
309
|
+
"duration_seconds": (float(end) - float(w["start_seconds"])) if isinstance(end, (int, float)) else None,
|
|
310
|
+
"payload": {"word": w.get("word")},
|
|
311
|
+
}
|
|
312
|
+
)
|
|
313
|
+
return out
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def detect_breaths(
|
|
317
|
+
energy: Sequence[float],
|
|
318
|
+
pauses: Sequence[Dict[str, Any]],
|
|
319
|
+
curve_rate: float = CURVE_RATE,
|
|
320
|
+
) -> List[Dict[str, Any]]:
|
|
321
|
+
"""Breath candidates: a sub-speech energy bump inside a word gap.
|
|
322
|
+
|
|
323
|
+
Honest heuristic (flagged low-confidence): within each pause, look for a
|
|
324
|
+
local energy peak that clears the pause's own floor but stays well under
|
|
325
|
+
speech level. Catches on-mic inhales; misses quiet ones.
|
|
326
|
+
"""
|
|
327
|
+
if not len(energy):
|
|
328
|
+
return []
|
|
329
|
+
arr = _np.asarray(energy, dtype=_np.float64) if _np is not None else None
|
|
330
|
+
breaths: List[Dict[str, Any]] = []
|
|
331
|
+
for pause in pauses:
|
|
332
|
+
start = float(pause["time_seconds"])
|
|
333
|
+
dur = float(pause.get("duration_seconds") or 0.0)
|
|
334
|
+
lo = int(start * curve_rate)
|
|
335
|
+
hi = min(int((start + dur) * curve_rate), len(energy))
|
|
336
|
+
if hi - lo < 3:
|
|
337
|
+
continue
|
|
338
|
+
seg = arr[lo:hi] if arr is not None else energy[lo:hi]
|
|
339
|
+
floor = float(_np.percentile(seg, 20)) if arr is not None else min(seg)
|
|
340
|
+
peak_idx = int(_np.argmax(seg)) if arr is not None else max(range(len(seg)), key=lambda i: seg[i])
|
|
341
|
+
peak = float(seg[peak_idx])
|
|
342
|
+
# Bump: clearly above the gap floor, clearly below speech (~1.0 scale).
|
|
343
|
+
if peak >= floor + 0.05 and peak <= 0.5:
|
|
344
|
+
breaths.append(
|
|
345
|
+
{
|
|
346
|
+
"time_seconds": (lo + peak_idx) / curve_rate,
|
|
347
|
+
"duration_seconds": None,
|
|
348
|
+
"payload": {"confidence": "low", "peak_energy": round(peak, 4)},
|
|
349
|
+
}
|
|
350
|
+
)
|
|
351
|
+
return breaths
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
# ── prosody: runner ──────────────────────────────────────────────────────────
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def run_prosody(project_root: str, clip_ref: Any) -> Dict[str, Any]:
|
|
358
|
+
"""Compute + persist prosody strata for one clip.
|
|
359
|
+
|
|
360
|
+
Writes curves pitch / vocal_energy / speech_rate and events pause /
|
|
361
|
+
breath / hesitation. Requires the clip's media file, ffmpeg, numpy, and
|
|
362
|
+
(for word-derived tracks) transcript_words rows.
|
|
363
|
+
"""
|
|
364
|
+
missing = _require("ffmpeg", "numpy")
|
|
365
|
+
if missing:
|
|
366
|
+
return missing
|
|
367
|
+
clip, err = _clip_row(project_root, clip_ref)
|
|
368
|
+
if err:
|
|
369
|
+
return err
|
|
370
|
+
conn = timeline_brain_db.connect(project_root)
|
|
371
|
+
words = strata.read_words(conn, clip["clip_uuid"])
|
|
372
|
+
|
|
373
|
+
try:
|
|
374
|
+
samples = decode_audio(clip["file_path"])
|
|
375
|
+
except (RuntimeError, subprocess.TimeoutExpired) as exc:
|
|
376
|
+
return {"success": False, "error": str(exc), "clip_uuid": clip["clip_uuid"]}
|
|
377
|
+
if samples.size == 0:
|
|
378
|
+
return {"success": False, "error": "clip has no decodable audio", "clip_uuid": clip["clip_uuid"]}
|
|
379
|
+
|
|
380
|
+
duration = clip["duration_seconds"] or (samples.size / AUDIO_SAMPLE_RATE)
|
|
381
|
+
energy = compute_energy_curve(samples)
|
|
382
|
+
pitch = compute_pitch_curve(samples, energy=energy)
|
|
383
|
+
pauses = detect_pauses(words)
|
|
384
|
+
hesitations = detect_hesitations(words)
|
|
385
|
+
breaths = detect_breaths(energy, pauses)
|
|
386
|
+
speech_rate = compute_speech_rate_curve(words, float(duration)) if words else []
|
|
387
|
+
|
|
388
|
+
with timeline_brain_db.transaction(project_root) as txn:
|
|
389
|
+
strata.write_curve(
|
|
390
|
+
txn, clip["clip_uuid"], "vocal_energy", energy,
|
|
391
|
+
sample_rate=CURVE_RATE, source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
392
|
+
)
|
|
393
|
+
strata.write_curve(
|
|
394
|
+
txn, clip["clip_uuid"], "pitch", pitch,
|
|
395
|
+
sample_rate=CURVE_RATE, source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
396
|
+
)
|
|
397
|
+
if speech_rate:
|
|
398
|
+
strata.write_curve(
|
|
399
|
+
txn, clip["clip_uuid"], "speech_rate", speech_rate,
|
|
400
|
+
sample_rate=10.0, source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
401
|
+
)
|
|
402
|
+
strata.replace_track_events(
|
|
403
|
+
txn, clip["clip_uuid"], "pause", pauses,
|
|
404
|
+
source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
405
|
+
)
|
|
406
|
+
strata.replace_track_events(
|
|
407
|
+
txn, clip["clip_uuid"], "hesitation", hesitations,
|
|
408
|
+
source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
409
|
+
)
|
|
410
|
+
strata.replace_track_events(
|
|
411
|
+
txn, clip["clip_uuid"], "breath", breaths,
|
|
412
|
+
source=PROSODY_SOURCE, analyzer_version=PROSODY_VERSION,
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
return {
|
|
416
|
+
"success": True,
|
|
417
|
+
"clip_uuid": clip["clip_uuid"],
|
|
418
|
+
"clip_name": clip["clip_name"],
|
|
419
|
+
"curves": {
|
|
420
|
+
"vocal_energy": len(energy),
|
|
421
|
+
"pitch": len(pitch),
|
|
422
|
+
"speech_rate": len(speech_rate),
|
|
423
|
+
},
|
|
424
|
+
"events": {
|
|
425
|
+
"pause": len(pauses),
|
|
426
|
+
"hesitation": len(hesitations),
|
|
427
|
+
"breath": len(breaths),
|
|
428
|
+
},
|
|
429
|
+
"had_words": bool(words),
|
|
430
|
+
"analyzer_version": PROSODY_VERSION,
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
# ── musical beat grid ────────────────────────────────────────────────────────
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def compute_beat_grid(
|
|
438
|
+
samples: "Any",
|
|
439
|
+
sample_rate: int = AUDIO_SAMPLE_RATE,
|
|
440
|
+
tempo_min: float = 60.0,
|
|
441
|
+
tempo_max: float = 200.0,
|
|
442
|
+
) -> Dict[str, Any]:
|
|
443
|
+
"""Onset-envelope beat tracking: spectral flux → tempo via autocorrelation
|
|
444
|
+
→ beat placement by phase search. numpy-only; no ML.
|
|
445
|
+
|
|
446
|
+
Returns {tempo_bpm, beats: [seconds], downbeats: [seconds], confidence}.
|
|
447
|
+
Downbeats are a phase-of-strongest-onset estimate every 4 beats —
|
|
448
|
+
explicitly low-confidence (no meter detection).
|
|
449
|
+
"""
|
|
450
|
+
hop = int(sample_rate * HOP_SECONDS)
|
|
451
|
+
win = int(sample_rate * FRAME_SECONDS)
|
|
452
|
+
frames, _, _ = _frame_signal(samples, sample_rate)
|
|
453
|
+
if frames.shape[0] < 16:
|
|
454
|
+
return {"tempo_bpm": None, "beats": [], "downbeats": [], "confidence": 0.0}
|
|
455
|
+
hann = _np.hanning(win)
|
|
456
|
+
mags = _np.abs(_np.fft.rfft(frames * hann, axis=1))
|
|
457
|
+
flux = _np.diff(mags, axis=0)
|
|
458
|
+
flux[flux < 0] = 0.0
|
|
459
|
+
onset = flux.sum(axis=1)
|
|
460
|
+
if onset.max() <= 0:
|
|
461
|
+
return {"tempo_bpm": None, "beats": [], "downbeats": [], "confidence": 0.0}
|
|
462
|
+
onset = onset / onset.max()
|
|
463
|
+
onset = onset - onset.mean()
|
|
464
|
+
|
|
465
|
+
# Tempo: autocorrelation peak in the plausible lag range.
|
|
466
|
+
ac = _np.correlate(onset, onset, mode="full")[len(onset) - 1:]
|
|
467
|
+
if ac[0] <= 0:
|
|
468
|
+
return {"tempo_bpm": None, "beats": [], "downbeats": [], "confidence": 0.0}
|
|
469
|
+
ac = ac / ac[0]
|
|
470
|
+
frame_rate = 1.0 / HOP_SECONDS
|
|
471
|
+
lag_min = max(1, int(frame_rate * 60.0 / tempo_max))
|
|
472
|
+
lag_max = min(len(ac) - 1, int(frame_rate * 60.0 / tempo_min))
|
|
473
|
+
if lag_max <= lag_min:
|
|
474
|
+
return {"tempo_bpm": None, "beats": [], "downbeats": [], "confidence": 0.0}
|
|
475
|
+
lag = int(_np.argmax(ac[lag_min:lag_max])) + lag_min
|
|
476
|
+
tempo_bpm = 60.0 * frame_rate / lag
|
|
477
|
+
confidence = float(ac[lag])
|
|
478
|
+
|
|
479
|
+
# Phase: shift the beat comb to maximize onset energy under it.
|
|
480
|
+
best_phase, best_score = 0, -1.0
|
|
481
|
+
for phase in range(lag):
|
|
482
|
+
score = float(onset[phase::lag].sum())
|
|
483
|
+
if score > best_score:
|
|
484
|
+
best_score, best_phase = score, phase
|
|
485
|
+
beat_frames = list(range(best_phase, len(onset), lag))
|
|
486
|
+
beats = [(f + 1) * HOP_SECONDS for f in beat_frames] # +1: flux is a diff
|
|
487
|
+
|
|
488
|
+
# Downbeat guess: strongest onset among the first 4 beats sets bar phase.
|
|
489
|
+
downbeats = []
|
|
490
|
+
if len(beat_frames) >= 4:
|
|
491
|
+
first_bar = beat_frames[:4]
|
|
492
|
+
strongest = max(range(len(first_bar)), key=lambda i: onset[first_bar[i]])
|
|
493
|
+
downbeats = beats[strongest::4]
|
|
494
|
+
|
|
495
|
+
return {
|
|
496
|
+
"tempo_bpm": round(tempo_bpm, 2),
|
|
497
|
+
"beats": beats,
|
|
498
|
+
"downbeats": downbeats,
|
|
499
|
+
"confidence": round(confidence, 3),
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def run_beat_grid(project_root: str, clip_ref: Any) -> Dict[str, Any]:
|
|
504
|
+
"""Compute + persist the musical beat grid for one clip."""
|
|
505
|
+
missing = _require("ffmpeg", "numpy")
|
|
506
|
+
if missing:
|
|
507
|
+
return missing
|
|
508
|
+
clip, err = _clip_row(project_root, clip_ref)
|
|
509
|
+
if err:
|
|
510
|
+
return err
|
|
511
|
+
try:
|
|
512
|
+
samples = decode_audio(clip["file_path"])
|
|
513
|
+
except (RuntimeError, subprocess.TimeoutExpired) as exc:
|
|
514
|
+
return {"success": False, "error": str(exc), "clip_uuid": clip["clip_uuid"]}
|
|
515
|
+
if samples.size == 0:
|
|
516
|
+
return {"success": False, "error": "clip has no decodable audio", "clip_uuid": clip["clip_uuid"]}
|
|
517
|
+
|
|
518
|
+
grid = compute_beat_grid(samples)
|
|
519
|
+
beat_events = [
|
|
520
|
+
{"time_seconds": t, "payload": {"tempo_bpm": grid["tempo_bpm"], "confidence": grid["confidence"]}}
|
|
521
|
+
for t in grid["beats"]
|
|
522
|
+
]
|
|
523
|
+
downbeat_events = [
|
|
524
|
+
{"time_seconds": t, "payload": {"confidence": "low"}} for t in grid["downbeats"]
|
|
525
|
+
]
|
|
526
|
+
with timeline_brain_db.transaction(project_root) as txn:
|
|
527
|
+
strata.replace_track_events(
|
|
528
|
+
txn, clip["clip_uuid"], "beat", beat_events,
|
|
529
|
+
source=BEATGRID_SOURCE, analyzer_version=BEATGRID_VERSION,
|
|
530
|
+
)
|
|
531
|
+
strata.replace_track_events(
|
|
532
|
+
txn, clip["clip_uuid"], "downbeat", downbeat_events,
|
|
533
|
+
source=BEATGRID_SOURCE, analyzer_version=BEATGRID_VERSION,
|
|
534
|
+
)
|
|
535
|
+
return {
|
|
536
|
+
"success": True,
|
|
537
|
+
"clip_uuid": clip["clip_uuid"],
|
|
538
|
+
"clip_name": clip["clip_name"],
|
|
539
|
+
"tempo_bpm": grid["tempo_bpm"],
|
|
540
|
+
"beat_count": len(beat_events),
|
|
541
|
+
"downbeat_count": len(downbeat_events),
|
|
542
|
+
"confidence": grid["confidence"],
|
|
543
|
+
"analyzer_version": BEATGRID_VERSION,
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
# ── motion energy ────────────────────────────────────────────────────────────
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def run_motion_energy(project_root: str, clip_ref: Any, timeout: int = 1800) -> Dict[str, Any]:
|
|
551
|
+
"""Per-frame luma difference (ffmpeg signalstats YDIF) → motion_energy curve.
|
|
552
|
+
|
|
553
|
+
Downsampled to MOTION_CURVE_RATE by mean-pooling so an hour of footage is
|
|
554
|
+
~140 KB. 0 = static, higher = more inter-frame change; normalized 0..1
|
|
555
|
+
against the clip's own 98th percentile.
|
|
556
|
+
"""
|
|
557
|
+
missing = _require("ffmpeg")
|
|
558
|
+
if missing:
|
|
559
|
+
return missing
|
|
560
|
+
clip, err = _clip_row(project_root, clip_ref)
|
|
561
|
+
if err:
|
|
562
|
+
return err
|
|
563
|
+
|
|
564
|
+
cmd = [
|
|
565
|
+
shutil.which("ffmpeg") or "ffmpeg",
|
|
566
|
+
"-v", "error",
|
|
567
|
+
"-i", clip["file_path"],
|
|
568
|
+
"-map", "0:v:0",
|
|
569
|
+
"-vf", "signalstats,metadata=print:key=lavfi.signalstats.YDIF:file=-",
|
|
570
|
+
"-f", "null", "-",
|
|
571
|
+
]
|
|
572
|
+
try:
|
|
573
|
+
proc = safe_run(cmd, capture_output=True, timeout=timeout, check=False)
|
|
574
|
+
except subprocess.TimeoutExpired:
|
|
575
|
+
return {"success": False, "error": "motion analysis timed out", "clip_uuid": clip["clip_uuid"]}
|
|
576
|
+
if proc.returncode != 0:
|
|
577
|
+
return {
|
|
578
|
+
"success": False,
|
|
579
|
+
"error": f"ffmpeg signalstats failed: {proc.stderr.decode('utf-8', 'replace')[:500]}",
|
|
580
|
+
"clip_uuid": clip["clip_uuid"],
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
times: List[float] = []
|
|
584
|
+
ydifs: List[float] = []
|
|
585
|
+
pending_time: Optional[float] = None
|
|
586
|
+
for line in proc.stdout.decode("utf-8", "replace").splitlines():
|
|
587
|
+
line = line.strip()
|
|
588
|
+
if line.startswith("frame:"):
|
|
589
|
+
# e.g. "frame:12 pts:6006 pts_time:0.25025"
|
|
590
|
+
for token in line.split():
|
|
591
|
+
if token.startswith("pts_time:"):
|
|
592
|
+
try:
|
|
593
|
+
pending_time = float(token.split(":", 1)[1])
|
|
594
|
+
except ValueError:
|
|
595
|
+
pending_time = None
|
|
596
|
+
elif line.startswith("lavfi.signalstats.YDIF=") and pending_time is not None:
|
|
597
|
+
try:
|
|
598
|
+
ydifs.append(float(line.split("=", 1)[1]))
|
|
599
|
+
times.append(pending_time)
|
|
600
|
+
except ValueError:
|
|
601
|
+
pass
|
|
602
|
+
pending_time = None
|
|
603
|
+
|
|
604
|
+
if not ydifs:
|
|
605
|
+
return {"success": False, "error": "no video frames analyzed", "clip_uuid": clip["clip_uuid"]}
|
|
606
|
+
|
|
607
|
+
duration = times[-1] if times else (clip["duration_seconds"] or 0.0)
|
|
608
|
+
n_out = max(1, int(math.ceil((duration or 1.0) * MOTION_CURVE_RATE)))
|
|
609
|
+
buckets: List[List[float]] = [[] for _ in range(n_out)]
|
|
610
|
+
for t, v in zip(times, ydifs):
|
|
611
|
+
idx = min(int(t * MOTION_CURVE_RATE), n_out - 1)
|
|
612
|
+
buckets[idx].append(v)
|
|
613
|
+
pooled = [sum(b) / len(b) if b else float("nan") for b in buckets]
|
|
614
|
+
finite = sorted(v for v in pooled if not math.isnan(v))
|
|
615
|
+
if finite:
|
|
616
|
+
scale = finite[min(len(finite) - 1, int(len(finite) * 0.98))] or 1.0
|
|
617
|
+
if scale <= 0:
|
|
618
|
+
scale = 1.0
|
|
619
|
+
pooled = [min(v / scale, 4.0) if not math.isnan(v) else v for v in pooled]
|
|
620
|
+
|
|
621
|
+
with timeline_brain_db.transaction(project_root) as txn:
|
|
622
|
+
strata.write_curve(
|
|
623
|
+
txn, clip["clip_uuid"], "motion_energy", pooled,
|
|
624
|
+
sample_rate=MOTION_CURVE_RATE, source=MOTION_SOURCE, analyzer_version=MOTION_VERSION,
|
|
625
|
+
)
|
|
626
|
+
return {
|
|
627
|
+
"success": True,
|
|
628
|
+
"clip_uuid": clip["clip_uuid"],
|
|
629
|
+
"clip_name": clip["clip_name"],
|
|
630
|
+
"frames_analyzed": len(ydifs),
|
|
631
|
+
"samples": len(pooled),
|
|
632
|
+
"analyzer_version": MOTION_VERSION,
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
# ── dispatcher ───────────────────────────────────────────────────────────────
|
|
637
|
+
|
|
638
|
+
def _run_face(project_root: str, clip_ref: Any) -> Dict[str, Any]:
|
|
639
|
+
from src.utils import strata_faces
|
|
640
|
+
|
|
641
|
+
return strata_faces.run_face_strata(project_root, clip_ref)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
ANALYZERS = {
|
|
645
|
+
"prosody": run_prosody,
|
|
646
|
+
"beat_grid": run_beat_grid,
|
|
647
|
+
"motion_energy": run_motion_energy,
|
|
648
|
+
"face": _run_face,
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def run_analyzers(
|
|
653
|
+
project_root: str,
|
|
654
|
+
clip_ref: Any,
|
|
655
|
+
analyzers: Optional[Sequence[str]] = None,
|
|
656
|
+
) -> Dict[str, Any]:
|
|
657
|
+
"""Run analyzers on a clip; per-analyzer honest results.
|
|
658
|
+
|
|
659
|
+
Explicitly-requested analyzers run (and fail loudly if deps are
|
|
660
|
+
missing); the default set runs only what this machine can run, and
|
|
661
|
+
names what it skipped.
|
|
662
|
+
"""
|
|
663
|
+
caps = capabilities()["analyzers"]
|
|
664
|
+
if analyzers:
|
|
665
|
+
names = list(analyzers)
|
|
666
|
+
skipped: List[str] = []
|
|
667
|
+
else:
|
|
668
|
+
names = [n for n in ANALYZERS if caps.get(n, {}).get("available")]
|
|
669
|
+
skipped = [n for n in ANALYZERS if n not in names]
|
|
670
|
+
unknown = [n for n in names if n not in ANALYZERS]
|
|
671
|
+
if unknown:
|
|
672
|
+
return {
|
|
673
|
+
"success": False,
|
|
674
|
+
"error": f"Unknown analyzer(s): {', '.join(unknown)}",
|
|
675
|
+
"available": sorted(ANALYZERS),
|
|
676
|
+
}
|
|
677
|
+
results = {name: ANALYZERS[name](project_root, clip_ref) for name in names}
|
|
678
|
+
out: Dict[str, Any] = {
|
|
679
|
+
"success": all(r.get("success") for r in results.values()),
|
|
680
|
+
"results": results,
|
|
681
|
+
}
|
|
682
|
+
if skipped:
|
|
683
|
+
out["skipped_unavailable"] = skipped
|
|
684
|
+
return out
|