davinci-resolve-mcp 2.61.0 → 2.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +104 -0
- package/README.md +1 -1
- package/install.py +1 -1
- package/package.json +1 -1
- package/src/granular/common.py +1 -1
- package/src/server.py +516 -189
- package/src/utils/analysis_store.py +39 -0
- package/src/utils/background_jobs.py +108 -0
- package/src/utils/deep_vision.py +1 -30
- package/src/utils/strata.py +87 -20
- package/src/utils/strata_analyzers.py +135 -91
- package/src/utils/strata_queries.py +43 -23
- package/src/utils/strata_story.py +4 -10
- package/src/utils/timeline_brain_db.py +18 -1
|
@@ -200,6 +200,45 @@ def resolve_clip_uuid(conn: sqlite3.Connection, ref: Any) -> Optional[str]:
|
|
|
200
200
|
return None
|
|
201
201
|
|
|
202
202
|
|
|
203
|
+
def resolve_clip_uuid_ingesting(
|
|
204
|
+
project_root: str, conn: sqlite3.Connection, clip_ref: Any
|
|
205
|
+
) -> Optional[str]:
|
|
206
|
+
"""resolve_clip_uuid plus the pre-v9 fallback: when the DB has no rows
|
|
207
|
+
for the ref, walk clips/*/analysis.json for a matching report and ingest
|
|
208
|
+
it. Shared by deep_vision and the strata layers so a clip that resolves
|
|
209
|
+
for one resolves for all."""
|
|
210
|
+
clip_uuid = resolve_clip_uuid(conn, clip_ref)
|
|
211
|
+
if clip_uuid:
|
|
212
|
+
return clip_uuid
|
|
213
|
+
ma = _ma()
|
|
214
|
+
clips_root = os.path.join(project_root, "clips")
|
|
215
|
+
candidate = str(clip_ref or "")
|
|
216
|
+
if not os.path.isdir(clips_root):
|
|
217
|
+
return None
|
|
218
|
+
for entry in sorted(os.listdir(clips_root)):
|
|
219
|
+
report_path = os.path.join(clips_root, entry, "analysis.json")
|
|
220
|
+
if not os.path.isfile(report_path):
|
|
221
|
+
continue
|
|
222
|
+
try:
|
|
223
|
+
with open(report_path, "r", encoding="utf-8") as handle:
|
|
224
|
+
report = json.load(handle)
|
|
225
|
+
except (OSError, json.JSONDecodeError):
|
|
226
|
+
continue
|
|
227
|
+
clip_block = report.get("clip") or {}
|
|
228
|
+
if candidate not in (
|
|
229
|
+
entry,
|
|
230
|
+
str(clip_block.get("clip_id") or ""),
|
|
231
|
+
str(clip_block.get("media_id") or ""),
|
|
232
|
+
ma.normalize_path(clip_block.get("file_path") or ""),
|
|
233
|
+
):
|
|
234
|
+
continue
|
|
235
|
+
ingest = ingest_report(project_root, report, clip_dir=os.path.join(clips_root, entry))
|
|
236
|
+
if ingest.get("success"):
|
|
237
|
+
return str(ingest["clip_uuid"])
|
|
238
|
+
return None
|
|
239
|
+
return None
|
|
240
|
+
|
|
241
|
+
|
|
203
242
|
# ── ingest ────────────────────────────────────────────────────────────────────
|
|
204
243
|
|
|
205
244
|
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""In-process registry for long DaVinci Resolve operations run off-thread.
|
|
2
|
+
|
|
3
|
+
Some Resolve calls (transcription, subtitle generation, scene-cut and Dolby
|
|
4
|
+
analysis, timeline export/import) block for minutes — longer than an MCP
|
|
5
|
+
client's tool-window timeout. start_job runs such a call on a daemon thread and
|
|
6
|
+
returns a short id immediately; the caller polls job_status until the job
|
|
7
|
+
reports "done" (with its result) or "error" (with the message).
|
|
8
|
+
|
|
9
|
+
The worker runs fn inside resolve_busy.long_resolve_op(label) so another
|
|
10
|
+
Resolve-touching tool call meets the advisory "busy" answer instead of colliding
|
|
11
|
+
on the single-threaded scripting bridge (job_status / list_jobs are
|
|
12
|
+
connection-free and never contend). Serialization comes from that preflight, not
|
|
13
|
+
from the bridge itself; the busy record tracks a single owner, so two jobs
|
|
14
|
+
started in quick succession have a brief window before the first's record shows.
|
|
15
|
+
|
|
16
|
+
Finished jobs are pruned once older than MAX_JOB_AGE_SECONDS on any access, so
|
|
17
|
+
the registry cannot grow without bound.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
import uuid
|
|
24
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
25
|
+
|
|
26
|
+
from src.utils.resolve_busy import long_resolve_op
|
|
27
|
+
|
|
28
|
+
# A finished job is dropped from the registry this long after it ended.
|
|
29
|
+
MAX_JOB_AGE_SECONDS = 60 * 60
|
|
30
|
+
|
|
31
|
+
_lock = threading.Lock()
|
|
32
|
+
_jobs: Dict[str, Dict[str, Any]] = {}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _prune_locked(now: float) -> None:
|
|
36
|
+
stale = [
|
|
37
|
+
job_id
|
|
38
|
+
for job_id, job in _jobs.items()
|
|
39
|
+
if job["status"] != "running"
|
|
40
|
+
and job["ended_at"] is not None
|
|
41
|
+
and now - job["ended_at"] > MAX_JOB_AGE_SECONDS
|
|
42
|
+
]
|
|
43
|
+
for job_id in stale:
|
|
44
|
+
del _jobs[job_id]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _run(job_id: str, label: str, fn: Callable[[], Any]) -> None:
|
|
48
|
+
try:
|
|
49
|
+
with long_resolve_op(label):
|
|
50
|
+
result = fn()
|
|
51
|
+
except Exception as exc:
|
|
52
|
+
with _lock:
|
|
53
|
+
job = _jobs.get(job_id)
|
|
54
|
+
if job is not None:
|
|
55
|
+
job["status"] = "error"
|
|
56
|
+
job["error"] = f"{type(exc).__name__}: {exc}"
|
|
57
|
+
job["ended_at"] = time.time()
|
|
58
|
+
return
|
|
59
|
+
with _lock:
|
|
60
|
+
job = _jobs.get(job_id)
|
|
61
|
+
if job is not None:
|
|
62
|
+
job["status"] = "done"
|
|
63
|
+
job["result"] = result
|
|
64
|
+
job["ended_at"] = time.time()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def start_job(label: str, fn: Callable[[], Any]) -> str:
|
|
68
|
+
"""Run fn on a daemon thread; return a short job id immediately."""
|
|
69
|
+
now = time.time()
|
|
70
|
+
with _lock:
|
|
71
|
+
_prune_locked(now)
|
|
72
|
+
job_id = uuid.uuid4().hex[:8]
|
|
73
|
+
while job_id in _jobs:
|
|
74
|
+
job_id = uuid.uuid4().hex[:8]
|
|
75
|
+
_jobs[job_id] = {
|
|
76
|
+
"id": job_id,
|
|
77
|
+
"label": str(label),
|
|
78
|
+
"status": "running",
|
|
79
|
+
"result": None,
|
|
80
|
+
"error": None,
|
|
81
|
+
"started_at": now,
|
|
82
|
+
"ended_at": None,
|
|
83
|
+
}
|
|
84
|
+
threading.Thread(
|
|
85
|
+
target=_run,
|
|
86
|
+
args=(job_id, str(label), fn),
|
|
87
|
+
name=f"bgjob-{job_id}",
|
|
88
|
+
daemon=True,
|
|
89
|
+
).start()
|
|
90
|
+
return job_id
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def job_status(job_id: str) -> Optional[Dict[str, Any]]:
|
|
94
|
+
"""Snapshot of one job, or None when the id is unknown."""
|
|
95
|
+
with _lock:
|
|
96
|
+
_prune_locked(time.time())
|
|
97
|
+
job = _jobs.get(job_id)
|
|
98
|
+
return dict(job) if job is not None else None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def list_jobs() -> List[Dict[str, Any]]:
|
|
102
|
+
"""Compact status of every known job."""
|
|
103
|
+
with _lock:
|
|
104
|
+
_prune_locked(time.time())
|
|
105
|
+
return [
|
|
106
|
+
{k: job[k] for k in ("id", "label", "status", "started_at", "ended_at")}
|
|
107
|
+
for job in _jobs.values()
|
|
108
|
+
]
|
package/src/utils/deep_vision.py
CHANGED
|
@@ -123,37 +123,8 @@ def _now() -> str:
|
|
|
123
123
|
def _clip_context(project_root: str, clip_ref: Any) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
|
124
124
|
"""Resolve a clip to (context, error). Auto-ingests the JSON report when
|
|
125
125
|
the DB has no rows yet (pre-v9 analysis roots)."""
|
|
126
|
-
ma = _ma()
|
|
127
126
|
conn = timeline_brain_db.connect(project_root)
|
|
128
|
-
clip_uuid = analysis_store.
|
|
129
|
-
if not clip_uuid:
|
|
130
|
-
# Pre-v9 report? Find the analysis.json by walking clips/ and ingest it.
|
|
131
|
-
clips_root = os.path.join(project_root, "clips")
|
|
132
|
-
candidate = str(clip_ref or "")
|
|
133
|
-
if os.path.isdir(clips_root):
|
|
134
|
-
for entry in sorted(os.listdir(clips_root)):
|
|
135
|
-
report_path = os.path.join(clips_root, entry, "analysis.json")
|
|
136
|
-
if not os.path.isfile(report_path):
|
|
137
|
-
continue
|
|
138
|
-
try:
|
|
139
|
-
with open(report_path, "r", encoding="utf-8") as handle:
|
|
140
|
-
report = json.load(handle)
|
|
141
|
-
except (OSError, json.JSONDecodeError):
|
|
142
|
-
continue
|
|
143
|
-
clip_block = report.get("clip") or {}
|
|
144
|
-
if candidate not in (
|
|
145
|
-
entry,
|
|
146
|
-
str(clip_block.get("clip_id") or ""),
|
|
147
|
-
str(clip_block.get("media_id") or ""),
|
|
148
|
-
ma.normalize_path(clip_block.get("file_path") or ""),
|
|
149
|
-
):
|
|
150
|
-
continue
|
|
151
|
-
ingest = analysis_store.ingest_report(
|
|
152
|
-
project_root, report, clip_dir=os.path.join(clips_root, entry)
|
|
153
|
-
)
|
|
154
|
-
if ingest.get("success"):
|
|
155
|
-
clip_uuid = ingest["clip_uuid"]
|
|
156
|
-
break
|
|
127
|
+
clip_uuid = analysis_store.resolve_clip_uuid_ingesting(project_root, conn, clip_ref)
|
|
157
128
|
if not clip_uuid:
|
|
158
129
|
return None, f"No analyzed clip found for {clip_ref!r} (run db_ingest if this is an older analysis root)"
|
|
159
130
|
clip_row = conn.execute("SELECT * FROM clips WHERE clip_uuid = ?", (clip_uuid,)).fetchone()
|
package/src/utils/strata.py
CHANGED
|
@@ -21,12 +21,14 @@ from __future__ import annotations
|
|
|
21
21
|
import json
|
|
22
22
|
import logging
|
|
23
23
|
import math
|
|
24
|
+
import os
|
|
24
25
|
import sqlite3
|
|
25
26
|
import time
|
|
26
|
-
from array import array
|
|
27
27
|
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
|
28
28
|
|
|
29
29
|
from src.utils import timeline_brain_db
|
|
30
|
+
from src.utils.embeddings import pack_vector as _pack_vector
|
|
31
|
+
from src.utils.embeddings import unpack_vector as _unpack_vector
|
|
30
32
|
|
|
31
33
|
logger = logging.getLogger("resolve-mcp.strata")
|
|
32
34
|
|
|
@@ -60,20 +62,67 @@ def _dumps(value: Any) -> str:
|
|
|
60
62
|
return json.dumps(value, sort_keys=True, default=str)
|
|
61
63
|
|
|
62
64
|
|
|
63
|
-
# ──
|
|
65
|
+
# ── clip resolution — the ONE resolver for every strata surface ──────────────
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def resolve_clip(
|
|
69
|
+
project_root: str,
|
|
70
|
+
clip_ref: Any,
|
|
71
|
+
*,
|
|
72
|
+
require_media: bool = False,
|
|
73
|
+
) -> Tuple[Optional[sqlite3.Connection], Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
|
|
74
|
+
"""Resolve a clip ref to ``(conn, clip, error)`` — exactly one of
|
|
75
|
+
``clip``/``error`` is set (conn is always usable when clip is set).
|
|
76
|
+
|
|
77
|
+
``clip`` carries {clip_uuid, clip_name, file_path, duration_seconds, fps}.
|
|
78
|
+
Uses the same pre-v9 auto-ingest fallback as deep_vision, so a ref that
|
|
79
|
+
resolves for deepen resolves identically for every strata action. With
|
|
80
|
+
``require_media=True`` the clip's media file must exist on disk.
|
|
81
|
+
"""
|
|
82
|
+
from src.utils import analysis_store
|
|
83
|
+
|
|
84
|
+
conn = timeline_brain_db.connect(project_root)
|
|
85
|
+
clip_uuid = analysis_store.resolve_clip_uuid_ingesting(project_root, conn, clip_ref)
|
|
86
|
+
if not clip_uuid:
|
|
87
|
+
return conn, None, {
|
|
88
|
+
"success": False,
|
|
89
|
+
"error": f"Unknown clip ref: {clip_ref!r} (older analysis root? run db_ingest first)",
|
|
90
|
+
}
|
|
91
|
+
row = conn.execute(
|
|
92
|
+
"SELECT clip_uuid, clip_name, file_path, duration_seconds, fps FROM clips WHERE clip_uuid = ?",
|
|
93
|
+
(clip_uuid,),
|
|
94
|
+
).fetchone()
|
|
95
|
+
if row is None:
|
|
96
|
+
return conn, None, {"success": False, "error": f"No clips row for {clip_uuid}"}
|
|
97
|
+
clip = {
|
|
98
|
+
"clip_uuid": clip_uuid,
|
|
99
|
+
"clip_name": row["clip_name"],
|
|
100
|
+
"file_path": row["file_path"],
|
|
101
|
+
"duration_seconds": row["duration_seconds"],
|
|
102
|
+
"fps": row["fps"],
|
|
103
|
+
}
|
|
104
|
+
if require_media:
|
|
105
|
+
file_path = clip["file_path"]
|
|
106
|
+
if not file_path or not os.path.isfile(file_path):
|
|
107
|
+
return conn, None, {
|
|
108
|
+
"success": False,
|
|
109
|
+
"error": f"Media file not accessible for clip {clip['clip_name']!r}: {file_path!r}",
|
|
110
|
+
"clip_uuid": clip_uuid,
|
|
111
|
+
}
|
|
112
|
+
return conn, clip, None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# ── float32 blob convention — ONE codec, shared with embeddings.vector ───────
|
|
64
116
|
|
|
65
117
|
|
|
66
118
|
def pack_curve(values: Sequence[float]) -> bytes:
|
|
67
119
|
"""Encode a float sequence as a little-endian float32 BLOB."""
|
|
68
|
-
|
|
69
|
-
return arr.tobytes()
|
|
120
|
+
return _pack_vector([float(v) for v in values])
|
|
70
121
|
|
|
71
122
|
|
|
72
123
|
def unpack_curve(blob: bytes) -> List[float]:
|
|
73
124
|
"""Decode a float32 BLOB back to a list of floats."""
|
|
74
|
-
|
|
75
|
-
arr.frombytes(blob)
|
|
76
|
-
return list(arr)
|
|
125
|
+
return _unpack_vector(blob)
|
|
77
126
|
|
|
78
127
|
|
|
79
128
|
def curve_stats(values: Sequence[float]) -> Dict[str, Any]:
|
|
@@ -230,9 +279,23 @@ def read_events(
|
|
|
230
279
|
# inside it — a 3 s pause that began before the window is still a pause.
|
|
231
280
|
# Point events (no duration) use the half-open [start, end) convention.
|
|
232
281
|
if start_seconds is not None:
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
282
|
+
start = float(start_seconds)
|
|
283
|
+
# A bare OR on the overlap test is non-sargable (ix_events_clip_track
|
|
284
|
+
# could never range-seek on time_seconds). Bound the lower edge by the
|
|
285
|
+
# track's longest span instead: no overlapping event can start earlier
|
|
286
|
+
# than start - MAX(duration). The MAX probe is a single b-tree descent
|
|
287
|
+
# via ix_events_clip_track_span (v15).
|
|
288
|
+
span_sql = "SELECT MAX(duration_seconds) FROM events WHERE clip_uuid = ?"
|
|
289
|
+
span_args: List[Any] = [clip_uuid]
|
|
290
|
+
if track:
|
|
291
|
+
span_sql += " AND track = ?"
|
|
292
|
+
span_args.append(track)
|
|
293
|
+
max_span = conn.execute(span_sql, span_args).fetchone()[0] or 0.0
|
|
294
|
+
sql += (
|
|
295
|
+
" AND time_seconds >= ?"
|
|
296
|
+
" AND (time_seconds >= ? OR time_seconds + COALESCE(duration_seconds, 0) > ?)"
|
|
297
|
+
)
|
|
298
|
+
args.extend([start - float(max_span), start, start])
|
|
236
299
|
if end_seconds is not None:
|
|
237
300
|
sql += " AND time_seconds < ?"
|
|
238
301
|
args.append(float(end_seconds))
|
|
@@ -476,13 +539,19 @@ def backfill_transcript_words(project_root: str) -> Dict[str, Any]:
|
|
|
476
539
|
re-running any analysis. Idempotent.
|
|
477
540
|
"""
|
|
478
541
|
conn = timeline_brain_db.connect(project_root)
|
|
479
|
-
|
|
480
|
-
clips_seen = 0
|
|
542
|
+
clips_seen = int(conn.execute("SELECT COUNT(*) FROM analysis_reports").fetchone()[0])
|
|
481
543
|
clips_with_words = 0
|
|
482
544
|
words_written = 0
|
|
483
545
|
with timeline_brain_db.transaction(project_root) as txn:
|
|
484
|
-
|
|
485
|
-
|
|
546
|
+
# Lazy cursor + substring prefilter: never hold every multi-MB report
|
|
547
|
+
# blob in memory at once, and skip parsing reports that cannot carry
|
|
548
|
+
# words. Reads and writes share the cached connection but touch
|
|
549
|
+
# different tables, so interleaving is safe.
|
|
550
|
+
cursor = txn.execute(
|
|
551
|
+
"SELECT clip_uuid, report_json FROM analysis_reports "
|
|
552
|
+
"WHERE report_json LIKE '%\"transcription\"%'"
|
|
553
|
+
)
|
|
554
|
+
for row in cursor:
|
|
486
555
|
try:
|
|
487
556
|
report = json.loads(row["report_json"])
|
|
488
557
|
except (TypeError, ValueError):
|
|
@@ -507,14 +576,12 @@ def backfill_transcript_words(project_root: str) -> Dict[str, Any]:
|
|
|
507
576
|
|
|
508
577
|
def strata_status(project_root: str, clip_ref: Any = None) -> Dict[str, Any]:
|
|
509
578
|
"""Project-level (or per-clip) strata inventory."""
|
|
510
|
-
from src.utils import analysis_store
|
|
511
|
-
|
|
512
579
|
conn = timeline_brain_db.connect(project_root)
|
|
513
580
|
if clip_ref is not None:
|
|
514
|
-
|
|
515
|
-
if
|
|
516
|
-
return
|
|
517
|
-
return {"success": True, "clip_uuid": clip_uuid, **list_tracks(conn, clip_uuid)}
|
|
581
|
+
conn, clip, err = resolve_clip(project_root, clip_ref)
|
|
582
|
+
if err:
|
|
583
|
+
return err
|
|
584
|
+
return {"success": True, "clip_uuid": clip["clip_uuid"], **list_tracks(conn, clip["clip_uuid"])}
|
|
518
585
|
|
|
519
586
|
def _count(sql: str) -> int:
|
|
520
587
|
return int(conn.execute(sql).fetchone()[0])
|
|
@@ -17,10 +17,8 @@ and DB writes.
|
|
|
17
17
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
-
import json
|
|
21
20
|
import logging
|
|
22
21
|
import math
|
|
23
|
-
import os
|
|
24
22
|
import shutil
|
|
25
23
|
import subprocess
|
|
26
24
|
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
@@ -71,33 +69,30 @@ def _ensure_tool_path() -> None:
|
|
|
71
69
|
|
|
72
70
|
|
|
73
71
|
def capabilities() -> Dict[str, Any]:
|
|
74
|
-
"""What the local analyzers can run on this machine.
|
|
72
|
+
"""What the local analyzers can run on this machine.
|
|
73
|
+
|
|
74
|
+
Derived from the ANALYZERS registry — adding an analyzer there is the
|
|
75
|
+
single edit; this report and run_analyzers' default set follow.
|
|
76
|
+
"""
|
|
75
77
|
_ensure_tool_path()
|
|
76
78
|
ffmpeg = shutil.which("ffmpeg")
|
|
79
|
+
have = {"ffmpeg": bool(ffmpeg), "numpy": _np is not None}
|
|
80
|
+
analyzers: Dict[str, Any] = {}
|
|
81
|
+
for name, spec in ANALYZERS.items():
|
|
82
|
+
probe = spec.get("capability")
|
|
83
|
+
if probe is not None:
|
|
84
|
+
analyzers[name] = probe()
|
|
85
|
+
continue
|
|
86
|
+
requires = spec["requires"]
|
|
87
|
+
analyzers[name] = {
|
|
88
|
+
"available": all(have.get(req, False) for req in requires),
|
|
89
|
+
"requires": list(requires),
|
|
90
|
+
"writes": spec["writes"],
|
|
91
|
+
}
|
|
77
92
|
return {
|
|
78
93
|
"ffmpeg": {"available": bool(ffmpeg), "path": ffmpeg},
|
|
79
94
|
"numpy": {"available": _np is not None},
|
|
80
|
-
"analyzers":
|
|
81
|
-
"prosody": {
|
|
82
|
-
"available": bool(ffmpeg) and _np is not None,
|
|
83
|
-
"requires": ["ffmpeg", "numpy"],
|
|
84
|
-
"writes": {
|
|
85
|
-
"curves": ["pitch", "vocal_energy", "speech_rate"],
|
|
86
|
-
"events": ["pause", "breath", "hesitation"],
|
|
87
|
-
},
|
|
88
|
-
},
|
|
89
|
-
"beat_grid": {
|
|
90
|
-
"available": bool(ffmpeg) and _np is not None,
|
|
91
|
-
"requires": ["ffmpeg", "numpy"],
|
|
92
|
-
"writes": {"events": ["beat", "downbeat"]},
|
|
93
|
-
},
|
|
94
|
-
"motion_energy": {
|
|
95
|
-
"available": bool(ffmpeg),
|
|
96
|
-
"requires": ["ffmpeg"],
|
|
97
|
-
"writes": {"curves": ["motion_energy"]},
|
|
98
|
-
},
|
|
99
|
-
"face": _face_capability(),
|
|
100
|
-
},
|
|
95
|
+
"analyzers": analyzers,
|
|
101
96
|
}
|
|
102
97
|
|
|
103
98
|
|
|
@@ -126,36 +121,10 @@ def _require(*needs: str) -> Optional[Dict[str, Any]]:
|
|
|
126
121
|
|
|
127
122
|
|
|
128
123
|
def _clip_row(project_root: str, clip_ref: Any) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
|
|
129
|
-
"""Resolve a clip ref to {clip_uuid, file_path, duration_seconds, fps}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
clip_uuid = analysis_store.resolve_clip_uuid(conn, clip_ref)
|
|
134
|
-
if not clip_uuid:
|
|
135
|
-
return None, {
|
|
136
|
-
"success": False,
|
|
137
|
-
"error": f"Unknown clip ref: {clip_ref!r} (older analysis root? run db_ingest first)",
|
|
138
|
-
}
|
|
139
|
-
row = conn.execute("SELECT * FROM clips WHERE clip_uuid = ?", (clip_uuid,)).fetchone()
|
|
140
|
-
if row is None:
|
|
141
|
-
return None, {"success": False, "error": f"No clips row for {clip_uuid}"}
|
|
142
|
-
file_path = row["file_path"]
|
|
143
|
-
if not file_path or not os.path.isfile(file_path):
|
|
144
|
-
return None, {
|
|
145
|
-
"success": False,
|
|
146
|
-
"error": f"Media file not accessible for clip {row['clip_name']!r}: {file_path!r}",
|
|
147
|
-
"clip_uuid": clip_uuid,
|
|
148
|
-
}
|
|
149
|
-
return (
|
|
150
|
-
{
|
|
151
|
-
"clip_uuid": clip_uuid,
|
|
152
|
-
"clip_name": row["clip_name"],
|
|
153
|
-
"file_path": file_path,
|
|
154
|
-
"duration_seconds": row["duration_seconds"],
|
|
155
|
-
"fps": row["fps"],
|
|
156
|
-
},
|
|
157
|
-
None,
|
|
158
|
-
)
|
|
124
|
+
"""Resolve a clip ref to {clip_uuid, file_path, duration_seconds, fps};
|
|
125
|
+
analyzers read the media file, so it must exist."""
|
|
126
|
+
_conn, clip, err = strata.resolve_clip(project_root, clip_ref, require_media=True)
|
|
127
|
+
return clip, err
|
|
159
128
|
|
|
160
129
|
|
|
161
130
|
# ── decoding ─────────────────────────────────────────────────────────────────
|
|
@@ -179,6 +148,40 @@ def decode_audio(path: str, sample_rate: int = AUDIO_SAMPLE_RATE, timeout: int =
|
|
|
179
148
|
return _np.frombuffer(proc.stdout, dtype=_np.float32)
|
|
180
149
|
|
|
181
150
|
|
|
151
|
+
def _audio_context(
|
|
152
|
+
project_root: str,
|
|
153
|
+
clip_ref: Any,
|
|
154
|
+
*,
|
|
155
|
+
clip: Optional[Dict[str, Any]] = None,
|
|
156
|
+
samples: "Any" = None,
|
|
157
|
+
) -> Tuple[Optional[Dict[str, Any]], "Any", Optional[Dict[str, Any]]]:
|
|
158
|
+
"""The shared audio-analyzer preamble: deps → clip row → decoded samples.
|
|
159
|
+
|
|
160
|
+
Returns (clip, samples, error). Pass a previously resolved ``clip`` and
|
|
161
|
+
decoded ``samples`` to skip the ffmpeg decode — run_analyzers uses this
|
|
162
|
+
so prosody and beat_grid share one decode of the same media file.
|
|
163
|
+
"""
|
|
164
|
+
missing = _require("ffmpeg", "numpy")
|
|
165
|
+
if missing:
|
|
166
|
+
return None, None, missing
|
|
167
|
+
if clip is None:
|
|
168
|
+
clip, err = _clip_row(project_root, clip_ref)
|
|
169
|
+
if err:
|
|
170
|
+
return None, None, err
|
|
171
|
+
if samples is None:
|
|
172
|
+
try:
|
|
173
|
+
samples = decode_audio(clip["file_path"])
|
|
174
|
+
except (RuntimeError, subprocess.TimeoutExpired) as exc:
|
|
175
|
+
return None, None, {"success": False, "error": str(exc), "clip_uuid": clip["clip_uuid"]}
|
|
176
|
+
if samples.size == 0:
|
|
177
|
+
return None, None, {
|
|
178
|
+
"success": False,
|
|
179
|
+
"error": "clip has no decodable audio",
|
|
180
|
+
"clip_uuid": clip["clip_uuid"],
|
|
181
|
+
}
|
|
182
|
+
return clip, samples, None
|
|
183
|
+
|
|
184
|
+
|
|
182
185
|
# ── prosody: pure compute ────────────────────────────────────────────────────
|
|
183
186
|
|
|
184
187
|
|
|
@@ -326,7 +329,9 @@ def detect_breaths(
|
|
|
326
329
|
"""
|
|
327
330
|
if not len(energy):
|
|
328
331
|
return []
|
|
329
|
-
|
|
332
|
+
# numpy assumed: every caller sits behind _require("numpy"), matching the
|
|
333
|
+
# sibling compute functions in this module.
|
|
334
|
+
arr = _np.asarray(energy, dtype=_np.float64)
|
|
330
335
|
breaths: List[Dict[str, Any]] = []
|
|
331
336
|
for pause in pauses:
|
|
332
337
|
start = float(pause["time_seconds"])
|
|
@@ -335,9 +340,9 @@ def detect_breaths(
|
|
|
335
340
|
hi = min(int((start + dur) * curve_rate), len(energy))
|
|
336
341
|
if hi - lo < 3:
|
|
337
342
|
continue
|
|
338
|
-
seg = arr[lo:hi]
|
|
339
|
-
floor = float(_np.percentile(seg, 20))
|
|
340
|
-
peak_idx = int(_np.argmax(seg))
|
|
343
|
+
seg = arr[lo:hi]
|
|
344
|
+
floor = float(_np.percentile(seg, 20))
|
|
345
|
+
peak_idx = int(_np.argmax(seg))
|
|
341
346
|
peak = float(seg[peak_idx])
|
|
342
347
|
# Bump: clearly above the gap floor, clearly below speech (~1.0 scale).
|
|
343
348
|
if peak >= floor + 0.05 and peak <= 0.5:
|
|
@@ -354,29 +359,26 @@ def detect_breaths(
|
|
|
354
359
|
# ── prosody: runner ──────────────────────────────────────────────────────────
|
|
355
360
|
|
|
356
361
|
|
|
357
|
-
def run_prosody(
|
|
362
|
+
def run_prosody(
|
|
363
|
+
project_root: str,
|
|
364
|
+
clip_ref: Any,
|
|
365
|
+
*,
|
|
366
|
+
clip: Optional[Dict[str, Any]] = None,
|
|
367
|
+
samples: "Any" = None,
|
|
368
|
+
) -> Dict[str, Any]:
|
|
358
369
|
"""Compute + persist prosody strata for one clip.
|
|
359
370
|
|
|
360
371
|
Writes curves pitch / vocal_energy / speech_rate and events pause /
|
|
361
372
|
breath / hesitation. Requires the clip's media file, ffmpeg, numpy, and
|
|
362
|
-
(for word-derived tracks) transcript_words rows.
|
|
373
|
+
(for word-derived tracks) transcript_words rows. ``clip``/``samples``
|
|
374
|
+
accept a pre-resolved row and pre-decoded audio (see _audio_context).
|
|
363
375
|
"""
|
|
364
|
-
|
|
365
|
-
if missing:
|
|
366
|
-
return missing
|
|
367
|
-
clip, err = _clip_row(project_root, clip_ref)
|
|
376
|
+
clip, samples, err = _audio_context(project_root, clip_ref, clip=clip, samples=samples)
|
|
368
377
|
if err:
|
|
369
378
|
return err
|
|
370
379
|
conn = timeline_brain_db.connect(project_root)
|
|
371
380
|
words = strata.read_words(conn, clip["clip_uuid"])
|
|
372
381
|
|
|
373
|
-
try:
|
|
374
|
-
samples = decode_audio(clip["file_path"])
|
|
375
|
-
except (RuntimeError, subprocess.TimeoutExpired) as exc:
|
|
376
|
-
return {"success": False, "error": str(exc), "clip_uuid": clip["clip_uuid"]}
|
|
377
|
-
if samples.size == 0:
|
|
378
|
-
return {"success": False, "error": "clip has no decodable audio", "clip_uuid": clip["clip_uuid"]}
|
|
379
|
-
|
|
380
382
|
duration = clip["duration_seconds"] or (samples.size / AUDIO_SAMPLE_RATE)
|
|
381
383
|
energy = compute_energy_curve(samples)
|
|
382
384
|
pitch = compute_pitch_curve(samples, energy=energy)
|
|
@@ -447,7 +449,6 @@ def compute_beat_grid(
|
|
|
447
449
|
Downbeats are a phase-of-strongest-onset estimate every 4 beats —
|
|
448
450
|
explicitly low-confidence (no meter detection).
|
|
449
451
|
"""
|
|
450
|
-
hop = int(sample_rate * HOP_SECONDS)
|
|
451
452
|
win = int(sample_rate * FRAME_SECONDS)
|
|
452
453
|
frames, _, _ = _frame_signal(samples, sample_rate)
|
|
453
454
|
if frames.shape[0] < 16:
|
|
@@ -500,20 +501,20 @@ def compute_beat_grid(
|
|
|
500
501
|
}
|
|
501
502
|
|
|
502
503
|
|
|
503
|
-
def run_beat_grid(
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
504
|
+
def run_beat_grid(
|
|
505
|
+
project_root: str,
|
|
506
|
+
clip_ref: Any,
|
|
507
|
+
*,
|
|
508
|
+
clip: Optional[Dict[str, Any]] = None,
|
|
509
|
+
samples: "Any" = None,
|
|
510
|
+
) -> Dict[str, Any]:
|
|
511
|
+
"""Compute + persist the musical beat grid for one clip.
|
|
512
|
+
|
|
513
|
+
``clip``/``samples`` accept a pre-resolved row and pre-decoded audio
|
|
514
|
+
(see _audio_context)."""
|
|
515
|
+
clip, samples, err = _audio_context(project_root, clip_ref, clip=clip, samples=samples)
|
|
509
516
|
if err:
|
|
510
517
|
return err
|
|
511
|
-
try:
|
|
512
|
-
samples = decode_audio(clip["file_path"])
|
|
513
|
-
except (RuntimeError, subprocess.TimeoutExpired) as exc:
|
|
514
|
-
return {"success": False, "error": str(exc), "clip_uuid": clip["clip_uuid"]}
|
|
515
|
-
if samples.size == 0:
|
|
516
|
-
return {"success": False, "error": "clip has no decodable audio", "clip_uuid": clip["clip_uuid"]}
|
|
517
518
|
|
|
518
519
|
grid = compute_beat_grid(samples)
|
|
519
520
|
beat_events = [
|
|
@@ -641,11 +642,36 @@ def _run_face(project_root: str, clip_ref: Any) -> Dict[str, Any]:
|
|
|
641
642
|
return strata_faces.run_face_strata(project_root, clip_ref)
|
|
642
643
|
|
|
643
644
|
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
645
|
+
# The single analyzer registry: run function + dependency/track metadata.
|
|
646
|
+
# capabilities() and run_analyzers' default set derive from it, so a new
|
|
647
|
+
# analyzer is one entry here (or a "capability" probe for stacks that
|
|
648
|
+
# self-describe, like face). "audio" marks analyzers that consume the shared
|
|
649
|
+
# mono PCM decode.
|
|
650
|
+
ANALYZERS: Dict[str, Dict[str, Any]] = {
|
|
651
|
+
"prosody": {
|
|
652
|
+
"run": run_prosody,
|
|
653
|
+
"requires": ("ffmpeg", "numpy"),
|
|
654
|
+
"writes": {
|
|
655
|
+
"curves": ["pitch", "vocal_energy", "speech_rate"],
|
|
656
|
+
"events": ["pause", "breath", "hesitation"],
|
|
657
|
+
},
|
|
658
|
+
"audio": True,
|
|
659
|
+
},
|
|
660
|
+
"beat_grid": {
|
|
661
|
+
"run": run_beat_grid,
|
|
662
|
+
"requires": ("ffmpeg", "numpy"),
|
|
663
|
+
"writes": {"events": ["beat", "downbeat"]},
|
|
664
|
+
"audio": True,
|
|
665
|
+
},
|
|
666
|
+
"motion_energy": {
|
|
667
|
+
"run": run_motion_energy,
|
|
668
|
+
"requires": ("ffmpeg",),
|
|
669
|
+
"writes": {"curves": ["motion_energy"]},
|
|
670
|
+
},
|
|
671
|
+
"face": {
|
|
672
|
+
"run": _run_face,
|
|
673
|
+
"capability": _face_capability,
|
|
674
|
+
},
|
|
649
675
|
}
|
|
650
676
|
|
|
651
677
|
|
|
@@ -658,7 +684,8 @@ def run_analyzers(
|
|
|
658
684
|
|
|
659
685
|
Explicitly-requested analyzers run (and fail loudly if deps are
|
|
660
686
|
missing); the default set runs only what this machine can run, and
|
|
661
|
-
names what it skipped.
|
|
687
|
+
names what it skipped. When several audio analyzers run, the media
|
|
688
|
+
file is decoded once and the samples are shared.
|
|
662
689
|
"""
|
|
663
690
|
caps = capabilities()["analyzers"]
|
|
664
691
|
if analyzers:
|
|
@@ -674,7 +701,24 @@ def run_analyzers(
|
|
|
674
701
|
"error": f"Unknown analyzer(s): {', '.join(unknown)}",
|
|
675
702
|
"available": sorted(ANALYZERS),
|
|
676
703
|
}
|
|
677
|
-
|
|
704
|
+
|
|
705
|
+
audio_names = [n for n in names if ANALYZERS[n].get("audio")]
|
|
706
|
+
shared_clip: Optional[Dict[str, Any]] = None
|
|
707
|
+
shared_samples: "Any" = None
|
|
708
|
+
shared_err: Optional[Dict[str, Any]] = None
|
|
709
|
+
if len(audio_names) > 1:
|
|
710
|
+
shared_clip, shared_samples, shared_err = _audio_context(project_root, clip_ref)
|
|
711
|
+
|
|
712
|
+
results: Dict[str, Any] = {}
|
|
713
|
+
for name in names:
|
|
714
|
+
if name in audio_names and shared_err is not None:
|
|
715
|
+
results[name] = shared_err
|
|
716
|
+
elif name in audio_names and shared_clip is not None:
|
|
717
|
+
results[name] = ANALYZERS[name]["run"](
|
|
718
|
+
project_root, clip_ref, clip=shared_clip, samples=shared_samples
|
|
719
|
+
)
|
|
720
|
+
else:
|
|
721
|
+
results[name] = ANALYZERS[name]["run"](project_root, clip_ref)
|
|
678
722
|
out: Dict[str, Any] = {
|
|
679
723
|
"success": all(r.get("success") for r in results.values()),
|
|
680
724
|
"results": results,
|