davinci-resolve-mcp 2.68.1 → 2.69.0

package/src/utils/beat_detection.py

@@ -0,0 +1,242 @@
+"""Beat detection for music-driven cutting.
+
+The single most-requested missing capability from the first public audience for
+this project. Music-video makers, live-music channels and motorsport editors all
+asked variants of the same thing, and one put the failure precisely: the tools
+here only cut interviews, they do not cut to music. A viewer editing drift
+footage predicted the exact consequence of pointing a speech-silence gate at his
+material — engine noise and smoke read as "dead space", and the tool cuts in all
+the wrong places.
+
+They are right, and the reason is structural. Everything else in this codebase
+finds edit points in *speech*: word boundaries, pauses, fillers. Music has none
+of those. It has pulse, and pulse is a completely different measurement.
+
+## What this gives you
+
+- **Beats** — the pulse. Cutting on every beat is relentless and usually wrong.
+- **Phrases** — beats grouped into bars and bars into phrases (typically 4 or 8
+  bars). Music resolves at phrase boundaries, which is where an edit feels
+  *inevitable* rather than merely synchronised. This is the one that matters.
+- **Frame-snapped cut points**, because a cut two frames off the beat reads as a
+  mistake rather than as syncopation.
+
+## Honest limits
+
+Tempo tracking is reliable on steady, percussive material and degrades on rubato,
+live performance without a strong downbeat, and anything heavily swung. The
+detector reports its own confidence and this module surfaces it rather than
+presenting every result as equally solid.
+
+Downbeats are **inferred**, not detected: the first beat is assumed to start a
+bar. That assumption is wrong whenever a track has a pickup, and it is the first
+thing to check if the cuts feel consistently one beat early. `beat_offset` exists
+to correct it.
+
+## Dependency
+
+`librosa` (ISC) is an **optional extra** — it is not in `requirements.txt`, it
+pulls a large scientific stack, and this module honest-refuses without it rather
+than degrading into a guess. Install with `pip install librosa`.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from typing import Any, Dict, List, Sequence
+
+#: Beats per bar. 4/4 covers the overwhelming majority of cuttable music; the
+#: parameter exists because 3/4 and 6/8 do turn up and silently forcing them
+#: into 4 produces phrase boundaries that land in musically wrong places.
+DEFAULT_BEATS_PER_BAR = 4
+#: Bars per phrase. 8 bars is the common pop/dance period; 4 suits shorter cuts.
+DEFAULT_BARS_PER_PHRASE = 8
+
+
+def librosa_available() -> bool:
+    """Presence check without importing — librosa costs seconds to import."""
+    try:
+        return importlib.util.find_spec("librosa") is not None
+    except (ImportError, ValueError):
+        return False
+
+
+def detect_beats(media_path: str, *, sample_rate: int = 22050) -> Dict[str, Any]:
+    """Tempo and beat times for an audio or video file.
+
+    Honest-refuses when librosa is absent. A fabricated tempo would produce cuts
+    that are confidently, rhythmically wrong — worse than no feature, because
+    the failure looks intentional.
+    """
+    if not librosa_available():
+        return {
+            "success": False,
+            "error": "librosa is not installed — beat detection is unavailable",
+            "remediation": (
+                "pip install librosa  (ISC licensed, optional extra; it pulls "
+                "numpy/scipy/numba and is deliberately not a core dependency)"
+            ),
+        }
+    import librosa
+
+    try:
+        y, sr = librosa.load(media_path, sr=sample_rate, mono=True)
+    except Exception as exc:
+        return {"success": False, "error": f"could not read audio from {media_path}: {exc}"}
+    if y is None or len(y) == 0:
+        return {"success": False, "error": "no audio samples decoded — is there an audio track?"}
+
+    try:
+        tempo, beat_frames = librosa.beat.beat_track(y=y, sr=sr, units="frames")
+        beat_times = librosa.frames_to_time(beat_frames, sr=sr)
+        onset_env = librosa.onset.onset_strength(y=y, sr=sr)
+    except Exception as exc:
+        return {"success": False, "error": f"beat tracking failed: {exc}"}
+
+    beats = [round(float(t), 4) for t in beat_times]
+    tempo_value = float(tempo if not hasattr(tempo, "__len__") else (tempo[0] if len(tempo) else 0.0))
+    return {
+        "success": True,
+        "tempo_bpm": round(tempo_value, 2),
+        "beat_count": len(beats),
+        "beats": beats,
+        "duration_seconds": round(float(len(y)) / sr, 3),
+        "confidence": _confidence(beats, onset_env),
+    }
+
+
+def _confidence(beats: Sequence[float], onset_env) -> Dict[str, Any]:
+    """How regular the detected pulse is.
+
+    A steady grid means the tracker locked on. Wildly varying intervals mean it
+    did not, and the caller should be told rather than handed a tidy-looking
+    list of times that happens to be wrong.
+    """
+    if len(beats) < 4:
+        return {"band": "low", "reason": "too few beats detected to judge regularity"}
+    intervals = [b - a for a, b in zip(beats, beats[1:])]
+    mean = sum(intervals) / len(intervals)
+    if mean <= 0:
+        return {"band": "low", "reason": "degenerate beat spacing"}
+    variance = sum((i - mean) ** 2 for i in intervals) / len(intervals)
+    jitter = (variance ** 0.5) / mean
+    if jitter < 0.05:
+        band, reason = "high", "beat spacing is steady — the tracker locked on"
+    elif jitter < 0.15:
+        band, reason = "medium", "some drift in beat spacing; check a few cuts against the music"
+    else:
+        band, reason = "low", (
+            "beat spacing is irregular — rubato, live playing or a weak downbeat. "
+            "Treat these cut points as suggestions"
+        )
+    return {"band": band, "jitter": round(jitter, 4), "reason": reason}
+
+
+def group_into_phrases(
+    beats: Sequence[float],
+    *,
+    beats_per_bar: int = DEFAULT_BEATS_PER_BAR,
+    bars_per_phrase: int = DEFAULT_BARS_PER_PHRASE,
+    beat_offset: int = 0,
+) -> Dict[str, Any]:
+    """Group beats into bars and phrases. Pure — no librosa needed.
+
+    `beat_offset` shifts which beat is treated as the first of a bar. Downbeats
+    are inferred rather than detected, so a track with a pickup will be off by
+    one until this is set; that is the first knob to reach for when cuts feel
+    consistently early.
+    """
+    if beats_per_bar < 1 or bars_per_phrase < 1:
+        raise ValueError("beats_per_bar and bars_per_phrase must be >= 1")
+    ordered = sorted(float(b) for b in beats)
+    per_phrase = beats_per_bar * bars_per_phrase
+    downbeats: List[float] = []
+    phrase_starts: List[float] = []
+    for index, time in enumerate(ordered):
+        position = index - beat_offset
+        if position < 0:
+            continue
+        if position % beats_per_bar == 0:
+            downbeats.append(round(time, 4))
+        if position % per_phrase == 0:
+            phrase_starts.append(round(time, 4))
+    return {
+        "beats_per_bar": beats_per_bar,
+        "bars_per_phrase": bars_per_phrase,
+        "beat_offset": beat_offset,
+        "downbeats": downbeats,
+        "phrase_starts": phrase_starts,
+        "bar_count": len(downbeats),
+        "phrase_count": len(phrase_starts),
+        "note": (
+            "Downbeats are INFERRED — the first beat is assumed to start a bar. "
+            "A track with a pickup will be off by one; correct it with beat_offset."
+        ),
+    }
+
+
+def snap_to_frames(times: Sequence[float], fps: float) -> List[int]:
+    """Times to frame numbers. A cut two frames off the beat reads as a mistake."""
+    if fps <= 0:
+        raise ValueError("fps must be positive")
+    return [int(round(float(t) * fps)) for t in times]
+
+
+def plan_beat_cuts(
+    beats: Sequence[float],
+    *,
+    fps: float,
+    mode: str = "phrase",
+    beats_per_bar: int = DEFAULT_BEATS_PER_BAR,
+    bars_per_phrase: int = DEFAULT_BARS_PER_PHRASE,
+    beat_offset: int = 0,
+    min_shot_seconds: float = 0.0,
+) -> Dict[str, Any]:
+    """Cut points on the beat, the bar, or the phrase. Pure and testable.
+
+    `mode`:
+      - `beat`   — every beat. Relentless; suits montage stings, little else.
+      - `bar`    — every downbeat.
+      - `phrase` — every phrase start (**default**). Music resolves here, which
+        is what makes a cut feel inevitable rather than merely synchronised.
+    """
+    if mode not in ("beat", "bar", "phrase"):
+        raise ValueError(f"unknown mode {mode!r}; expected beat, bar or phrase")
+    grouped = group_into_phrases(
+        beats, beats_per_bar=beats_per_bar,
+        bars_per_phrase=bars_per_phrase, beat_offset=beat_offset,
+    )
+    times = {
+        "beat": sorted(float(b) for b in beats),
+        "bar": grouped["downbeats"],
+        "phrase": grouped["phrase_starts"],
+    }[mode]
+
+    dropped = 0
+    if min_shot_seconds > 0 and times:
+        kept = [times[0]]
+        for time in times[1:]:
+            if time - kept[-1] >= min_shot_seconds:
+                kept.append(time)
+            else:
+                dropped += 1
+        times = kept
+
+    return {
+        "success": True,
+        "kind": "beat_cuts",
+        "mode": mode,
+        "cut_count": len(times),
+        "cut_times": [round(t, 4) for t in times],
+        "cut_frames": snap_to_frames(times, fps),
+        "grouping": grouped,
+        "dropped_for_min_shot": dropped,
+        "note": (
+            f"Cut points on every {mode}. "
+            + (f"{dropped} were dropped to honour min_shot_seconds={min_shot_seconds}. "
+               if dropped else "")
+            + "Frame-snapped: a cut two frames off the beat reads as a mistake rather "
+            "than syncopation. These are cut POINTS, not an assembly — what goes "
+            "between them is yours."
+        ),
+    }

package/src/utils/bridge_differential.py

@@ -182,12 +182,22 @@ def probe_surface(roots: Sequence[str] = SOURCE_ROOTS) -> Dict[str, Any]:
 #: MediaPoolItem or Graph — which between them carry most of the read surface
 #: worth qualifying (LUTs, node graphs, source timings, metadata).
 _INDEX = "[]"
+#: Pick the first element of a returned list for which `method()` is truthy.
+#: Indexing blindly is not good enough: a timeline's first video item is often a
+#: generator or a title, which has no MediaPoolItem and answers None to every
+#: node-graph question. Measured on a real timeline — `GetNumNodes` and
+#: `GetNodeGraph` both None on an SMPTE Color Bar, and 1 and a live graph on the
+#: clip sitting next to it. Taking index 0 therefore reported the whole grading
+#: surface as unreachable, which is the silent coverage gap this module exists to
+#: refuse.
+_FIRST_WITH = "[first-with]"
 
 _PM = ("GetProjectManager", ())
 _PROJECT = (_PM, ("GetCurrentProject", ()))
 _POOL = _PROJECT + (("GetMediaPool", ()),)
 _TIMELINE = _PROJECT + (("GetCurrentTimeline", ()),)
-_TIMELINE_ITEM = _TIMELINE + (("GetItemListInTrack", ("video", 1)), (_INDEX, (0,)))
+_TIMELINE_ITEM = _TIMELINE + (("GetItemListInTrack", ("video", 1)),
+                              (_FIRST_WITH, ("GetMediaPoolItem",)))
 _GALLERY = _PROJECT + (("GetGallery", ()),)
 
 OBJECT_GRAPH: Tuple[Tuple[str, Tuple[Any, ...]], ...] = (
@@ -229,6 +239,22 @@ def walk(resolve: Any, chain: Iterable[Tuple[str, Tuple[Any, ...]]]) -> Any:
                 return None
             current = current[index]
             continue
+        if method == _FIRST_WITH:
+            if not isinstance(current, (list, tuple)):
+                return None
+            probe = args[0]
+            chosen = None
+            for candidate in current:
+                try:
+                    if getattr(candidate, probe, lambda: None)():
+                        chosen = candidate
+                        break
+                except Exception:  # noqa: BLE001 - an unusable candidate, not a verdict
+                    continue
+            if chosen is None:
+                return None
+            current = chosen
+            continue
         function = getattr(current, method, None)
         if function is None:
             return None

package/src/utils/broll_placement.py

@@ -0,0 +1,201 @@
+"""Placing B-roll against an A-roll spine.
+
+Asked for repeatedly, and the clearest version came from someone making recipe
+videos: *could it find the one second where I am putting the spoon in my mouth?*
+That is a `find_similar` query against visual strata pointed at a specific
+moment in a voiceover, and the machinery for both halves already exists — it has
+simply never been joined up.
+
+## The shape of the problem
+
+An A-roll spine (a voiceover or a talking head) has **beats**: sentences,
+clauses, topic shifts. B-roll goes over some of them. Which B-roll goes where is
+a content question, answered by whatever similarity the caller supplies; this
+module answers the *placement* question, which is different and mechanical:
+
+- **Where can B-roll go at all** — over a beat, not across one. Cutting away
+  mid-sentence and back is a specific effect, not a default.
+- **How long can it stay** — bounded by the beat, and by how long the candidate
+  actually is.
+- **What must not happen** — no gaps left uncovered that the caller thought were
+  covered, and no B-roll placed over a moment where the speaker is on camera
+  making a point the audience needs to see.
+
+## The judgement this refuses to make
+
+Whether a shot *illustrates* a line is not measurable here. The caller supplies
+candidates with a relevance score from whatever matched them — semantic
+similarity, a keyword hit, a human's choice — and this module places them and
+reports the score it was given. **It does not invent relevance and it does not
+re-rank on anything it cannot see.** A tool that quietly reorders a human's
+choices by its own opinion of relevance is worse than one that places them in
+the order it was handed.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Mapping, Sequence
+
+#: A B-roll shot shorter than this reads as a flash rather than a cutaway.
+MIN_BROLL_SECONDS = 0.8
+
+#: Leave this much of a beat uncovered at each end so the cutaway sits inside
+#: the thought rather than straddling its edges.
+BEAT_INSET_SECONDS = 0.15
+
+#: Below this the caller's own matcher was not confident, and placing it anyway
+#: would launder a weak match into a decision.
+MIN_RELEVANCE = 0.35
+
+
+def placeable_beats(
+    beats: Sequence[Mapping[str, Any]],
+    *,
+    min_seconds: float = MIN_BROLL_SECONDS,
+    inset: float = BEAT_INSET_SECONDS,
+) -> Dict[str, Any]:
+    """Windows inside the A-roll where a cutaway can sit without straddling a beat.
+
+    Each beat: `{"start_seconds", "end_seconds", "text"?, "protected"?}`.
+    A beat marked `protected` is never covered — that is how a caller says "the
+    speaker is making a point here and the audience needs to see them say it".
+    """
+    windows: List[Dict[str, Any]] = []
+    skipped: List[Dict[str, Any]] = []
+    for index, beat in enumerate(beats):
+        start = float(beat.get("start_seconds", 0)) + inset
+        end = float(beat.get("end_seconds", 0)) - inset
+        available = end - start
+        if beat.get("protected"):
+            skipped.append({
+                "beat_index": index,
+                "text": beat.get("text"),
+                "reason": "protected — the caller marked this as must-see on camera",
+            })
+            continue
+        if available < min_seconds:
+            skipped.append({
+                "beat_index": index,
+                "text": beat.get("text"),
+                "reason": f"only {max(0.0, available):.2f}s of usable window — under the "
+                          f"{min_seconds}s floor, a cutaway here would read as a flash",
+            })
+            continue
+        windows.append({
+            "beat_index": index,
+            "start_seconds": round(start, 3),
+            "end_seconds": round(end, 3),
+            "available_seconds": round(available, 3),
+            "text": beat.get("text"),
+        })
+    return {"windows": windows, "skipped": skipped}
+
+
+def place(
+    beats: Sequence[Mapping[str, Any]],
+    candidates: Sequence[Mapping[str, Any]],
+    *,
+    min_relevance: float = MIN_RELEVANCE,
+    min_seconds: float = MIN_BROLL_SECONDS,
+    allow_reuse: bool = False,
+) -> Dict[str, Any]:
+    """Place B-roll candidates into A-roll windows, in the order supplied.
+
+    Each candidate: `{"name", "duration_seconds", "relevance"?, "beat_index"?}`.
+    A candidate carrying `beat_index` is a caller's explicit choice and is placed
+    there or not at all — never silently moved somewhere it fits better.
+    """
+    if not beats:
+        return {"success": False, "error": "No A-roll beats supplied"}
+    if not candidates:
+        return {"success": False, "error": "No B-roll candidates supplied"}
+
+    available = placeable_beats(beats, min_seconds=min_seconds)
+    windows = {w["beat_index"]: w for w in available["windows"]}
+    used_windows: set = set()
+    used_clips: set = set()
+
+    placements: List[Dict[str, Any]] = []
+    unplaced: List[Dict[str, Any]] = []
+
+    for candidate in candidates:
+        name = candidate.get("name")
+        relevance = candidate.get("relevance")
+        if relevance is not None and float(relevance) < min_relevance:
+            unplaced.append({
+                "clip": name,
+                "reason": f"relevance {float(relevance):.2f} is below {min_relevance} — "
+                          "the matcher that produced it was not confident, and placing "
+                          "it anyway would launder a weak match into a decision",
+            })
+            continue
+        if not allow_reuse and name in used_clips:
+            unplaced.append({"clip": name, "reason": "already placed; allow_reuse is off"})
+            continue
+
+        requested = candidate.get("beat_index")
+        if requested is not None:
+            window = windows.get(int(requested))
+            if window is None:
+                unplaced.append({
+                    "clip": name,
+                    "reason": f"beat {requested} is not placeable (protected, too short, "
+                              "or out of range) — an explicit choice is placed there or "
+                              "not at all, never moved somewhere it fits better",
+                })
+                continue
+            if int(requested) in used_windows:
+                unplaced.append({"clip": name, "reason": f"beat {requested} already covered"})
+                continue
+            target = window
+        else:
+            free = [w for i, w in sorted(windows.items()) if i not in used_windows]
+            if not free:
+                unplaced.append({"clip": name, "reason": "no uncovered window left"})
+                continue
+            target = free[0]
+
+        duration = float(candidate.get("duration_seconds") or 0.0)
+        if duration < min_seconds:
+            unplaced.append({
+                "clip": name,
+                "reason": f"{duration:.2f}s of media — under the {min_seconds}s floor",
+            })
+            continue
+        length = min(duration, target["available_seconds"])
+        used_windows.add(target["beat_index"])
+        used_clips.add(name)
+        placements.append({
+            "clip": name,
+            "beat_index": target["beat_index"],
+            "over_text": target["text"],
+            "start_seconds": target["start_seconds"],
+            "end_seconds": round(target["start_seconds"] + length, 3),
+            "length_seconds": round(length, 3),
+            "trimmed": length < duration,
+            "relevance": float(relevance) if relevance is not None else None,
+            "relevance_source": (
+                "supplied by the caller's matcher; not re-scored here"
+                if relevance is not None else "not supplied"
+            ),
+        })
+
+    uncovered = [w for i, w in sorted(windows.items()) if i not in used_windows]
+    return {
+        "success": True,
+        "kind": "broll_placement",
+        "placement_count": len(placements),
+        "placements": placements,
+        "unplaced": unplaced,
+        "uncovered_windows": uncovered,
+        "unusable_beats": available["skipped"],
+        "note": (
+            f"{len(placements)} cutaways placed over {len(windows)} usable windows; "
+            f"{len(uncovered)} windows left uncovered and "
+            f"{len(available['skipped'])} beats were not usable at all. Placement is "
+            "mechanical — **relevance is whatever your matcher said it was and is not "
+            "re-scored here**, because whether a shot illustrates a line is not "
+            "something this can see. Cutaways sit inside a beat rather than straddling "
+            "one; protected beats are never covered. Nothing is cut."
+        ),
+    }