davinci-resolve-mcp 2.68.2 → 2.69.1

package/src/utils/beat_detection.py

@@ -0,0 +1,242 @@
+"""Beat detection for music-driven cutting.
+
+The single most-requested missing capability from the first public audience for
+this project. Music-video makers, live-music channels and motorsport editors all
+asked variants of the same thing, and one put the failure precisely: the tools
+here only cut interviews, they do not cut to music. A viewer editing drift
+footage predicted the exact consequence of pointing a speech-silence gate at his
+material — engine noise and smoke read as "dead space", and the tool cuts in all
+the wrong places.
+
+They are right, and the reason is structural. Everything else in this codebase
+finds edit points in *speech*: word boundaries, pauses, fillers. Music has none
+of those. It has pulse, and pulse is a completely different measurement.
+
+## What this gives you
+
+- **Beats** — the pulse. Cutting on every beat is relentless and usually wrong.
+- **Phrases** — beats grouped into bars and bars into phrases (typically 4 or 8
+  bars). Music resolves at phrase boundaries, which is where an edit feels
+  *inevitable* rather than merely synchronised. This is the one that matters.
+- **Frame-snapped cut points**, because a cut two frames off the beat reads as a
+  mistake rather than as syncopation.
+
+## Honest limits
+
+Tempo tracking is reliable on steady, percussive material and degrades on rubato,
+live performance without a strong downbeat, and anything heavily swung. The
+detector reports its own confidence and this module surfaces it rather than
+presenting every result as equally solid.
+
+Downbeats are **inferred**, not detected: the first beat is assumed to start a
+bar. That assumption is wrong whenever a track has a pickup, and it is the first
+thing to check if the cuts feel consistently one beat early. `beat_offset` exists
+to correct it.
+
+## Dependency
+
+`librosa` (ISC) is an **optional extra** — it is not in `requirements.txt`, it
+pulls a large scientific stack, and this module honest-refuses without it rather
+than degrading into a guess. Install with `pip install librosa`.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from typing import Any, Dict, List, Sequence
+
+#: Beats per bar. 4/4 covers the overwhelming majority of cuttable music; the
+#: parameter exists because 3/4 and 6/8 do turn up and silently forcing them
+#: into 4 produces phrase boundaries that land in musically wrong places.
+DEFAULT_BEATS_PER_BAR = 4
+#: Bars per phrase. 8 bars is the common pop/dance period; 4 suits shorter cuts.
+DEFAULT_BARS_PER_PHRASE = 8
+
+
+def librosa_available() -> bool:
+    """Presence check without importing — librosa costs seconds to import."""
+    try:
+        return importlib.util.find_spec("librosa") is not None
+    except (ImportError, ValueError):
+        return False
+
+
+def detect_beats(media_path: str, *, sample_rate: int = 22050) -> Dict[str, Any]:
+    """Tempo and beat times for an audio or video file.
+
+    Honest-refuses when librosa is absent. A fabricated tempo would produce cuts
+    that are confidently, rhythmically wrong — worse than no feature, because
+    the failure looks intentional.
+    """
+    if not librosa_available():
+        return {
+            "success": False,
+            "error": "librosa is not installed — beat detection is unavailable",
+            "remediation": (
+                "pip install librosa  (ISC licensed, optional extra; it pulls "
+                "numpy/scipy/numba and is deliberately not a core dependency)"
+            ),
+        }
+    import librosa
+
+    try:
+        y, sr = librosa.load(media_path, sr=sample_rate, mono=True)
+    except Exception as exc:
+        return {"success": False, "error": f"could not read audio from {media_path}: {exc}"}
+    if y is None or len(y) == 0:
+        return {"success": False, "error": "no audio samples decoded — is there an audio track?"}
+
+    try:
+        tempo, beat_frames = librosa.beat.beat_track(y=y, sr=sr, units="frames")
+        beat_times = librosa.frames_to_time(beat_frames, sr=sr)
+        onset_env = librosa.onset.onset_strength(y=y, sr=sr)
+    except Exception as exc:
+        return {"success": False, "error": f"beat tracking failed: {exc}"}
+
+    beats = [round(float(t), 4) for t in beat_times]
+    tempo_value = float(tempo if not hasattr(tempo, "__len__") else (tempo[0] if len(tempo) else 0.0))
+    return {
+        "success": True,
+        "tempo_bpm": round(tempo_value, 2),
+        "beat_count": len(beats),
+        "beats": beats,
+        "duration_seconds": round(float(len(y)) / sr, 3),
+        "confidence": _confidence(beats, onset_env),
+    }
+
+
+def _confidence(beats: Sequence[float], onset_env) -> Dict[str, Any]:
+    """How regular the detected pulse is.
+
+    A steady grid means the tracker locked on. Wildly varying intervals mean it
+    did not, and the caller should be told rather than handed a tidy-looking
+    list of times that happens to be wrong.
+    """
+    if len(beats) < 4:
+        return {"band": "low", "reason": "too few beats detected to judge regularity"}
+    intervals = [b - a for a, b in zip(beats, beats[1:])]
+    mean = sum(intervals) / len(intervals)
+    if mean <= 0:
+        return {"band": "low", "reason": "degenerate beat spacing"}
+    variance = sum((i - mean) ** 2 for i in intervals) / len(intervals)
+    jitter = (variance ** 0.5) / mean
+    if jitter < 0.05:
+        band, reason = "high", "beat spacing is steady — the tracker locked on"
+    elif jitter < 0.15:
+        band, reason = "medium", "some drift in beat spacing; check a few cuts against the music"
+    else:
+        band, reason = "low", (
+            "beat spacing is irregular — rubato, live playing or a weak downbeat. "
+            "Treat these cut points as suggestions"
+        )
+    return {"band": band, "jitter": round(jitter, 4), "reason": reason}
+
+
+def group_into_phrases(
+    beats: Sequence[float],
+    *,
+    beats_per_bar: int = DEFAULT_BEATS_PER_BAR,
+    bars_per_phrase: int = DEFAULT_BARS_PER_PHRASE,
+    beat_offset: int = 0,
+) -> Dict[str, Any]:
+    """Group beats into bars and phrases. Pure — no librosa needed.
+
+    `beat_offset` shifts which beat is treated as the first of a bar. Downbeats
+    are inferred rather than detected, so a track with a pickup will be off by
+    one until this is set; that is the first knob to reach for when cuts feel
+    consistently early.
+    """
+    if beats_per_bar < 1 or bars_per_phrase < 1:
+        raise ValueError("beats_per_bar and bars_per_phrase must be >= 1")
+    ordered = sorted(float(b) for b in beats)
+    per_phrase = beats_per_bar * bars_per_phrase
+    downbeats: List[float] = []
+    phrase_starts: List[float] = []
+    for index, time in enumerate(ordered):
+        position = index - beat_offset
+        if position < 0:
+            continue
+        if position % beats_per_bar == 0:
+            downbeats.append(round(time, 4))
+        if position % per_phrase == 0:
+            phrase_starts.append(round(time, 4))
+    return {
+        "beats_per_bar": beats_per_bar,
+        "bars_per_phrase": bars_per_phrase,
+        "beat_offset": beat_offset,
+        "downbeats": downbeats,
+        "phrase_starts": phrase_starts,
+        "bar_count": len(downbeats),
+        "phrase_count": len(phrase_starts),
+        "note": (
+            "Downbeats are INFERRED — the first beat is assumed to start a bar. "
+            "A track with a pickup will be off by one; correct it with beat_offset."
+        ),
+    }
+
+
+def snap_to_frames(times: Sequence[float], fps: float) -> List[int]:
+    """Times to frame numbers. A cut two frames off the beat reads as a mistake."""
+    if fps <= 0:
+        raise ValueError("fps must be positive")
+    return [int(round(float(t) * fps)) for t in times]
+
+
+def plan_beat_cuts(
+    beats: Sequence[float],
+    *,
+    fps: float,
+    mode: str = "phrase",
+    beats_per_bar: int = DEFAULT_BEATS_PER_BAR,
+    bars_per_phrase: int = DEFAULT_BARS_PER_PHRASE,
+    beat_offset: int = 0,
+    min_shot_seconds: float = 0.0,
+) -> Dict[str, Any]:
+    """Cut points on the beat, the bar, or the phrase. Pure and testable.
+
+    `mode`:
+      - `beat`   — every beat. Relentless; suits montage stings, little else.
+      - `bar`    — every downbeat.
+      - `phrase` — every phrase start (**default**). Music resolves here, which
+        is what makes a cut feel inevitable rather than merely synchronised.
+    """
+    if mode not in ("beat", "bar", "phrase"):
+        raise ValueError(f"unknown mode {mode!r}; expected beat, bar or phrase")
+    grouped = group_into_phrases(
+        beats, beats_per_bar=beats_per_bar,
+        bars_per_phrase=bars_per_phrase, beat_offset=beat_offset,
+    )
+    times = {
+        "beat": sorted(float(b) for b in beats),
+        "bar": grouped["downbeats"],
+        "phrase": grouped["phrase_starts"],
+    }[mode]
+
+    dropped = 0
+    if min_shot_seconds > 0 and times:
+        kept = [times[0]]
+        for time in times[1:]:
+            if time - kept[-1] >= min_shot_seconds:
+                kept.append(time)
+            else:
+                dropped += 1
+        times = kept
+
+    return {
+        "success": True,
+        "kind": "beat_cuts",
+        "mode": mode,
+        "cut_count": len(times),
+        "cut_times": [round(t, 4) for t in times],
+        "cut_frames": snap_to_frames(times, fps),
+        "grouping": grouped,
+        "dropped_for_min_shot": dropped,
+        "note": (
+            f"Cut points on every {mode}. "
+            + (f"{dropped} were dropped to honour min_shot_seconds={min_shot_seconds}. "
+               if dropped else "")
+            + "Frame-snapped: a cut two frames off the beat reads as a mistake rather "
+            "than syncopation. These are cut POINTS, not an assembly — what goes "
+            "between them is yours."
+        ),
+    }

package/src/utils/bridge_differential.py

@@ -61,6 +61,28 @@ VOLATILE_METHODS = frozenset({
     "IsPlayheadOnVideoFrame",
 })
 
+#: App-global state that **the harness's own probes move**, which is a different
+#: problem from `VOLATILE_METHODS` drifting on its own.
+#:
+#: Measured on Studio 19.1.3.7: the render probes navigate Resolve to the Deliver
+#: page, and the bridge pass runs before the native pass — so the native pass
+#: reads a page the bridge pass changed. That produced a `value_mismatch` on
+#: `GetCurrentPage` (bridge `edit`, native `deliver`) on every run that did not
+#: happen to start on Deliver, and none on the runs that did. Read at the same
+#: instant the two transports always agreed.
+#:
+#: Excusing it unconditionally would throw away a real signal — a bridge that
+#: genuinely reported the wrong page is exactly the bug this harness is for. So
+#: each pass records the page on the way in and on the way out, and the value is
+#: only excused when a pass proves it did not hold still. If both passes were
+#: stable and the values still disagree, that is the transport and it is reported.
+ORDER_SENSITIVE_METHODS = frozenset({"GetCurrentPage"})
+
+#: Key `run_probes` adds for `compare`'s own use rather than as a probe result.
+#: Underscore-prefixed so the diff skips it instead of reading it as a method
+#: that one transport has and the other does not.
+PASS_META_KEY = "_pass_meta"
+
 #: Never invoked by the harness, whatever the enumeration says. These either
 #: change the project, touch the filesystem, or block on UI. The write paths are
 #: exercised by named scenarios instead, where the setup and teardown are
@@ -326,6 +348,19 @@ def probe_one(target: Any, method: str) -> Dict[str, Any]:
     return {"outcome": "returned", "value": normalise(value), "type": describe(value)}
 
 
+def _current_page(resolve: Any) -> Optional[str]:
+    """The page, or None if it cannot be read.
+
+    Never raises: this is bookkeeping around the probes, and a failure here must
+    not take down a pass that is otherwise fine.
+    """
+    try:
+        page = resolve.GetCurrentPage()
+    except Exception:  # noqa: BLE001 - an unreadable page is simply unknown
+        return None
+    return page if isinstance(page, str) else None
+
+
 def run_probes(resolve: Any, methods: Sequence[str]) -> Dict[str, Dict[str, Any]]:
     """Every enumerated safe method, against every object that has it.
 
@@ -333,6 +368,7 @@ def run_probes(resolve: Any, methods: Sequence[str]) -> Dict[str, Dict[str, Any]
     not merely that they did.
     """
     results: Dict[str, Dict[str, Any]] = {}
+    page_before = _current_page(resolve)
     for label, chain in OBJECT_GRAPH:
         target = walk(resolve, chain)
         if target is None:
@@ -342,13 +378,28 @@ def run_probes(resolve: Any, methods: Sequence[str]) -> Dict[str, Dict[str, Any]
             if getattr(target, method, None) is None:
                 continue  # not on this object; absence is reported by the diff
             results[f"{label}.{method}"] = probe_one(target, method)
+    page_after = _current_page(resolve)
+    results[PASS_META_KEY] = {
+        "page_before": page_before,
+        "page_after": page_after,
+        # None on either side means we could not tell, which must not read as
+        # "stable" — an unknown has to disable the value comparison, not enable it.
+        "page_stable": page_before is not None and page_before == page_after,
+    }
     return results
 
 
 def compare(native: Dict[str, Dict[str, Any]], bridged: Dict[str, Dict[str, Any]]) -> Dict[str, Any]:
     """Diff two probe runs. Empty `differences` is the pass condition."""
     differences: List[Dict[str, Any]] = []
+    # A pass that moved the page cannot have its page compared by value against
+    # another pass taken at a different moment. Both must have held still.
+    native_meta = native.get(PASS_META_KEY) or {}
+    bridge_meta = bridged.get(PASS_META_KEY) or {}
+    page_held_still = bool(native_meta.get("page_stable")) and bool(bridge_meta.get("page_stable"))
     for key in sorted(set(native) | set(bridged)):
+        if key.startswith("_"):
+            continue  # harness bookkeeping, not a probe result
         left, right = native.get(key), bridged.get(key)
         if left is None or right is None:
             differences.append({
@@ -364,17 +415,32 @@ def compare(native: Dict[str, Dict[str, Any]], bridged: Dict[str, Dict[str, Any]
                 differences.append({"key": key, "kind": "volatile_type_mismatch",
                                     "native": left.get("type"), "bridge": right.get("type")})
             continue
+        if method in ORDER_SENSITIVE_METHODS and not page_held_still:
+            # The harness moved it mid-run, so the two readings are of different
+            # moments. Still hold the transport to returning the same kind of
+            # thing, and say why the value was not compared.
+            if left.get("type") != right.get("type") or left["outcome"] != right["outcome"]:
+                differences.append({"key": key, "kind": "order_sensitive_type_mismatch",
+                                    "native": left.get("type"), "bridge": right.get("type")})
+            continue
         if left != right:
             differences.append({"key": key, "kind": "value_mismatch",
                                 "native": left, "bridge": right})
-    agreed = len(set(native) & set(bridged)) - len(
+    probe_keys = {k for k in set(native) | set(bridged) if not k.startswith("_")}
+    agreed = len({k for k in set(native) & set(bridged) if not k.startswith("_")}) - len(
         [d for d in differences if d["kind"] != "only_one_transport_reached_it"]
     )
     return {
-        "compared": len(set(native) | set(bridged)),
+        "compared": len(probe_keys),
         "agreed": agreed,
         "differences": differences,
         "identical": not differences,
+        # Reported either way. "The page was not compared" is a fact about the
+        # run that a reader needs, and burying it would recreate the silence
+        # this fix exists to end.
+        "page_compared_by_value": page_held_still,
+        "page_native": {k: native_meta.get(k) for k in ("page_before", "page_after")},
+        "page_bridge": {k: bridge_meta.get(k) for k in ("page_before", "page_after")},
     }
 
 

package/src/utils/broll_placement.py

@@ -0,0 +1,201 @@
+"""Placing B-roll against an A-roll spine.
+
+Asked for repeatedly, and the clearest version came from someone making recipe
+videos: *could it find the one second where I am putting the spoon in my mouth?*
+That is a `find_similar` query against visual strata pointed at a specific
+moment in a voiceover, and the machinery for both halves already exists — it has
+simply never been joined up.
+
+## The shape of the problem
+
+An A-roll spine (a voiceover or a talking head) has **beats**: sentences,
+clauses, topic shifts. B-roll goes over some of them. Which B-roll goes where is
+a content question, answered by whatever similarity the caller supplies; this
+module answers the *placement* question, which is different and mechanical:
+
+- **Where can B-roll go at all** — over a beat, not across one. Cutting away
+  mid-sentence and back is a specific effect, not a default.
+- **How long can it stay** — bounded by the beat, and by how long the candidate
+  actually is.
+- **What must not happen** — no gaps left uncovered that the caller thought were
+  covered, and no B-roll placed over a moment where the speaker is on camera
+  making a point the audience needs to see.
+
+## The judgement this refuses to make
+
+Whether a shot *illustrates* a line is not measurable here. The caller supplies
+candidates with a relevance score from whatever matched them — semantic
+similarity, a keyword hit, a human's choice — and this module places them and
+reports the score it was given. **It does not invent relevance and it does not
+re-rank on anything it cannot see.** A tool that quietly reorders a human's
+choices by its own opinion of relevance is worse than one that places them in
+the order it was handed.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Mapping, Sequence
+
+#: A B-roll shot shorter than this reads as a flash rather than a cutaway.
+MIN_BROLL_SECONDS = 0.8
+
+#: Leave this much of a beat uncovered at each end so the cutaway sits inside
+#: the thought rather than straddling its edges.
+BEAT_INSET_SECONDS = 0.15
+
+#: Below this the caller's own matcher was not confident, and placing it anyway
+#: would launder a weak match into a decision.
+MIN_RELEVANCE = 0.35
+
+
+def placeable_beats(
+    beats: Sequence[Mapping[str, Any]],
+    *,
+    min_seconds: float = MIN_BROLL_SECONDS,
+    inset: float = BEAT_INSET_SECONDS,
+) -> Dict[str, Any]:
+    """Windows inside the A-roll where a cutaway can sit without straddling a beat.
+
+    Each beat: `{"start_seconds", "end_seconds", "text"?, "protected"?}`.
+    A beat marked `protected` is never covered — that is how a caller says "the
+    speaker is making a point here and the audience needs to see them say it".
+    """
+    windows: List[Dict[str, Any]] = []
+    skipped: List[Dict[str, Any]] = []
+    for index, beat in enumerate(beats):
+        start = float(beat.get("start_seconds", 0)) + inset
+        end = float(beat.get("end_seconds", 0)) - inset
+        available = end - start
+        if beat.get("protected"):
+            skipped.append({
+                "beat_index": index,
+                "text": beat.get("text"),
+                "reason": "protected — the caller marked this as must-see on camera",
+            })
+            continue
+        if available < min_seconds:
+            skipped.append({
+                "beat_index": index,
+                "text": beat.get("text"),
+                "reason": f"only {max(0.0, available):.2f}s of usable window — under the "
+                          f"{min_seconds}s floor, a cutaway here would read as a flash",
+            })
+            continue
+        windows.append({
+            "beat_index": index,
+            "start_seconds": round(start, 3),
+            "end_seconds": round(end, 3),
+            "available_seconds": round(available, 3),
+            "text": beat.get("text"),
+        })
+    return {"windows": windows, "skipped": skipped}
+
+
+def place(
+    beats: Sequence[Mapping[str, Any]],
+    candidates: Sequence[Mapping[str, Any]],
+    *,
+    min_relevance: float = MIN_RELEVANCE,
+    min_seconds: float = MIN_BROLL_SECONDS,
+    allow_reuse: bool = False,
+) -> Dict[str, Any]:
+    """Place B-roll candidates into A-roll windows, in the order supplied.
+
+    Each candidate: `{"name", "duration_seconds", "relevance"?, "beat_index"?}`.
+    A candidate carrying `beat_index` is a caller's explicit choice and is placed
+    there or not at all — never silently moved somewhere it fits better.
+    """
+    if not beats:
+        return {"success": False, "error": "No A-roll beats supplied"}
+    if not candidates:
+        return {"success": False, "error": "No B-roll candidates supplied"}
+
+    available = placeable_beats(beats, min_seconds=min_seconds)
+    windows = {w["beat_index"]: w for w in available["windows"]}
+    used_windows: set = set()
+    used_clips: set = set()
+
+    placements: List[Dict[str, Any]] = []
+    unplaced: List[Dict[str, Any]] = []
+
+    for candidate in candidates:
+        name = candidate.get("name")
+        relevance = candidate.get("relevance")
+        if relevance is not None and float(relevance) < min_relevance:
+            unplaced.append({
+                "clip": name,
+                "reason": f"relevance {float(relevance):.2f} is below {min_relevance} — "
+                          "the matcher that produced it was not confident, and placing "
+                          "it anyway would launder a weak match into a decision",
+            })
+            continue
+        if not allow_reuse and name in used_clips:
+            unplaced.append({"clip": name, "reason": "already placed; allow_reuse is off"})
+            continue
+
+        requested = candidate.get("beat_index")
+        if requested is not None:
+            window = windows.get(int(requested))
+            if window is None:
+                unplaced.append({
+                    "clip": name,
+                    "reason": f"beat {requested} is not placeable (protected, too short, "
+                              "or out of range) — an explicit choice is placed there or "
+                              "not at all, never moved somewhere it fits better",
+                })
+                continue
+            if int(requested) in used_windows:
+                unplaced.append({"clip": name, "reason": f"beat {requested} already covered"})
+                continue
+            target = window
+        else:
+            free = [w for i, w in sorted(windows.items()) if i not in used_windows]
+            if not free:
+                unplaced.append({"clip": name, "reason": "no uncovered window left"})
+                continue
+            target = free[0]
+
+        duration = float(candidate.get("duration_seconds") or 0.0)
+        if duration < min_seconds:
+            unplaced.append({
+                "clip": name,
+                "reason": f"{duration:.2f}s of media — under the {min_seconds}s floor",
+            })
+            continue
+        length = min(duration, target["available_seconds"])
+        used_windows.add(target["beat_index"])
+        used_clips.add(name)
+        placements.append({
+            "clip": name,
+            "beat_index": target["beat_index"],
+            "over_text": target["text"],
+            "start_seconds": target["start_seconds"],
+            "end_seconds": round(target["start_seconds"] + length, 3),
+            "length_seconds": round(length, 3),
+            "trimmed": length < duration,
+            "relevance": float(relevance) if relevance is not None else None,
+            "relevance_source": (
+                "supplied by the caller's matcher; not re-scored here"
+                if relevance is not None else "not supplied"
+            ),
+        })
+
+    uncovered = [w for i, w in sorted(windows.items()) if i not in used_windows]
+    return {
+        "success": True,
+        "kind": "broll_placement",
+        "placement_count": len(placements),
+        "placements": placements,
+        "unplaced": unplaced,
+        "uncovered_windows": uncovered,
+        "unusable_beats": available["skipped"],
+        "note": (
+            f"{len(placements)} cutaways placed over {len(windows)} usable windows; "
+            f"{len(uncovered)} windows left uncovered and "
+            f"{len(available['skipped'])} beats were not usable at all. Placement is "
+            "mechanical — **relevance is whatever your matcher said it was and is not "
+            "re-scored here**, because whether a shot illustrates a line is not "
+            "something this can see. Cutaways sit inside a beat rather than straddling "
+            "one; protected beats are never covered. Nothing is cut."
+        ),
+    }