@lemoncat7/dsh-partner 2.8.1 → 2.8.2

package/resources/skills/video-recap-production/upstream/skills/video-assemble/scripts/adoption/audio_mix_binding.py

@@ -0,0 +1,279 @@
+"""Validate, render, and record an explicitly adopted prepared-bed plus narration mix."""
+
+import json
+from pathlib import Path
+import subprocess
+
+from assemble_constants import frame_clock_samples
+from adoption.frozen_audio import probe_audio_packets
+import adoption.narration_binding as narration_binding
+from pair_media import probe_picture, validate_pair_timing
+import source_score
+from adoption.strict_inputs import (
+    read_json_bytes, require_fields, require_integer, require_local_path, require_number,
+    run_logged, without_digests, write_json_atomic,
+)
+
+
+ARTIFACT = "audio_mix_binding"
+FILENAME = "audio_mix_binding.json"
+RATE = 48_000
+CHANNELS = 2
+CODEC = "pcm_f32le"
+
+
+def _picture_format(picture):
+    samples = frame_clock_samples(picture["frame_count"], picture["fps"], RATE)
+    return samples, {
+        "fps": picture["fps"], "frame_count": picture["frame_count"],
+        "duration": picture["duration"], "start": picture["start"],
+        "decoder": picture["decoder"],
+    }
+
+
+def load_adoption(path, *, input_video, narration_adoption_path, tts_segments):
+    """Strict read-only preflight; no work artifacts are created here."""
+    adoption_path, _, value = read_json_bytes(path, "audio mix adoption")
+    value = without_digests(value, "audio mix adoption")
+    require_fields(value, ["artifact", "schema_version", "prepared_receipt", "format", "segments",
+                           "master_gain_db"], "audio mix adoption")
+    if value["artifact"] != "audio_mix_adoption" or type(value["schema_version"]) is not int \
+            or value["schema_version"] != 1:
+        raise ValueError("unsupported audio_mix_adoption schema")
+    input_video = require_local_path(input_video, "picture")
+    picture = probe_picture(input_video)
+    picture_samples, picture_summary = _picture_format(picture)
+
+    narration_path = require_local_path(narration_adoption_path, "narration adoption")
+    require_fields(value["format"], ["sample_rate", "channels", "total_samples"], "mix format")
+    mix_format = value["format"]
+    if mix_format != {"sample_rate": RATE, "channels": CHANNELS,
+                      "total_samples": picture_samples}:
+        raise ValueError("mix format differs from the actual picture sample clock")
+    prepared_receipt = source_score.validate_prepared_receipt(
+        value["prepared_receipt"], mix_format
+    )
+
+    if not isinstance(value["segments"], list) or len(value["segments"]) != len(tts_segments):
+        raise ValueError("audio mix segments must exactly cover narration segments")
+    normalized = []
+    seen = set()
+    for adopted, segment in zip(value["segments"], tts_segments):
+        adopted = without_digests(adopted, "audio mix segment")
+        require_fields(adopted, ["index", "output_start_sample", "gain"], "audio mix segment")
+        index = require_integer(adopted["index"], "audio mix segment index")
+        if index in seen or index != segment["index"]:
+            raise ValueError("audio mix segment index/order differs from narration")
+        seen.add(index)
+        normalized.append({
+            "index": index,
+            "output_start_sample": require_integer(
+                adopted["output_start_sample"], "output_start_sample"),
+            "gain": require_number(adopted["gain"], "narration gain", 0, 16),
+        })
+    return {
+        "path": str(adoption_path),
+        "picture": {"path": str(input_video), "clock": picture_summary},
+        "picture_identity": picture,
+        "prepared_receipt": prepared_receipt["reference"],
+        "prepared": prepared_receipt["outputs"],
+        "narration_adoption": {"path": str(narration_path)},
+        "format": dict(mix_format), "segments": normalized,
+        "master_gain_db": require_number(value["master_gain_db"], "master_gain_db", -24, 24),
+        "runtime": None,
+    }
+
+
+def _run(command, work_dir, label):
+    run_logged(command, work_dir, label, prefix="explicit_")
+
+
+def _channels(path):
+    result = subprocess.run(
+        ["ffprobe", "-v", "error", "-select_streams", "a:0", "-show_entries",
+         "stream=channels", "-of", "default=nw=1:nk=1", str(path)],
+        capture_output=True, text=True, timeout=600,
+    )
+    try:
+        channels = int(result.stdout.strip())
+    except ValueError as exc:
+        raise ValueError("narration channel probe failed") from exc
+    if result.returncode or channels not in {1, 2}:
+        raise ValueError("explicit narration supports only mono or stereo input")
+    return channels
+
+
+def render_explicit_mix(context, narration_context, tts_segments, work_dir):
+    """Render complete direct 48 kHz placements, voice bus, premaster, and fixed master."""
+    directory = Path(work_dir).resolve() / ".explicit_audio_mix"
+    directory.mkdir(parents=True, exist_ok=False)
+    by_index = {item["index"]: item for item in narration_context["segments"]}
+    segment_memory = {item["index"]: item for item in tts_segments}
+    rendered = []
+    for position, adopted in enumerate(context["segments"]):
+        source = by_index[adopted["index"]]["snapshot"]["path"]
+        channels = _channels(source)
+        placed = directory / f"placed_{position:04d}_{adopted['index']}.wav"
+        if channels == 1:
+            channel_filter = (
+                "aformat=sample_fmts=flt,"
+                "pan=stereo|c0=0.7071067811865476*c0|c1=0.7071067811865476*c0,"
+                "aresample=48000:async=0:first_pts=0,aformat=sample_fmts=flt:"
+                "sample_rates=48000:channel_layouts=stereo"
+            )
+            matrix = "mono_equal_power"
+        else:
+            channel_filter = (
+                "aresample=48000:async=0:first_pts=0,aformat=sample_fmts=flt:"
+                "sample_rates=48000:channel_layouts=stereo"
+            )
+            matrix = "stereo_identity"
+        _run(["ffmpeg", "-nostdin", "-v", "error", "-n", "-i", str(source),
+              "-map", "0:a:0", "-af", channel_filter, "-c:a", CODEC, str(placed)],
+             directory, f"place_{position:04d}")
+        facts = source_score._output_facts(placed)
+        start = adopted["output_start_sample"]
+        end = start + facts["pcm"]["samples"]
+        if end > context["format"]["total_samples"]:
+            raise ValueError("complete narration segment does not fit the adopted mix clock")
+        rendered.append({**adopted, "output_end_sample": end, "input_channels": channels,
+                         "channel_matrix": matrix, "placed": facts})
+        memory = segment_memory[adopted["index"]]
+        memory.update({
+            "placed_audio_path": str(placed),
+            "narration_conversion_path": str(placed),
+            "audio_duration": facts["pcm"]["samples"] / RATE,
+            "placed_audio_duration": facts["pcm"]["samples"] / RATE,
+            "actual_place_start": start / RATE, "actual_place_end": end / RATE,
+            "output_start_sample": start, "output_end_sample": end,
+            "adopted_gain": adopted["gain"],
+            "global_narration_speed": 1.0, "segment_tempo_factor": 1.0,
+            "effective_tempo": 1.0, "fit_status": "placed", "blocking": False,
+            "truncated": False, "truncate_reason": "none",
+        })
+    ordered = sorted(rendered, key=lambda item: item["output_start_sample"])
+    if any(current["output_start_sample"] < previous["output_end_sample"]
+           for previous, current in zip(ordered, ordered[1:])):
+        raise ValueError("explicit narration placements overlap")
+
+    total = context["format"]["total_samples"]
+    voice_bus = directory / "voice_bus.wav"
+    command = ["ffmpeg", "-nostdin", "-v", "error", "-n"]
+    for item in rendered:
+        command += ["-i", item["placed"]["path"]]
+    filters = [f"anullsrc=r={RATE}:cl=stereo,atrim=end_sample={total}[base]"]
+    labels = ["[base]"]
+    for position, item in enumerate(rendered):
+        filters.append(
+            f"[{position}:a]volume={item['gain']:.17g},"
+            f"adelay={item['output_start_sample']}S:all=1[v{position}]"
+        )
+        labels.append(f"[v{position}]")
+    filters.append("".join(labels) + f"amix=inputs={len(labels)}:duration=first:normalize=0,"
+                   f"atrim=end_sample={total},aformat=sample_fmts=flt:sample_rates={RATE}:"
+                   "channel_layouts=stereo[out]")
+    command += ["-filter_complex", ";".join(filters), "-map", "[out]", "-c:a", CODEC,
+                str(voice_bus)]
+    _run(command, directory, "voice_bus")
+    voice_facts = source_score._output_facts(voice_bus)
+    narration_binding.seal_render_inputs(narration_context, tts_segments, voice_bus)
+    narration_context["sealed"]["narration_bus"]["consumption_status"] = \
+        "CONSUMED_BY_EXPLICIT_MIX"
+
+    prepared = context["prepared"]["prepared_bed.wav"]["path"]
+    premaster = directory / "premaster.wav"
+    _run(["ffmpeg", "-nostdin", "-v", "error", "-n", "-i", prepared, "-i", str(voice_bus),
+          "-filter_complex", f"[0:a][1:a]amix=inputs=2:duration=first:normalize=0,"
+          f"atrim=end_sample={total},aformat=sample_fmts=flt:sample_rates={RATE}:"
+          "channel_layouts=stereo[out]", "-map", "[out]", "-c:a", CODEC, str(premaster)],
+         directory, "premaster")
+    master = directory / "master.wav"
+    gain = 10 ** (context["master_gain_db"] / 20)
+    _run(["ffmpeg", "-nostdin", "-v", "error", "-n", "-i", str(premaster),
+          "-af", f"volume={gain:.17g},aformat=sample_fmts=flt:sample_rates={RATE}:"
+          "channel_layouts=stereo", "-c:a", CODEC, str(master)], directory, "master")
+    runtime = {
+        "segments": rendered, "voice_bus": voice_facts,
+        "premaster": source_score._output_facts(premaster),
+        "master": source_score._output_facts(master), "master_gain_linear": gain,
+    }
+    if any(runtime[key]["pcm"]["samples"] != total
+           for key in ("voice_bus", "premaster", "master")):
+        raise ValueError("explicit mix derivative sample count differs from picture clock")
+    context["runtime"] = runtime
+    return runtime
+
+
+def finalize_binding(context, narration_record, rendered_output, final_output, work_dir):
+    """Probe the rendered candidate and write ``audio_mix_binding.json`` into work_dir."""
+    if not context.get("runtime"):
+        raise RuntimeError("explicit audio mix must be rendered before finalization")
+    rendered = require_local_path(rendered_output, "rendered output")
+    output_picture = probe_picture(rendered)
+    input_clock = _picture_format(context["picture_identity"])[1]
+    output_clock = _picture_format(output_picture)[1]
+    for key in ("fps", "frame_count", "duration", "start"):
+        if output_clock[key] != input_clock[key]:
+            raise ValueError("rendered output picture frame clock changed")
+    packet_identity = (
+        "EXACT" if output_picture == context["picture_identity"]
+        else "REENCODED_CLOCK_MATCH"
+    )
+    audio = probe_audio_packets(rendered, 0)
+    validate_pair_timing(output_picture, audio)
+    decoded = Path(context["runtime"]["master"]["path"]).parent / "final_aac_decoded.wav"
+    _run(["ffmpeg", "-nostdin", "-v", "error", "-n", "-i", str(rendered),
+          "-map", "0:a:0", "-af", "aformat=sample_fmts=flt:sample_rates=48000:"
+          "channel_layouts=stereo", "-c:a", CODEC, str(decoded)], decoded.parent,
+         "final_decode")
+    context["runtime"]["final_decoded_pcm"] = source_score._output_facts(decoded)
+    report = {
+        "artifact": ARTIFACT, "schema_version": 1, "status": "FINALIZED",
+        "adoption": {"path": context["path"]},
+        "picture": context["picture"],
+        "output_picture": {
+            **output_clock, "packet_identity": packet_identity,
+        },
+        "prepared_receipt": context["prepared_receipt"],
+        "prepared": context["prepared"], "format": context["format"],
+        "segments": context["runtime"]["segments"],
+        "voice_bus": context["runtime"]["voice_bus"],
+        "premaster": context["runtime"]["premaster"],
+        "master": {**context["runtime"]["master"], "gain_db": context["master_gain_db"],
+                   "gain_linear": context["runtime"]["master_gain_linear"]},
+        "narration_input_binding": {"path": narration_record["path"], "status": "FINALIZED"},
+        "final_output": {
+            "path": str(Path(final_output).resolve()),
+            "decoded_pcm": context["runtime"]["final_decoded_pcm"],
+            "audio_stream": {
+                "decoder": audio["decoder"], "packet_count": audio["packet_count"],
+                "payload_bytes": audio["payload_bytes"],
+                "start_time": audio["start_time"], "duration": audio["duration"],
+            },
+        },
+        "direct_listening": "NOT_CHECKED", "normal_speed_review": "NOT_CHECKED",
+        "release_approved": False,
+    }
+    write_json_atomic(Path(work_dir).resolve() / FILENAME, report)
+    return report
+
+
+def binding_record(work_dir):
+    """The finalized mix binding's path and status; None when absent or malformed."""
+    path = Path(work_dir) / FILENAME
+    if not path.is_file():
+        return None
+    try:
+        report = json.loads(path.read_text(encoding="utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        return None
+    if not isinstance(report, dict) or report.get("artifact") != ARTIFACT \
+            or report.get("schema_version") != 1 or report.get("status") != "FINALIZED":
+        return None
+    final = report.get("final_output")
+    if not isinstance(final, dict) or not Path(str(final.get("path", ""))).is_file():
+        return None
+    narration = report.get("narration_input_binding")
+    if not isinstance(narration, dict) or not Path(str(narration.get("path", ""))).is_file():
+        return None
+    return {"path": str(path.resolve()), "status": "FINALIZED"}

package/resources/skills/video-recap-production/upstream/skills/video-assemble/scripts/adoption/frozen_audio.py

@@ -0,0 +1,187 @@
+"""Probe and verify an adopted AAC stream without decoding or rewriting it."""
+
+import json
+from fractions import Fraction
+from pathlib import Path
+
+from lib import run_cmd
+
+
+def _fraction(value):
+    return Fraction(str(value))
+
+
+def _packet_span(audio):
+    """The stream duration proved by its packets: last packet end minus first packet start."""
+    packets = audio["packets"]
+    if not packets or packets[0]["pts"] is None or packets[-1]["pts"] is None \
+            or packets[-1]["duration"] is None:
+        return None
+    return Fraction(packets[-1]["pts"]) + Fraction(packets[-1]["duration"]) \
+        - Fraction(packets[0]["pts"])
+
+
+def _rational_ticks(value, time_base):
+    if value in (None, "N/A"):
+        return None
+    value = Fraction(int(value)) * Fraction(time_base)
+    return f"{value.numerator}/{value.denominator}"
+
+
+def _probe(path, *args):
+    result = run_cmd([
+        "ffprobe", "-v", "error", *args, "-of", "json", str(path),
+    ])
+    if result.returncode != 0:
+        raise RuntimeError(f"无法探测媒体 {path}: {result.stderr.strip()}")
+    try:
+        return json.loads(result.stdout)
+    except (TypeError, ValueError) as exc:
+        raise RuntimeError(f"ffprobe 返回无效 JSON: {path}") from exc
+
+
+def probe_audio_packets(path, audio_stream_index):
+    """Return packet sizes and rational timestamps for one audio stream.
+
+    ``audio_stream_index`` is the zero-based audio-stream ordinal accepted by
+    ffmpeg's ``0:a:N`` selector, not the file-wide absolute stream index.
+    """
+    payload = _probe(
+        path,
+        "-select_streams", f"a:{audio_stream_index}",
+        "-show_streams", "-show_packets",
+        "-show_entries",
+        "stream=index,codec_name,time_base,start_pts,start_time,duration_ts,duration,sample_rate,channels,"
+        "channel_layout:packet=pts,dts,duration,size,side_data_list",
+    )
+    streams = payload.get("streams", [])
+    if len(streams) != 1:
+        raise RuntimeError(f"找不到音频流 a:{audio_stream_index}: {Path(path)}")
+    stream = streams[0]
+    time_base = stream.get("time_base")
+    if not time_base:
+        raise RuntimeError(f"音频流 a:{audio_stream_index} 缺少 time_base")
+    codec = stream.get("codec_name")
+    sample_rate = int(stream["sample_rate"]) if stream.get("sample_rate") else None
+    channels = int(stream["channels"]) if stream.get("channels") else None
+    channel_layout = stream.get("channel_layout")
+    decoder = {
+        "codec": codec,
+        "sample_rate": sample_rate,
+        "channels": channels,
+        "channel_layout": channel_layout,
+    }
+    packets = []
+    for packet in payload.get("packets", []):
+        if packet.get("size") is None:
+            raise RuntimeError(f"音频流 a:{audio_stream_index} 的 packet 缺少 size")
+        packets.append({
+            "size": int(packet["size"]),
+            "pts": _rational_ticks(packet.get("pts"), time_base),
+            "dts": _rational_ticks(packet.get("dts"), time_base),
+            "duration": _rational_ticks(packet.get("duration"), time_base),
+            "side_data_list": packet.get("side_data_list", []),
+        })
+    return {
+        "selected_audio_stream_index": audio_stream_index,
+        "absolute_stream_index": int(stream["index"]),
+        "codec": codec,
+        "time_base": time_base,
+        "start_time": stream.get("start_time"),
+        "duration": stream.get("duration"),
+        "sample_rate": sample_rate,
+        "channels": channels,
+        "channel_layout": channel_layout,
+        "decoder": decoder,
+        "packet_count": len(packets),
+        "payload_bytes": sum(packet["size"] for packet in packets),
+        "packets": packets,
+    }
+
+
+def _picture_interval(path):
+    payload = _probe(
+        path,
+        "-select_streams", "v:0", "-show_streams",
+        "-show_entries", "stream=time_base,start_pts,start_time,duration_ts,duration,avg_frame_rate",
+    )
+    streams = payload.get("streams", [])
+    if len(streams) != 1:
+        raise RuntimeError(f"找不到主画面流 v:0: {Path(path)}")
+    stream = streams[0]
+    start = _fraction(stream.get("start_time", "0"))
+    if stream.get("duration") in (None, "N/A"):
+        raise RuntimeError("主画面流缺少可验证时长")
+    duration = _fraction(stream["duration"])
+    frame_rate = Fraction(stream.get("avg_frame_rate", "0/1"))
+    frame = Fraction(1, 1000) if frame_rate <= 0 else 1 / frame_rate
+    return start, duration, frame
+
+
+def validate_adopted_source(path, audio_stream_index):
+    """Fail unless the selected input is copyable AAC spanning the picture interval."""
+    audio = probe_audio_packets(path, audio_stream_index)
+    if audio["codec"] != "aac":
+        raise RuntimeError(
+            f"adopted-packet-copy 当前只支持 MP4 AAC stream-copy；"
+            f"a:{audio_stream_index} 是 {audio['codec'] or 'unknown'}"
+        )
+    if not audio["packets"]:
+        raise RuntimeError(f"音频流 a:{audio_stream_index} 没有 packet，不能采用")
+    picture_start, picture_duration, frame_tolerance = _picture_interval(path)
+    if audio["start_time"] in (None, "N/A") or audio["duration"] in (None, "N/A"):
+        raise RuntimeError(f"音频流 a:{audio_stream_index} 缺少可验证起止时间")
+    audio_start = _fraction(audio["start_time"])
+    audio_duration = _fraction(audio["duration"])
+    packet_durations = [
+        _fraction(packet["duration"])
+        for packet in audio["packets"] if packet["duration"] is not None
+    ]
+    tolerance = max([frame_tolerance, Fraction(1, 1000), *packet_durations])
+    if (
+        abs(audio_start - picture_start) > tolerance
+        or abs(audio_duration - picture_duration) > tolerance
+    ):
+        raise RuntimeError(
+            "采用音频与画面时长/起点不兼容: "
+            f"picture={float(picture_start):.6f}+{float(picture_duration):.6f}s, "
+            f"audio={float(audio_start):.6f}+{float(audio_duration):.6f}s"
+        )
+    return audio
+
+
+def verify_adopted_audio(input_path, output_path, input_stream_index, output_stream_index=0):
+    """Probe the rendered output and check packet count, payload bytes, duration and timing."""
+    expected = probe_audio_packets(input_path, input_stream_index)
+    actual = probe_audio_packets(output_path, output_stream_index)
+    if expected["decoder"] != actual["decoder"]:
+        raise RuntimeError("采用音频 decoder 参数已改变")
+    if expected["packet_count"] != actual["packet_count"]:
+        raise RuntimeError(
+            f"采用音频 packet count 已改变: {expected['packet_count']} -> {actual['packet_count']}"
+        )
+    if expected["payload_bytes"] != actual["payload_bytes"]:
+        raise RuntimeError("采用音频 packet payload 总字节数不一致")
+    if _packet_span(expected) != _packet_span(actual):
+        raise RuntimeError("采用音频流时长已改变")
+    expected_packet_core = [
+        {key: value for key, value in packet.items() if key != "side_data_list"}
+        for packet in expected["packets"]
+    ]
+    actual_packet_core = [
+        {key: value for key, value in packet.items() if key != "side_data_list"}
+        for packet in actual["packets"]
+    ]
+    if expected_packet_core != actual_packet_core:
+        raise RuntimeError("采用音频 packet PTS/DTS/duration/size 不一致")
+    if [packet["side_data_list"] for packet in expected["packets"]] != [
+        packet["side_data_list"] for packet in actual["packets"]
+    ]:
+        raise RuntimeError("采用音频 packet side data 已改变")
+    return {
+        "verified": True,
+        "selected_audio_stream_index": input_stream_index,
+        "output_audio_stream_index": output_stream_index,
+        "input": expected,
+        "output": actual,
+    }