@lemoncat7/dsh-partner 2.8.1 → 2.8.3

package/resources/skills/video-recap-production/upstream/skills/video-understanding/scripts/asr.py

@@ -0,0 +1,279 @@
+import base64
+import json
+import os
+import re
+import time
+from pathlib import Path
+
+from lib import CONFIG
+from lib import log, run_cmd, get_video_duration, mimo_asr_api_call
+from detect import _audio_meta_path, _write_audio_meta
+from asr_timing_evidence import (
+    EVIDENCE_FILENAME,
+    load_glossary_names,
+    write_asr_timing_evidence,
+)
+
+# ── Step 3: ASR 转录（MiMo mimo-v2.5-asr，云端 API）────────────────────────
+
+_ASR_AUDIO_MIME = "audio/wav"
+
+
+class ASRProviderError(RuntimeError):
+    """Provider/API response failed, so an empty transcript is not cacheable success."""
+
+
+def _load_name_glossary(work_dir):
+    """从 background_research.json 收集已知人名（characters 键 + character_details 键及别名）。
+
+    返回去重后、长度 >=2 的人名列表（按长度降序，长名优先匹配）。文件缺失或无名字时返回 []。
+    """
+    return load_glossary_names(work_dir)
+
+
+def _correct_text_with_glossary(text, names):
+    """用人名表修正 ASR 同音字错误（如 叶青眉 → 叶轻眉）。
+
+    对每个已知人名，扫描文本中所有等长窗口；若某窗口与人名恰好相差一个字符（仅一处不同），
+    则替换为该人名。严格约束为「恰好一字之差」，避免过度纠正（叶轻风 与 叶轻眉 也是一字之差，
+    但这正是限制为单字替换的边界——只有当窗口本身不是任何已知人名时才会被改写）。
+    """
+    if not text or not names:
+        return text
+    name_set = set(names)
+    for name in names:
+        n = len(name)
+        if n < 2 or len(text) < n:
+            continue
+        i = 0
+        while i <= len(text) - n:
+            window = text[i:i + n]
+            # Only rewrite a one-char-off window when it is NOT itself a known name —
+            # otherwise a distinct real name one char away (叶轻风 vs 叶轻眉) would be corrupted.
+            if window != name and window not in name_set and _one_char_diff(window, name):
+                text = text[:i] + name + text[i + n:]
+                i += n
+            else:
+                i += 1
+    return text
+
+
+def _one_char_diff(a, b):
+    """两个等长字符串是否恰好相差一个字符（仅一处不同）。"""
+    if len(a) != len(b):
+        return False
+    diff = 0
+    for ca, cb in zip(a, b):
+        if ca != cb:
+            diff += 1
+            if diff > 1:
+                return False
+    return diff == 1
+
+
+def _apply_glossary_corrections(segments, work_dir):
+    """对已转录的 segments 就地应用人名表修正；无人名表时为 no-op。"""
+    names = _load_name_glossary(work_dir)
+    if not names:
+        return segments
+    for seg in segments:
+        original = seg["text"]
+        corrected = _correct_text_with_glossary(original, names)
+        if corrected != original:
+            seg["text"] = corrected
+    return segments
+
+
+
+def transcribe_audio(video_path, work_dir):
+    """提取音频并用 MiMo ASR 分段转录，通过分段合成时间戳。"""
+    work_dir = Path(work_dir)
+    asr_file = work_dir / "asr_result.json"
+    (work_dir / EVIDENCE_FILENAME).unlink(missing_ok=True)
+
+    if not CONFIG["mimo_asr_api_key"]:
+        key_name = CONFIG["mimo_asr_env_var"]
+        log(f"ASR 跳过：未设置 {key_name}（MiMo ASR 需要；VLM/TTS 也需要同一个 key）。"
+            f"如不需要对白可加 --skip-asr")
+        asr_file.write_text(json.dumps([], ensure_ascii=False, indent=2), encoding="utf-8")
+        write_asr_timing_evidence(
+            work_dir, video_path, "UNAVAILABLE_NO_KEY", final_segments=[]
+        )
+        return []
+
+    # 提取音频
+    audio_wav = work_dir / "audio.wav"
+    audio_meta = _audio_meta_path(work_dir)
+    audio_wav.unlink(missing_ok=True)
+    audio_meta.unlink(missing_ok=True)
+    cmd = ["ffmpeg", "-y", "-i", str(video_path), "-vn",
+           "-ar", "16000", "-ac", "1", str(audio_wav)]
+    try:
+        result = run_cmd(cmd)
+    except Exception as exc:
+        audio_wav.unlink(missing_ok=True)
+        audio_meta.unlink(missing_ok=True)
+        asr_file.unlink(missing_ok=True)
+        write_asr_timing_evidence(work_dir, video_path, "FAILED_AUDIO_EXTRACTION")
+        raise RuntimeError("音频提取失败: ffmpeg 无法完成") from exc
+    if result.returncode != 0:
+        audio_wav.unlink(missing_ok=True)
+        audio_meta.unlink(missing_ok=True)
+        asr_file.unlink(missing_ok=True)
+        write_asr_timing_evidence(work_dir, video_path, "FAILED_AUDIO_EXTRACTION")
+        raise RuntimeError(f"音频提取失败: {result.stderr}")
+    _write_audio_meta(work_dir, video_path)
+
+    # 获取音频时长；ffprobe 失败时不伪造时长（否则会向 asr_result.json 写入虚构时间戳），
+    # 而是记录 UNAVAILABLE_NO_DURATION 证据并跳过转录
+    try:
+        duration = get_video_duration(audio_wav)
+    except RuntimeError as exc:
+        log(f"ASR 警告: 无法获取音频时长，跳过 ASR 转录: {exc}")
+        asr_file.write_text(json.dumps([], ensure_ascii=False, indent=2), encoding="utf-8")
+        write_asr_timing_evidence(
+            work_dir,
+            video_path,
+            "UNAVAILABLE_NO_DURATION",
+            final_segments=[],
+            audio_path=audio_wav,
+        )
+        return []
+
+    segments_dir = work_dir / "audio_segments"
+    segments_dir.mkdir(exist_ok=True)
+
+    segment_length = int(CONFIG["asr_segment_seconds"])
+    try:
+        if duration <= segment_length:
+            # 短音频，整段转录
+            text = _run_asr(audio_wav)
+            asr_result = [{"start": 0.0, "end": round(duration, 2), "text": text}]
+        else:
+            # 长音频，分段转录（固定粗窗口，不是词级或对白边界对齐）
+            asr_result = _segment_and_transcribe(
+                audio_wav, segments_dir, duration, segment_length
+            )
+    except ASRProviderError:
+        asr_file.unlink(missing_ok=True)
+        write_asr_timing_evidence(
+            work_dir, video_path, "FAILED_PROVIDER", audio_path=audio_wav
+        )
+        raise
+
+    # 用 background_research.json 的人名表修正 ASR 同音字错误（如 叶青眉 → 叶轻眉）；无人名表时为 no-op
+    observed_result = [dict(segment) for segment in asr_result]
+    _apply_glossary_corrections(asr_result, work_dir)
+
+    # 保存
+    asr_file.write_text(json.dumps(asr_result, ensure_ascii=False, indent=2), encoding="utf-8")
+    status = "AVAILABLE_COARSE" if any(s["text"] for s in asr_result) else "EMPTY_UNKNOWN"
+    write_asr_timing_evidence(
+        work_dir,
+        video_path,
+        status,
+        observed_segments=observed_result,
+        final_segments=asr_result,
+        audio_path=audio_wav,
+    )
+
+    total_text = " ".join(s["text"] for s in asr_result if s["text"])
+    empty = sum(1 for s in asr_result if not s["text"])
+    suffix = f"（{empty} 段无文本：原因未知，不代表已证实静音）" if empty else ""
+    log(f"ASR 转录完成: {len(asr_result)} 段, 共 {len(total_text)} 字{suffix}")
+    return asr_result
+
+
+def _strip_reasoning_residue(text):
+    """Remove MiMo reasoning-model <think>…</think> leakage from ASR content.
+
+    Thinking-disable is not applied to -asr models (lib._prepare_api_payload), so the reasoning
+    model can leak a <think> block — full, truncated/unclosed, or a leading orphan/residual tag
+    (a bare "think>" prefix) — into the transcript. The dubbing implementation carries its own
+    copy of this policy; keep behavior aligned through tests.
+    """
+    text = re.sub(r"(?is)<think\b.*?</think\s*>", "", text)  # full <think>…</think> block
+    text = re.sub(r"(?is)<think\b.*\Z", "", text)            # unclosed/truncated <think tail
+    return re.sub(r"(?i)^\s*<?/?think\s*>\s*", "", text)     # leading orphan/residual think tag
+
+
+def _run_asr(wav_path):
+    """用 MiMo ASR (mimo-v2.5-asr) 转录单个 wav 文件，返回纯文本。
+
+    音频以 base64 data-URI 放进 OpenAI 风格的 chat/completions 消息里，转写文本回到
+    choices[0].message.content。API/响应结构失败会抛错，避免把瞬时失败缓存成空转写；
+    只有无音频、超体积等确定不可发送的片段返回空串。
+    """
+    try:
+        raw = Path(wav_path).read_bytes()
+    except OSError as e:
+        log(f"ASR 警告: 无法读取音频 {wav_path}: {e}")
+        return ""
+    if not raw:
+        return ""
+
+    b64 = base64.b64encode(raw).decode("ascii")
+    max_b64_bytes = int(float(CONFIG["mimo_asr_base64_max_mb"]) * 1024 * 1024)
+    if len(b64) > max_b64_bytes:
+        log(f"ASR 警告: 分片 base64 体积 {len(b64) / 1024 / 1024:.1f}MB 超过 MiMo 上限 "
+            f"{CONFIG['mimo_asr_base64_max_mb']}MB，跳过该段；可调小 ASR_SEGMENT_SECONDS")
+        return ""
+
+    payload = {
+        "model": CONFIG["mimo_asr_model"],
+        "messages": [{
+            "role": "user",
+            "content": [{
+                "type": "input_audio",
+                "input_audio": {"data": f"data:{_ASR_AUDIO_MIME};base64,{b64}"},
+            }],
+        }],
+        "asr_options": {"language": CONFIG["mimo_asr_language"]},
+    }
+    try:
+        resp = mimo_asr_api_call(payload)
+    except Exception as e:
+        raise ASRProviderError(f"MiMo ASR 调用失败: {e}") from e
+    try:
+        return _strip_reasoning_residue(str(resp["choices"][0]["message"]["content"] or "")).strip()
+    except (KeyError, IndexError, TypeError):
+        raise ASRProviderError(
+            f"MiMo ASR 返回结构异常: {json.dumps(resp, ensure_ascii=False)[:200]}"
+        )
+
+
+def _segment_and_transcribe(audio_wav, segments_dir, total_duration, segment_length=None):
+    """分段转录长音频"""
+    if segment_length is None:
+        segment_length = int(CONFIG["asr_segment_seconds"])
+    # 长视频 ASR 是顺序调用；可选节流让调用间隔开，降低踩到集群限流的频率（默认 0=不节流）
+    try:
+        throttle = max(0.0, float(os.environ.get("ASR_THROTTLE_SECONDS", "0") or 0))
+    except ValueError:
+        throttle = 0.0
+    results = []
+
+    for i, start in enumerate(range(0, int(total_duration), segment_length)):
+        if throttle and i:
+            time.sleep(throttle)
+        end = min(start + segment_length, total_duration)
+        seg_wav = segments_dir / f"seg_{i:03d}.wav"
+
+        cmd = ["ffmpeg", "-y", "-i", str(audio_wav),
+               "-ss", str(start), "-to", str(end),
+               "-ar", "16000", "-ac", "1", str(seg_wav)]
+        cut = run_cmd(cmd)
+        if cut.returncode != 0:
+            # 切分失败时不要对磁盘上的陈旧/残缺音频转录，否则会得到错位文本
+            log(f"  段 {i+1}: 切分失败，跳过转录 ({cut.stderr.strip()[:200]})")
+            text = ""
+        else:
+            text = _run_asr(seg_wav)
+        results.append({
+            "start": round(start, 2),
+            "end": round(end, 2),
+            "text": text,
+        })
+        log(f"  段 {i+1}: {start:.0f}s-{end:.0f}s => {len(text)} 字")
+
+    return results

package/resources/skills/video-recap-production/upstream/skills/video-understanding/scripts/asr_timing_evidence.py

@@ -0,0 +1,270 @@
+"""Write and validate honest timing/provenance evidence for coarse MiMo ASR."""
+
+import json
+import math
+import os
+import tempfile
+from pathlib import Path
+
+from lib import file_identity, load_background_research
+
+
+EVIDENCE_FILENAME = "asr_timing_evidence.json"
+SCHEMA_VERSION = 2
+VALID_STATUSES = {
+    "AVAILABLE_COARSE",
+    "EXPLICITLY_SKIPPED",
+    "UNAVAILABLE_NO_KEY",
+    "UNAVAILABLE_NO_DURATION",
+    "FAILED_AUDIO_EXTRACTION",
+    "FAILED_PROVIDER",
+    "EMPTY_UNKNOWN",
+    "LEGACY_UNVERIFIED",
+}
+_TOP_KEYS = {
+    "schema_version", "status", "source_video", "audio", "asr_result",
+    "glossary", "precision", "windows",
+}
+_WINDOW_KEYS = {
+    "index", "start", "end", "text_availability", "observed_text",
+    "post_glossary_text", "glossary_modified",
+}
+_GLOSSARY_KEYS = {"names", "name_count"}
+_PRECISION = {
+    "window_timing": "COARSE_SEGMENT_WINDOWS",
+    "dialogue_boundaries": "NOT_VERIFIED",
+    "word_alignment": "NOT_PERFORMED",
+    "empty_text_meaning": "UNKNOWN_NOT_PROVEN_SILENCE",
+}
+
+
+def _identity(path):
+    """{size, mtime_ns} of a bound file, or None when it does not exist."""
+    path = Path(path)
+    return file_identity(path) if path.exists() else None
+
+
+def load_glossary_names(work_dir):
+    """Return normalized names that can actually affect ASR correction."""
+    data = load_background_research(work_dir)
+    names = set()
+    characters = data.get("characters")
+    if isinstance(characters, dict):
+        names.update(characters.keys())
+    details = data.get("character_details")
+    if isinstance(details, dict):
+        for name, info in details.items():
+            names.add(name)
+            if isinstance(info, dict) and isinstance(info.get("aliases"), list):
+                names.update(alias for alias in info["aliases"] if isinstance(alias, str))
+    return sorted(
+        {name for name in names if isinstance(name, str) and len(name) >= 2},
+        key=lambda value: (-len(value), value),
+    )
+
+
+def _glossary_binding(work_dir, *, legacy=False):
+    """The glossary names that could have corrected this transcription (legacy: unknown)."""
+    if legacy:
+        return {"names": None, "name_count": None}
+    names = load_glossary_names(work_dir)
+    return {"names": names, "name_count": len(names)}
+
+
+def _atomic_json_write(path, payload):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            json.dump(payload, handle, ensure_ascii=False, indent=2)
+            handle.write("\n")
+        os.replace(temporary, path)
+    finally:
+        try:
+            os.unlink(temporary)
+        except FileNotFoundError:
+            pass
+
+
+def _window_evidence(observed_segments, final_segments, legacy):
+    observed_segments = observed_segments or []
+    windows = []
+    for index, final in enumerate(final_segments or []):
+        observed = observed_segments[index] if index < len(observed_segments) else None
+        observed_text = None if legacy or observed is None else str(observed.get("text") or "")
+        final_text = str(final.get("text") or "")
+        windows.append({
+            "index": index,
+            "start": final.get("start"),
+            "end": final.get("end"),
+            "text_availability": "AVAILABLE" if final_text else "UNAVAILABLE_UNKNOWN",
+            "observed_text": observed_text,
+            "post_glossary_text": final_text,
+            "glossary_modified": None if observed_text is None else observed_text != final_text,
+        })
+    return windows
+
+
+def write_asr_timing_evidence(
+    work_dir, video_path, status, *, observed_segments=None, final_segments=None,
+    audio_path=None,
+):
+    """Write a sidecar recording which source/audio/result files it describes, without
+    changing the legacy result."""
+    if status not in VALID_STATUSES:
+        raise ValueError(f"unknown ASR timing evidence status: {status}")
+    work_dir = Path(work_dir)
+    legacy = status == "LEGACY_UNVERIFIED"
+    payload = {
+        "schema_version": SCHEMA_VERSION,
+        "status": status,
+        "source_video": _identity(video_path),
+        "audio": _identity(audio_path) if audio_path else None,
+        "asr_result": _identity(work_dir / "asr_result.json"),
+        "glossary": _glossary_binding(work_dir, legacy=legacy),
+        "precision": dict(_PRECISION),
+        "windows": _window_evidence(observed_segments, final_segments, legacy),
+    }
+    path = work_dir / EVIDENCE_FILENAME
+    _atomic_json_write(path, payload)
+    return path
+
+
+def _is_int(value):
+    return isinstance(value, int) and not isinstance(value, bool)
+
+
+def _is_number(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
+
+
+def _is_identity(value):
+    return (
+        isinstance(value, dict) and set(value) == {"size", "mtime_ns"}
+        and _is_int(value["size"]) and _is_int(value["mtime_ns"])
+    )
+
+
+def _valid_glossary(payload, work_dir, legacy):
+    if not isinstance(payload, dict) or set(payload) != _GLOSSARY_KEYS:
+        return False
+    if legacy:
+        return payload == _glossary_binding(work_dir, legacy=True)
+    return payload == _glossary_binding(work_dir) and _is_int(payload.get("name_count"))
+
+
+def _valid_status_relationships(status, result, audio):
+    texts = [str(segment.get("text") or "") for segment in result]
+    if status == "AVAILABLE_COARSE":
+        return bool(result) and any(texts) and _is_identity(audio)
+    if status == "EMPTY_UNKNOWN":
+        return bool(result) and not any(texts) and _is_identity(audio)
+    if status == "UNAVAILABLE_NO_DURATION":
+        return result == [] and _is_identity(audio)
+    if status in {"EXPLICITLY_SKIPPED", "UNAVAILABLE_NO_KEY", "LEGACY_UNVERIFIED"}:
+        return audio is None and (status == "LEGACY_UNVERIFIED" or result == [])
+    return False
+
+
+def validate_asr_timing_evidence(evidence_path, video_path, asr_result_path):
+    """Return whether a sidecar still describes the current source/audio/result files
+    (by size + mtime_ns) and the limited timing it claims."""
+    evidence_path = Path(evidence_path)
+    result_path = Path(asr_result_path)
+    try:
+        evidence = json.loads(evidence_path.read_text(encoding="utf-8"))
+        result = json.loads(result_path.read_text(encoding="utf-8"))
+    except (OSError, ValueError, TypeError):
+        return False
+    if not isinstance(evidence, dict) or set(evidence) != _TOP_KEYS:
+        return False
+    if not isinstance(result, list) or not all(isinstance(item, dict) for item in result):
+        return False
+    if not _is_int(evidence.get("schema_version")) or evidence["schema_version"] != SCHEMA_VERSION:
+        return False
+    status = evidence.get("status")
+    if status not in VALID_STATUSES or status.startswith("FAILED_"):
+        return False
+    if evidence.get("precision") != _PRECISION:
+        return False
+    source_identity = _identity(video_path)
+    result_identity = _identity(result_path)
+    if not _is_identity(source_identity) or evidence.get("source_video") != source_identity:
+        return False
+    if not _is_identity(result_identity) or evidence.get("asr_result") != result_identity:
+        return False
+    audio = evidence.get("audio")
+    if audio is not None and not _is_identity(audio):
+        return False
+    if not _valid_status_relationships(status, result, audio):
+        return False
+    legacy = status == "LEGACY_UNVERIFIED"
+    if not _valid_glossary(evidence.get("glossary"), evidence_path.parent, legacy):
+        return False
+
+    if audio is not None:
+        audio_path = evidence_path.parent / "audio.wav"
+        if audio != _identity(audio_path):
+            return False
+        try:
+            audio_meta = json.loads(
+                (evidence_path.parent / "audio.wav.meta.json").read_text(encoding="utf-8")
+            )
+        except (OSError, ValueError, TypeError):
+            return False
+        if not isinstance(audio_meta, dict):
+            return False
+        if audio_meta.get("source_video_identity") != source_identity:
+            return False
+
+    windows = evidence.get("windows")
+    if not isinstance(windows, list) or len(windows) != len(result):
+        return False
+    any_modified = False
+    for index, (window, segment) in enumerate(zip(windows, result)):
+        if not isinstance(window, dict) or set(window) != _WINDOW_KEYS:
+            return False
+        start, end = segment.get("start"), segment.get("end")
+        text = str(segment.get("text") or "")
+        observed = window.get("observed_text")
+        modified = window.get("glossary_modified")
+        if not _is_number(start) or not _is_number(end) or end <= start:
+            return False
+        if (
+            not _is_int(window.get("index")) or window["index"] != index
+            or not _is_number(window.get("start")) or not _is_number(window.get("end"))
+            or window["start"] != start or window["end"] != end
+            or not isinstance(window.get("post_glossary_text"), str)
+            or window["post_glossary_text"] != text
+            or window.get("text_availability") != ("AVAILABLE" if text else "UNAVAILABLE_UNKNOWN")
+        ):
+            return False
+        if legacy:
+            if observed is not None or modified is not None:
+                return False
+        elif (
+            not isinstance(observed, str) or not isinstance(modified, bool)
+            or modified != (observed != text)
+        ):
+            return False
+        any_modified = any_modified or modified is True
+    if any_modified and evidence["glossary"]["name_count"] == 0:
+        return False
+    return True
+
+
+def asr_evidence_summary_for_brief(work_dir, video_path):
+    """Return a compact validated banner; never upgrade missing/stale evidence."""
+    work_dir = Path(work_dir)
+    evidence_path = work_dir / EVIDENCE_FILENAME
+    result_path = work_dir / "asr_result.json"
+    valid = bool(video_path) and validate_asr_timing_evidence(
+        evidence_path, video_path, result_path
+    )
+    if valid:
+        payload = json.loads(evidence_path.read_text(encoding="utf-8"))
+        return {"status": payload["status"],
+                "glossary_modifications": None if payload["status"] == "LEGACY_UNVERIFIED"
+                else sum(w["glossary_modified"] is True for w in payload["windows"])}
+    return {"status": "MISSING_OR_STALE", "glossary_modifications": None}

package/resources/skills/video-recap-production/upstream/skills/video-understanding/scripts/brief.py

@@ -0,0 +1,9 @@
+"""Public narration-brief API for this self-contained skill."""
+
+from briefing.builder import build_agent_brief
+from briefing.context import assess_understanding_substrate
+
+__all__ = [
+    "assess_understanding_substrate",
+    "build_agent_brief",
+]

package/resources/skills/video-recap-production/upstream/skills/video-understanding/scripts/briefing/__init__.py

@@ -0,0 +1 @@
+"""Brief-generation subpackage: builds the creative agent brief from ASR/scene/timeline evidence."""