@oh-story/dsh 0.1.3 → 0.1.5

package/lib/video-recap/skills/video-recap/scripts/mimo_qc_payload.py

@@ -0,0 +1,227 @@
+"""Build bounded semantic and multimodal MiMo QC payloads."""
+
+from __future__ import annotations
+
+
+import json
+
+from typing import Any, Mapping, Sequence
+
+
+from mimo_qc_evidence import (
+    _cache_evidence,
+    _fingerprint_value,
+    _redact,
+    safe_mimo_config,
+)
+from mimo_qc_contract import ARTIFACT_NAME, DEFAULT_STAGE, MAX_FRAMES
+
+
+def _semantic_evidence(evidence: Mapping[str, Any], *, stage: str) -> dict[str, Any]:
+    """Keep real artifact values visible to MiMo without sending irrelevant paths.
+
+    ``collect_evidence`` already bounds every artifact independently.  The additional
+    summarization here therefore needs enough depth to retain the narration/ASR/TTS
+    scalars nested inside those summaries.  A shallow second pass used to replace the
+    values with fingerprints, which made live QC invent missing-script and failed-TTS
+    findings from evidence it could no longer read.
+    """
+    semantic = dict(evidence)
+    semantic.pop("work_dir", None)
+    semantic.pop("fingerprint", None)
+    semantic["evidence_roles"] = {
+        "narration.json": (
+            "Planned recap voiceover on the OUTPUT timeline; judge its factual and temporal "
+            "fit rather than expecting verbatim source dialogue."
+        ),
+        "tts_meta.json": (
+            "Synthesis and placement metadata for narration.json; an empty failures list means "
+            "all requested narration segments synthesized successfully."
+        ),
+        "source_asr": (
+            "Transcript of the SOURCE media audio, not a transcript of generated TTS. "
+            "Wording differences from narration are expected; use this evidence for factual "
+            "support and original-dialogue timing, not verbatim equality."
+        ),
+        "generated_subtitles": (
+            "Recap subtitles derived from generated narration/TTS during assembly. They are not "
+            "source ASR and must never be used as independent factual support. Visible text that "
+            "matches one of these cues is the intended generated caption, including when it sits "
+            "on the black source-subtitle mask band."
+        ),
+        "visual_metadata": (
+            "Diagnostic storyboard/frame metadata. labels_burned refers only to diagnostic "
+            "timestamp labels and does not mean labels were burned into the final video."
+        ),
+        "final_output": (
+            "Only present for post_render and limited to candidates that actually exist."
+        ),
+        "post_render_frame_limits": (
+            "Sampled final-output frames are silent still images: they contain no audible audio. "
+            "Do not infer which audio track is playing from visible source captions. Narration/TTS "
+            "timings are segment-level, not word-level, so an internal phrase cannot be assigned to "
+            "an exact sampled-frame timestamp."
+        ),
+        "narration_and_subtitle_gaps": (
+            "Generated narration/subtitle gaps may be deliberate original-audio blocks. In those "
+            "gaps the source audio and its existing captions can intentionally return; do not call "
+            "the absence of generated subtitles a defect without evidence of accidental silence."
+        ),
+        "actual_audio_timing": (
+            "narration.json start/end values are authoring slots, not proof that speech fills the "
+            "whole slot. For post-render timing, assembly_manifest audio_segments "
+            "actual_place_start/actual_place_end are authoritative. A sampled frame after "
+            "actual_place_end is in an original-audio gap."
+        ),
+    }
+    if stage != "post_render":
+        semantic.pop("final_output", None)
+        # These files can be leftovers from an earlier assembly when narration is being revised.
+        # Pre-assemble QC must evaluate current narration/TTS, not stale rendered captions.
+        semantic.pop("generated_subtitles", None)
+        semantic["evidence_roles"].pop("generated_subtitles", None)
+        semantic["evidence_roles"].pop("post_render_frame_limits", None)
+        semantic["evidence_roles"].pop("actual_audio_timing", None)
+    else:
+        final_output = semantic.get("final_output")
+        candidates = (
+            final_output.get("candidates", [])
+            if isinstance(final_output, Mapping)
+            else []
+        )
+        semantic["final_output"] = {
+            "candidates": [
+                dict(candidate)
+                for candidate in candidates
+                if isinstance(candidate, Mapping) and candidate.get("exists") is True
+            ]
+        }
+    # Artifact summaries are already bounded at collection time; summarizing them again
+    # would wrap their ``items`` arrays in another summary layer and obscure the values.
+    return _redact(semantic)
+
+
+def build_payload(
+    evidence: Mapping[str, Any],
+    *,
+    stage: str = DEFAULT_STAGE,
+    config: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build the report-safe semantic payload (never contains image base64)."""
+    cfg = safe_mimo_config(config)
+    payload = {
+        "stage": stage,
+        "artifact": ARTIFACT_NAME,
+        "model": cfg["model"],
+        "config": cfg,
+        "instructions": (
+            "你是 video-recap 的建议性质量审阅器。结合解说、剪辑计划、字幕/ASR、TTS、"
+            "组装元数据和抽样画面，指出语义或审美问题。只返回主观观察，不做自动修复，"
+            "只根据可见的实际字段判断，不得从文件字节数、指纹、截断或省略标记推断内容缺失；"
+            "source_asr 是源素材原声证据，不是生成后 TTS 的逐字转录；generated_subtitles 是本轮生成字幕，"
+            "二者角色不可混淆，解说改写与源台词不一致本身不是问题；"
+            "空 failures 表示没有失败，storyboard 的 labels_burned 仅表示诊断图上的时间标签，"
+            "不代表标签烧进最终视频；pre_assemble 阶段尚无 final_output 属于正常状态；"
+            "成片抽样画面是带时间戳的稀疏点样本，只能判断该具体时刻，不得用单帧否定整段内其他时刻的画面。"
+            "这些抽样帧是无声静帧，不能从画面中的源字幕推断当时实际播放哪条音轨；"
+            "源字幕遮罩策略为 off 时，保留的源字幕无需与解说逐字一致；解说/TTS 只有段级时间，"
+            "没有词级对齐，不得把句中某个短语强行对应到抽样帧的精确秒数。"
+            "生成解说和字幕之间的空档可能是刻意保留的原声块，此时原声和源字幕回归是正常设计，"
+            "没有意外静音证据时不得把生成字幕空档当成缺失。只输出可采取行动的疑似问题，"
+            "不要把正常、一致、相符或通过项作为 observation。不得提出阻断决定。"
+            "narration.json 的 start/end 是创作槽位，不表示旁白铺满整段；成片阶段必须以 "
+            "assembly_manifest.audio_segments 的 actual_place_start/actual_place_end 判断实际旁白。"
+            "抽样点超过 actual_place_end 时属于原声块；一个字幕时间段含多个分句时，静帧匹配其中任一分句都不算错位。"
+            "只有同一编号静帧中同时清晰可读两层不同字幕，才能报告字幕重叠；不同静帧分别出现解说字幕和源字幕不算重叠。"
+            "遮罩 opacity=1.0 表示源字幕像素已被不透明覆盖，不得臆测遮罩下仍有可见文字。"
+            "黑色遮罩带上与 generated_subtitles cue 一致的白字是本轮预期生成字幕，不是残留源字幕；"
+            "必须先逐字对照 generated_subtitles，再判断是否另有第二层源字幕。"
+            "每张抽样图都由同编号 BEGIN/END 文本包围，必须按编号和时间戳独立判断，"
+            "不得把一张图里的文字或人物归到另一张图。"
+            '返回 JSON：{"observations":[{"code":...,'
+            '"message":...,"category":"semantic|aesthetic",'
+            '"confidence":"low|medium|high","sample_policy":'
+            '"semantic|aesthetic|sampled","evidence":{...}}]}。最多 12 条。'
+        ),
+        "evidence": _semantic_evidence(evidence, stage=stage),
+        "evidence_fingerprint": evidence.get("fingerprint")
+        or _fingerprint_value(_cache_evidence(evidence)),
+    }
+    payload["payload_fingerprint"] = _fingerprint_value(payload)
+    return _redact(payload)
+
+
+def _request_payload(
+    payload: Mapping[str, Any], frame_samples: Sequence[Mapping[str, Any]]
+) -> dict[str, Any]:
+    request_evidence = {
+        "stage": payload["stage"],
+        "instructions": payload["instructions"],
+        "evidence": payload["evidence"],
+    }
+    content: list[dict[str, Any]] = [
+        {
+            "type": "text",
+            "text": json.dumps(
+                request_evidence, ensure_ascii=False, separators=(",", ":")
+            ),
+        }
+    ]
+    for index, sample in enumerate(list(frame_samples)[:MAX_FRAMES], start=1):
+        data_url = sample.get("data_url") if isinstance(sample, Mapping) else None
+        if isinstance(data_url, str) and data_url.startswith("data:image/jpeg;base64,"):
+            timestamp = sample.get("timestamp")
+            try:
+                timestamp_label = f"{float(timestamp):.3f}s"
+            except (TypeError, ValueError):
+                timestamp_label = "unknown"
+            content.append(
+                {
+                    "type": "text",
+                    "text": (
+                        f"BEGIN QC_FRAME_{index}: final-output timestamp {timestamp_label}. "
+                        f"Judge only this image and do not transfer its content to another frame."
+                    ),
+                }
+            )
+            content.append({"type": "image_url", "image_url": {"url": data_url}})
+            content.append(
+                {
+                    "type": "text",
+                    "text": f"END QC_FRAME_{index}: timestamp {timestamp_label}.",
+                }
+            )
+    return {
+        "model": payload["model"],
+        "messages": [{"role": "user", "content": content}],
+        "max_completion_tokens": 1600,
+        "thinking": {"type": "disabled"},
+    }
+
+
+def _strip_json_fence(text: str) -> str:
+    value = text.strip()
+    if value.startswith("```") and value.endswith("```"):
+        lines = value.splitlines()
+        if len(lines) >= 3:
+            value = "\n".join(lines[1:-1]).strip()
+    return value
+
+
+def _validated_live_output(response: Any) -> Any:
+    try:
+        content = response["choices"][0]["message"]["content"]
+    except (KeyError, IndexError, TypeError):
+        raise ValueError("malformed_response") from None
+    if isinstance(content, str):
+        try:
+            content = json.loads(_strip_json_fence(content))
+        except (TypeError, ValueError):
+            raise ValueError("malformed_json_content") from None
+    if isinstance(content, list):
+        return content
+    if isinstance(content, Mapping) and isinstance(content.get("observations"), list):
+        return content
+    if isinstance(content, Mapping) and isinstance(content.get("findings"), list):
+        return content
+    raise ValueError("malformed_observations")

package/lib/video-recap/skills/video-recap/scripts/mimo_qc_report.py

@@ -0,0 +1,362 @@
+"""Sample frames and build aggregate MiMo QC reports."""
+
+from __future__ import annotations
+
+
+import base64
+
+import hashlib
+
+import json
+
+import math
+
+import os
+
+
+import subprocess
+
+import tempfile
+
+from pathlib import Path
+
+from typing import Any, Callable, Mapping, Sequence
+
+import qc_contract
+
+from mimo_qc_evidence import (
+    _cache_evidence,
+    _effective_config,
+    _existing_final_output,
+    _fingerprint_value,
+    _redact,
+    collect_evidence,
+    safe_mimo_config,
+)
+from mimo_qc_observations import normalize_observations
+from mimo_qc_payload import _request_payload, _validated_live_output, build_payload
+from mimo_qc_client import mimo_qc_api_call
+from mimo_qc_contract import (
+    ARTIFACT_NAME,
+    DEFAULT_STAGE,
+    FRAME_SAMPLER_VERSION,
+    MAX_FRAME_DIMENSION,
+    MAX_FRAMES,
+)
+
+JudgeCallable = Callable[
+    [Mapping[str, Any]], Mapping[str, Any] | Sequence[Mapping[str, Any]]
+]
+
+FrameSampler = Callable[..., Sequence[Mapping[str, Any]]]
+
+
+def _probe_duration(path: Path) -> float:
+    result = subprocess.run(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "json",
+            str(path),
+        ],
+        capture_output=True,
+        text=True,
+        timeout=20,
+        check=False,
+    )
+    if result.returncode != 0:
+        return 0.0
+    try:
+        return max(0.0, float(json.loads(result.stdout)["format"]["duration"]))
+    except (KeyError, TypeError, ValueError, json.JSONDecodeError):
+        return 0.0
+
+
+def sample_video_frames(
+    video_path: str | Path,
+    *,
+    max_frames: int = MAX_FRAMES,
+    max_dimension: int = MAX_FRAME_DIMENSION,
+) -> list[dict[str, Any]]:
+    """Extract at most six bounded JPEGs; returned base64 is request-only."""
+    path = Path(video_path)
+    if not path.is_file() or max_frames <= 0:
+        return []
+    try:
+        duration = _probe_duration(path)
+    except (OSError, subprocess.SubprocessError):
+        return []
+    if duration <= 0:
+        return []
+    count = min(int(max_frames), max(1, int(math.ceil(duration / 15.0))))
+    samples = []
+    with tempfile.TemporaryDirectory(prefix="video-recap-mimo-qc-") as tmp:
+        for index in range(count):
+            timestamp = duration * (index + 0.5) / count
+            destination = Path(tmp) / f"frame-{index:02d}.jpg"
+            try:
+                result = subprocess.run(
+                    [
+                        "ffmpeg",
+                        "-hide_banner",
+                        "-loglevel",
+                        "error",
+                        "-y",
+                        "-ss",
+                        f"{timestamp:.3f}",
+                        "-i",
+                        str(path),
+                        "-frames:v",
+                        "1",
+                        "-vf",
+                        (
+                            f"scale={int(max_dimension)}:{int(max_dimension)}:"
+                            "force_original_aspect_ratio=decrease"
+                        ),
+                        "-q:v",
+                        "4",
+                        str(destination),
+                    ],
+                    capture_output=True,
+                    timeout=30,
+                    check=False,
+                )
+            except (OSError, subprocess.SubprocessError):
+                continue
+            if result.returncode != 0 or not destination.is_file():
+                continue
+            raw = destination.read_bytes()
+            samples.append(
+                {
+                    "data_url": "data:image/jpeg;base64,"
+                    + base64.b64encode(raw).decode("ascii"),
+                    "sha256": hashlib.sha256(raw).hexdigest(),
+                    "timestamp": round(timestamp, 3),
+                }
+            )
+    return samples[:MAX_FRAMES]
+
+
+def _frame_metadata(samples: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
+    return {
+        "count": min(len(samples), MAX_FRAMES),
+        "max_frames": MAX_FRAMES,
+        "max_dimension": MAX_FRAME_DIMENSION,
+        "sampler_version": FRAME_SAMPLER_VERSION,
+        "samples": [
+            {
+                "sha256": sample.get("sha256"),
+                "timestamp": sample.get("timestamp"),
+            }
+            for sample in list(samples)[:MAX_FRAMES]
+        ],
+    }
+
+
+def _cache_input(
+    stage: str,
+    payload: Mapping[str, Any],
+    evidence: Mapping[str, Any],
+    frames: Mapping[str, Any],
+) -> dict[str, Any]:
+    return {
+        "stage": stage,
+        "model": payload.get("model"),
+        "payload_fingerprint": payload.get("payload_fingerprint"),
+        "evidence": _cache_evidence(evidence),
+        "frames": frames,
+        "contract": qc_contract.SCHEMA_VERSION,
+    }
+
+
+def _stage_reports(path: Path) -> dict[str, dict[str, Any]]:
+    if not path.is_file():
+        return {}
+    try:
+        report = json.loads(path.read_text(encoding="utf-8"))
+        qc_contract.validate_report(report)
+    except (OSError, ValueError, TypeError, qc_contract.QCContractError):
+        return {}
+    stages = report.get("metadata", {}).get("stages")
+    if isinstance(stages, Mapping):
+        valid = {}
+        for stage, stage_report in stages.items():
+            try:
+                qc_contract.validate_report(stage_report)
+            except (TypeError, qc_contract.QCContractError):
+                continue
+            valid[str(stage)] = dict(stage_report)
+        return valid
+    return {str(report["stage"]): report}
+
+
+def _error_name(exc: Exception) -> str:
+    text = str(_redact(str(exc))).strip()
+    return (text or type(exc).__name__)[:120]
+
+
+def build_report(
+    work_dir: str | Path,
+    *,
+    stage: str = DEFAULT_STAGE,
+    fixture: Any | None = None,
+    dry_run: bool = False,
+    judge: JudgeCallable | None = None,
+    config: Mapping[str, Any] | None = None,
+    final_output: str | Path | None = None,
+    live: bool = False,
+    refresh: bool = False,
+    frame_sampler: FrameSampler | None = None,
+    existing: Mapping[str, Any] | None = None,
+    api_call: Callable[..., Any] | None = None,
+) -> dict[str, Any]:
+    """Build one validated stage report; all live failures remain successful QC."""
+    root = Path(work_dir)
+    evidence = collect_evidence(root, final_output=final_output)
+    payload = build_payload(evidence, stage=stage, config=config)
+    cfg = safe_mimo_config(config)
+    samples: Sequence[Mapping[str, Any]] = []
+    if stage == "post_render" and (fixture is not None or judge is not None or live):
+        output = _existing_final_output(root, final_output)
+        if output is not None:
+            sampler = frame_sampler or sample_video_frames
+            try:
+                samples = list(
+                    sampler(
+                        output, max_frames=MAX_FRAMES, max_dimension=MAX_FRAME_DIMENSION
+                    )
+                )[:MAX_FRAMES]
+            except Exception:
+                samples = []
+    frame_meta = _frame_metadata(samples)
+    cache_input = _cache_input(stage, payload, evidence, frame_meta)
+    cache_key = _fingerprint_value(cache_input)
+
+    if (
+        not refresh
+        and existing
+        and existing.get("metadata", {}).get("cache_key") == cache_key
+    ):
+        cached = json.loads(json.dumps(existing))
+        cached["metadata"]["status"] = "cached"
+        cached["metadata"]["mode"] = "live_cache"
+        cached["metadata"]["request_count"] = 0
+        qc_contract.validate_report(cached)
+        return cached
+
+    status = "completed"
+    error = None
+    if fixture is not None:
+        model_output = fixture
+        mode = "fixture"
+    elif judge is not None and not dry_run:
+        mode = "injected_judge"
+        try:
+            model_output = judge(payload)
+        except Exception as exc:
+            model_output = {"observations": []}
+            status, error = "failed", _error_name(exc)
+    elif live and not dry_run:
+        mode = "live"
+        if not cfg["key_present"]:
+            model_output = {"observations": []}
+            status, error = "unavailable", "missing_key"
+        else:
+            try:
+                response = (api_call or mimo_qc_api_call)(
+                    _request_payload(payload, samples),
+                    config=_effective_config(config),
+                    timeout=60,
+                )
+                model_output = _validated_live_output(response)
+            except Exception as exc:
+                model_output = {"observations": []}
+                status, error = "failed", _error_name(exc)
+    else:
+        model_output = {"observations": []}
+        mode = "dry_run"
+        status = "dry_run"
+
+    findings = normalize_observations(
+        model_output, stage=stage, payload=payload, model_config=config
+    )
+    metadata = {
+        "mode": mode,
+        "status": status,
+        "error": error,
+        "report_only": True,
+        "auto_repair": False,
+        "pipeline_blocking": False,
+        "deterministic": False,
+        "request_count": 1
+        if mode == "live" and status in {"completed", "failed"}
+        else 0,
+        "cache_key": cache_key,
+        "cache_input": cache_input,
+        "frame_samples": frame_meta,
+        "evidence": evidence,
+        "payload": payload,
+        "config": cfg,
+    }
+    report = qc_contract.build_report(
+        artifact=ARTIFACT_NAME,
+        stage=stage,
+        findings=findings,
+        metadata=metadata,
+    )
+    qc_contract.validate_report(report)
+    return _redact(report)
+
+
+def _aggregate_reports(
+    stage_reports: Mapping[str, Mapping[str, Any]], current_stage: str
+) -> dict[str, Any]:
+    current = stage_reports[current_stage]
+    findings = [
+        finding
+        for stage_report in stage_reports.values()
+        for finding in stage_report.get("findings", [])
+    ]
+    metadata = dict(current.get("metadata", {}))
+    metadata["stages"] = {
+        stage: dict(report) for stage, report in stage_reports.items()
+    }
+    return qc_contract.build_report(
+        artifact=ARTIFACT_NAME,
+        stage=current_stage,
+        findings=findings,
+        metadata=metadata,
+    )
+
+
+def write_report(
+    work_dir: str | Path, report: Mapping[str, Any], *, output: str | Path | None = None
+) -> tuple[Path, dict[str, Any]]:
+    """Atomically merge one stage into mimo_qc.json and return the aggregate."""
+    path = Path(output) if output else Path(work_dir) / ARTIFACT_NAME
+    path.parent.mkdir(parents=True, exist_ok=True)
+    safe_stage = _redact(dict(report))
+    qc_contract.validate_report(safe_stage)
+    stages = _stage_reports(path)
+    stages[safe_stage["stage"]] = safe_stage
+    aggregate = _aggregate_reports(stages, safe_stage["stage"])
+    qc_contract.validate_report(aggregate)
+    serialized = json.dumps(_redact(aggregate), ensure_ascii=False, indent=2) + "\n"
+    descriptor, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
+    try:
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(serialized)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temp_name, path)
+    except Exception:
+        try:
+            os.unlink(temp_name)
+        except OSError:
+            pass
+        raise
+    return path, aggregate