@remixmate/cli 0.9.57 → 0.9.59

package/skills/render-video/scripts/_digital_human_segments.py

@@ -0,0 +1,204 @@
+"""数字人连续段：分段、驱动音频规划、起播帧注入。
+
+一个 ``source=gen-digital-human`` 的 avatar 资产被**相邻**几个场景的
+``props.avatarAssetId`` 同时引用 = 一个连续段：生成一段视频，人物跨场景不重启。
+要让第 k 个场景的第 0 帧正好对上视频里的对应位置，驱动音频必须是「段内各场景
+旁白、每段补静音到场景最终时长」拼成的一条——所以数字人只能在**排轴之后**生成
+（场景时长由真实 TTS 时长决定，见 render_video.adjust_timeline_to_audio）。
+
+本模块只放纯函数与 ffmpeg 拼接；调度（何时生成、如何回写 RenderPlan 状态）在
+render_video.resolve_digital_human_segments。全部按 RenderPlan 的通用字段工作，
+不识别具体模板：模板是否支持由 template.json 的 ``capabilities.digitalHuman`` 声明，
+写没写 avatar 资产由上游决定。
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from typing import Optional
+
+DIGITAL_HUMAN_SOURCE = "gen-digital-human"
+
+#: 驱动音频下限。飞影要求 5 秒 ～ 30 分钟；不足时在段尾补静音。
+MIN_DRIVE_SEC = 5.0
+
+#: 拼接输出的采样率 / 声道。口型驱动不需要立体声，统一成单声道避免混格式失败。
+_SAMPLE_RATE = 44100
+
+
+def _entry_props(entry: dict) -> dict:
+    props = entry.get("props")
+    return props if isinstance(props, dict) else {}
+
+
+def digital_human_asset_ids(render_plan: dict) -> set[str]:
+    return {
+        a.get("assetId")
+        for a in render_plan.get("resolvedAssets", [])
+        if a.get("source") == DIGITAL_HUMAN_SOURCE and a.get("assetId")
+    }
+
+
+def collect_segments(render_plan: dict) -> dict[str, list[int]]:
+    """avatar 资产 id → 引用它的 timeline 下标（按出场顺序）。
+
+    只收 gen-digital-human 资产：``existing`` 的数字人视频是作者自备的，起播点由作者
+    自己负责（``digitalHuman.startFromSec``），管线不去猜。
+    """
+    dh_ids = digital_human_asset_ids(render_plan)
+    segments: dict[str, list[int]] = {}
+    for idx, entry in enumerate(render_plan.get("timeline", [])):
+        ref = _entry_props(entry).get("avatarAssetId")
+        if ref in dh_ids:
+            segments.setdefault(ref, []).append(idx)
+    return segments
+
+
+def is_contiguous(indices: list[int]) -> bool:
+    return all(b == a + 1 for a, b in zip(indices, indices[1:]))
+
+
+def segment_role(pos: int, count: int) -> str:
+    if count == 1:
+        return "only"
+    if pos == 0:
+        return "first"
+    return "last" if pos == count - 1 else "middle"
+
+
+def inject_segment_props(render_plan: dict, segments: dict[str, list[int]]) -> list[str]:
+    """给段内每个场景写 ``avatarOffsetFrames`` / ``avatarSegmentRole``。
+
+    偏移用 RenderPlan 里最终的 ``durationFrames`` 累加，不按秒重算，避免累计舍入。
+    返回告警（不相邻的同一资产：偏移仍按出场顺序累加，但中间隔开的场景会让口型错位）。
+    """
+    warnings: list[str] = []
+    timeline = render_plan.get("timeline", [])
+    for asset_id, indices in segments.items():
+        if not is_contiguous(indices):
+            scene_ids = [timeline[i].get("sceneId", str(i)) for i in indices]
+            warnings.append(
+                f"digital-human asset {asset_id} is referenced by non-adjacent scenes "
+                f"{scene_ids}; one video can only span adjacent scenes — lip sync will drift"
+            )
+        offset = 0
+        for pos, idx in enumerate(indices):
+            entry = timeline[idx]
+            props = entry.setdefault("props", {})
+            props["avatarOffsetFrames"] = offset
+            props["avatarSegmentRole"] = segment_role(pos, len(indices))
+            offset += int(entry.get("durationFrames") or 0)
+    return warnings
+
+
+def plan_drive_pieces(render_plan: dict, indices: list[int]) -> list[dict]:
+    """段内每个场景一块：``{"sceneId", "url" | None, "durationSec"}``。
+
+    ``url`` 取场景旁白资产（``props.narrationAssetId``）已生成的地址；没有旁白的场景
+    是整段静音。块长 = 场景最终时长；总长不足 MIN_DRIVE_SEC 时把最后一块拉长。
+    """
+    fps = (render_plan.get("renderConfig") or {}).get("fps") or 30
+    assets = {a.get("assetId"): a for a in render_plan.get("resolvedAssets", [])}
+    timeline = render_plan.get("timeline", [])
+    pieces: list[dict] = []
+    for idx in indices:
+        entry = timeline[idx]
+        nar = assets.get(_entry_props(entry).get("narrationAssetId"))
+        url = nar.get("url") if nar and nar.get("status") == "generated" else None
+        pieces.append({
+            "sceneId": entry.get("sceneId", str(idx)),
+            "url": url or None,
+            "durationSec": round(int(entry.get("durationFrames") or 0) / fps, 3),
+        })
+    total = sum(p["durationSec"] for p in pieces)
+    if pieces and total < MIN_DRIVE_SEC:
+        pieces[-1]["durationSec"] = round(pieces[-1]["durationSec"] + MIN_DRIVE_SEC - total, 3)
+    return pieces
+
+
+def direct_drive_url(render_plan: dict, indices: list[int]) -> Optional[str]:
+    """单场景段、且旁白本身就够长时，直接拿旁白音频驱动，不必拼接。
+
+    视频比场景短几百毫秒（旁白尾巴）无妨：模板默认 ``onEnd: freeze`` 停在最后一帧。
+    """
+    if len(indices) != 1:
+        return None
+    entry = render_plan.get("timeline", [])[indices[0]]
+    assets = {a.get("assetId"): a for a in render_plan.get("resolvedAssets", [])}
+    nar = assets.get(_entry_props(entry).get("narrationAssetId"))
+    if not nar or nar.get("status") != "generated" or not nar.get("url"):
+        return None
+    duration_ms = nar.get("duration") or 0
+    if duration_ms and duration_ms / 1000.0 >= MIN_DRIVE_SEC:
+        return nar["url"]
+    return None
+
+
+def split_segment_per_scene(render_plan: dict, asset_id: str, indices: list[int]) -> list[str]:
+    """把一个连续段拆成每场景一段（拿不到 ffmpeg 拼接时的降级）。
+
+    人物会在场景边界重新起播，但每段都能直接用本场景旁白驱动，口型不会错位。
+    新资产 id = ``<原 id>.<sceneId>``；原资产从 resolvedAssets 移除，场景改指新 id。
+    """
+    assets = render_plan.get("resolvedAssets", [])
+    original = next((a for a in assets if a.get("assetId") == asset_id), None)
+    if original is None:
+        return []
+    timeline = render_plan.get("timeline", [])
+    new_ids: list[str] = []
+    for idx in indices:
+        entry = timeline[idx]
+        new_id = f"{asset_id}.{entry.get('sceneId', idx)}"
+        clone = {**original, "assetId": new_id, "status": "pending", "url": "", "splitFrom": asset_id}
+        assets.append(clone)
+        entry.setdefault("props", {})["avatarAssetId"] = new_id
+        new_ids.append(new_id)
+    render_plan["resolvedAssets"] = [a for a in assets if a.get("assetId") != asset_id]
+    return new_ids
+
+
+def build_concat_command(pieces: list[dict], out_path: str, ffmpeg: str = "ffmpeg") -> list[str]:
+    """ffmpeg 参数：每块重采样成单声道 → 补静音 → 精确截到块长 → 顺序拼接成 mp3。"""
+    cmd = [ffmpeg, "-y", "-loglevel", "error"]
+    filters: list[str] = []
+    labels: list[str] = []
+    for i, piece in enumerate(pieces):
+        dur = f"{max(piece['durationSec'], 0.01):.3f}"
+        if piece.get("url"):
+            cmd += ["-i", piece["url"]]
+        else:
+            cmd += ["-f", "lavfi", "-t", dur, "-i", f"anullsrc=r={_SAMPLE_RATE}:cl=mono"]
+        filters.append(
+            f"[{i}:a]aresample={_SAMPLE_RATE},aformat=sample_fmts=fltp:channel_layouts=mono,"
+            f"apad,atrim=0:{dur},asetpts=N/SR/TB[a{i}]"
+        )
+        labels.append(f"[a{i}]")
+    filters.append(f"{''.join(labels)}concat=n={len(pieces)}:v=0:a=1[out]")
+    cmd += [
+        "-filter_complex", ";".join(filters),
+        "-map", "[out]",
+        "-c:a", "libmp3lame", "-b:a", "128k",
+        out_path,
+    ]
+    return cmd
+
+
+def find_ffmpeg() -> Optional[str]:
+    return shutil.which("ffmpeg")
+
+
+def concat_drive_audio(pieces: list[dict], out_dir: str, name: str, ffmpeg: str, timeout: int = 300) -> str:
+    """拼出驱动音频，返回本地文件路径。失败抛 RuntimeError。"""
+    os.makedirs(out_dir, exist_ok=True)
+    safe = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in name)
+    out_path = os.path.join(out_dir, f"dh-drive-{safe}.mp3")
+    cmd = build_concat_command(pieces, out_path, ffmpeg)
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL)
+    except subprocess.TimeoutExpired as exc:
+        raise RuntimeError(f"ffmpeg concat timed out after {timeout}s") from exc
+    if result.returncode != 0 or not os.path.exists(out_path):
+        raise RuntimeError(f"ffmpeg concat failed: {(result.stderr or '').strip()[:500]}")
+    return out_path

package/skills/render-video/scripts/render_video.py

@@ -1198,8 +1198,19 @@ ASSET_RESOLVERS = {
 }
 
 
-def _resolve_single_asset(asset: dict, render_plan: dict, private_token: str, max_retries: int, timeout: int) -> bool:
-    """Resolve one asset. Returns True if generated, False if failed/skipped."""
+def _resolve_single_asset(
+    asset: dict,
+    render_plan: dict,
+    private_token: str,
+    max_retries: int,
+    timeout: int,
+    payload_override: Optional[dict] = None,
+) -> bool:
+    """Resolve one asset. Returns True if generated, False if failed/skipped.
+
+    ``payload_override`` 整份替换资产 payload，且**不回写 DSL**：数字人连续段的驱动
+    音频是按本次排轴现拼的，写回 DSL 的话下次改了旁白还会拿旧音频去驱动。
+    """
     source = asset.get("source", "")
     resolver = ASSET_RESOLVERS.get(source)
     if not resolver:
@@ -1218,11 +1229,14 @@ def _resolve_single_asset(asset: dict, render_plan: dict, private_token: str, ma
             break
 
     asset_with_payload = asset.copy()
-    if dsl_asset and "payload" in dsl_asset:
+    if payload_override is not None:
+        asset_with_payload["payload"] = dict(payload_override)
+    elif dsl_asset and "payload" in dsl_asset:
         asset_with_payload["payload"] = dsl_asset["payload"]
 
-    # For digital-human assets: inject audioUrl from a resolved TTS narration asset
-    if source == "gen-digital-human":
+    # Legacy：没走连续段调度（见 resolve_digital_human_segments）的数字人资产，
+    # 沿用「取第一个已生成的旁白」—— 只对全片一个数字人成立。
+    if source == "gen-digital-human" and payload_override is None:
         payload = asset_with_payload.get("payload", {})
         if not payload.get("audioUrl"):
             # Find the first generated TTS audio asset and use its URL
@@ -1319,12 +1333,21 @@ def apply_stub_urls(render_plan: dict, stub_image_url: str = "", stub_video_url:
     return stubbed
 
 
-def resolve_assets(render_plan: dict, private_token: str, max_retries: int, timeout: int) -> dict:
+def resolve_assets(
+    render_plan: dict,
+    private_token: str,
+    max_retries: int,
+    timeout: int,
+    defer_digital_human: bool = False,
+) -> dict:
     """Resolve all pending assets in the render plan.
 
     Two-phase resolution:
       Phase 1: resolve non-avatar assets (TTS, images, video) — **in parallel**
       Phase 2: resolve avatar/digital-human assets which may depend on generated audio URLs — serial
+
+    ``defer_digital_human=True`` 时跳过 Phase 2：场景引用的数字人要等排轴之后按连续段
+    生成（驱动音频要按场景最终时长补静音），由 resolve_digital_human_segments 接手。
     """
     from concurrent.futures import ThreadPoolExecutor, as_completed
 
@@ -1340,6 +1363,10 @@ def resolve_assets(render_plan: dict, private_token: str, max_retries: int, time
     phase1 = [a for a in pending if a.get("source") != "gen-digital-human"]
     # Phase 2: resolve digital-human assets (can now use generated audio URLs)
     phase2 = [a for a in pending if a.get("source") == "gen-digital-human"]
+    if defer_digital_human:
+        from _digital_human_segments import collect_segments
+        referenced = set(collect_segments(render_plan))
+        phase2 = [a for a in phase2 if a.get("assetId") not in referenced]
 
     generated = 0
     failed = 0
@@ -1390,6 +1417,133 @@ def resolve_assets(render_plan: dict, private_token: str, max_retries: int, time
     return render_plan
 
 
+def resolve_digital_human_segments(
+    render_plan: dict,
+    private_token: str,
+    max_retries: int,
+    timeout: int,
+    work_dir: Optional[str] = None,
+) -> dict:
+    """排轴之后按连续段生成数字人，并给段内场景注入起播帧 / 段角色。
+
+    必须在 adjust_timeline_to_audio 之后调：驱动音频 = 段内各场景旁白、每段补静音
+    到该场景**最终**时长，视频第 Σ前序场景帧 帧才正好是本场景第 0 帧。
+
+    - 单场景段且旁白 ≥ 5 秒：直接用旁白驱动，不拼接。
+    - 其余情况用 ffmpeg 拼一条（ab-agent 镜像自带 ffmpeg），上传后驱动。
+    - 没有 ffmpeg：多场景段拆成每场景一段（人物在场景边界重新起播，但口型不错位），
+      并在 logs 里说明。
+    - 已生成的资产（续跑 / 复用）不重做，只重新注入偏移——偏移只取决于排轴。
+    """
+    import _digital_human_segments as dhs
+
+    segments = dhs.collect_segments(render_plan)
+    if not segments:
+        return render_plan
+
+    assets = {a.get("assetId"): a for a in render_plan.get("resolvedAssets", [])}
+    dsl_payloads = {
+        a.get("assetId"): dict(a.get("payload") or {})
+        for a in render_plan.get("_dsl_assets", [])
+        if a.get("assetId")
+    }
+    ffmpeg = dhs.find_ffmpeg()
+    out_dir = work_dir or os.path.join(OUTPUT_DIR, "digital-human")
+    failed = 0
+
+    def log(message: str, level: str = "info") -> None:
+        render_plan.setdefault("logs", []).append({
+            "phase": "digital-human",
+            "level": level,
+            "message": message,
+            "timestamp": now_iso(),
+        })
+        LogPrint(f"   🧑‍💼 {message}", file=sys.stderr)
+
+    # 先决定每段怎么驱动；没有 ffmpeg 时把多场景段拆开，拆完再统一生成。
+    jobs: list[tuple[str, dict]] = []  # (assetId, payload)
+    for asset_id, indices in list(segments.items()):
+        asset = assets.get(asset_id)
+        if not asset or asset.get("status") != "pending":
+            continue
+        base_payload = dsl_payloads.get(asset_id, {})
+        if base_payload.get("audioUrl"):
+            # 作者显式给了驱动音频：尊重它，不再拼接。
+            jobs.append((asset_id, base_payload))
+            continue
+        if len(indices) > 1 and not ffmpeg:
+            new_ids = dhs.split_segment_per_scene(render_plan, asset_id, indices)
+            log(
+                f"ffmpeg not found; split continuous segment {asset_id} into {len(new_ids)} "
+                "per-scene videos (the presenter restarts at each scene boundary)",
+                level="warn",
+            )
+            timeline = render_plan.get("timeline", [])
+            for idx, new_id in zip(indices, new_ids):
+                if not dhs.plan_drive_pieces(render_plan, [idx])[0].get("url"):
+                    # 没有旁白的场景单独成段就没有东西可驱动：这一屏不出人，而不是整段失败。
+                    timeline[idx].get("props", {}).pop("avatarAssetId", None)
+                    render_plan["resolvedAssets"] = [
+                        a for a in render_plan["resolvedAssets"] if a.get("assetId") != new_id
+                    ]
+                    log(f"{new_id}: scene has no narration to drive a standalone presenter; skipped", level="warn")
+                    continue
+                dsl_payloads[new_id] = base_payload
+                jobs.append((new_id, base_payload))
+            continue
+        jobs.append((asset_id, base_payload))
+
+    segments = dhs.collect_segments(render_plan)
+    assets = {a.get("assetId"): a for a in render_plan.get("resolvedAssets", [])}
+
+    for asset_id, base_payload in jobs:
+        asset = assets.get(asset_id)
+        indices = segments.get(asset_id, [])
+        if not asset or not indices:
+            continue
+        payload = dict(base_payload)
+        if not payload.get("audioUrl"):
+            try:
+                url = dhs.direct_drive_url(render_plan, indices)
+                if not url:
+                    pieces = dhs.plan_drive_pieces(render_plan, indices)
+                    if not any(p.get("url") for p in pieces):
+                        raise RuntimeError("no generated narration audio in this segment to drive the presenter")
+                    if not ffmpeg:
+                        # 单场景但旁白不足 5 秒，又没法补静音：照样提交，由服务端判定。
+                        url = next(p["url"] for p in pieces if p.get("url"))
+                        log(f"{asset_id}: narration shorter than {dhs.MIN_DRIVE_SEC:.0f}s and ffmpeg not found; submitting unpadded", level="warn")
+                    else:
+                        local = dhs.concat_drive_audio(pieces, out_dir, asset_id, ffmpeg)
+                        from upload_video import upload_media_file
+                        url = upload_media_file(local, title=f"digital-human drive {asset_id}")["fileUrl"]
+                        total = sum(p["durationSec"] for p in pieces)
+                        log(f"{asset_id}: drive audio for {len(pieces)} scene(s), {total:.1f}s")
+                payload["audioUrl"] = url
+            except Exception as exc:  # noqa: BLE001 — 任一段失败不拖垮其它段
+                asset["status"] = "failed"
+                failed += 1
+                render_plan.setdefault("errors", []).append({
+                    "phase": "digital-human",
+                    "message": f"drive audio failed: {exc}"[:1000],
+                    "assetId": asset_id,
+                    "timestamp": now_iso(),
+                })
+                log(f"{asset_id}: drive audio failed: {exc}", level="error")
+                continue
+        # 口型音轨就是驱动音频，文本 / 音色字段留着会让服务端改走 TTS 驱动。
+        payload.pop("text", None)
+        payload.pop("voiceId", None)
+        if not _resolve_single_asset(asset, render_plan, private_token, max_retries, timeout, payload_override=payload):
+            failed += 1
+
+    for warning in dhs.inject_segment_props(render_plan, dhs.collect_segments(render_plan)):
+        log(warning, level="warn")
+    if failed:
+        render_plan["status"] = "failed"
+    return render_plan
+
+
 def _auto_highlight_map(
     entry: dict,
     narration_asset: Optional[dict],
@@ -2658,15 +2812,24 @@ Examples:
                 private_token=private_token,
                 max_retries=args.max_asset_retries,
                 timeout=args.asset_timeout,
+                defer_digital_human=True,
             )
         else:
             LogPrint(f"✅ all assets ready", file=sys.stderr)
             render_plan["status"] = "assets-ready"
 
-    render_plan.pop("_dsl_assets", None)
-
     if not args.skip_asset_resolve:
         render_plan = adjust_timeline_to_audio(render_plan)
+        # 数字人按连续段生成要用到排轴后的场景时长，所以排在 adjust 之后；
+        # 它还要读 DSL 里的资产 payload，_dsl_assets 因此晚一步再丢。
+        render_plan = resolve_digital_human_segments(
+            render_plan,
+            private_token=private_token,
+            max_retries=args.max_asset_retries,
+            timeout=args.asset_timeout,
+        )
+
+    render_plan.pop("_dsl_assets", None)
 
     output_path = os.path.join(OUTPUT_DIR, "video.mp4")
     rp_output = args.render_plan_output or os.path.join(OUTPUT_DIR, "render-plan.json")

package/skills/render-video/version.json

@@ -2,6 +2,6 @@
   "skillName": "render-video",
   "repoName": "agent-skill-media-maker",
   "skillId": "473",
-  "version": "V32",
+  "version": "V33",
   "skillDescription": "Final-render skill: loads a persisted RenderPlan by `job_id` and drives the Remotion engine to produce the final video.\n\nUse this skill as soon as the user mentions any of these intents (after assets are already prepared):\n- Render the video, composite the video, export the video\n- Turn the prepared assets into the final clip\n- Render with Remotion\n\nPrerequisite: assets must already be generated via `prepare_video_assets`. This skill never resolves or regenerates assets — pass it a `job_id` from a previous `prepare_video_assets` call.\n\n⚠️ Stop-and-confirm gate: never call this skill until the user has explicitly confirmed the assets prepared by `prepare_video_assets`. If those assets were prepared in the current turn and the user has not replied since, stop and ask instead of rendering."
 }

package/skills/template-registry/scripts/render_job_client.py

@@ -172,6 +172,23 @@ def save_manifest(job_id: int, manifest: dict, priv_token: str) -> None:
     _post("/renderJob/saveManifest", {"jobId": job_id, "manifest": manifest}, priv_token)
 
 
+def latest_job(conversation_id: str, priv_token: str) -> dict:
+    """
+    按会话取最近一次**已存 RenderPlan** 的渲染任务摘要（POST /renderJob/latest）。
+
+    多轮对话里模型看不到之前的工具结果，拿不到 jobId；消费上一轮成果的 skill
+    （如 export-jianying）用注入的 CONVERSATION_ID 反查最新一版，而不是让模型从
+    聊天记录里拼素材 URL —— 那样会拿到过期的旧版素材。
+
+    Returns:
+        {"jobId": int, "status": str, "attempt": int, "createTime": str, "updateTime": str}
+    """
+    data = _post("/renderJob/latest", {"conversationId": conversation_id}, priv_token)
+    if not data.get("jobId"):
+        raise RuntimeError("no render job with a saved RenderPlan in this conversation")
+    return data
+
+
 def get_job(job_id: int, priv_token: str) -> dict:
     """
     获取渲染任务完整详情。

package/skills/template-registry/version.json

@@ -2,6 +2,6 @@
   "skillName": "template-registry",
   "repoName": "agent-skill-media-maker",
   "skillId": "475",
-  "version": "V25",
+  "version": "V26",
   "skillDescription": "Video-template registry skill. Stores every video-template definition and lists the available templates (templateId / name / aspect ratio / style tags).\n\nUse this skill as soon as the user mentions any of these intents:\n- View available templates / list every template\n\nNote: DSL→TemplateBinding is no longer a separate exposed step — once prepare_video_assets receives a template_id it builds the binding internally."
 }