@remixmate/cli 0.13.9 → 0.13.10

package/skills/gen-script/scripts/gen_script.py

@@ -20,7 +20,9 @@ import json
 import re
 import math
 import os
+import subprocess
 import sys
+import urllib.request
 import uuid
 from copy import deepcopy
 from dataclasses import dataclass
@@ -85,6 +87,10 @@ except ImportError:
 
 
 try:
+    from video_dsl.runtime.presenter_speech import (
+        plan_speech_scenes as _plan_speech_scenes,
+        transcript_from_asr as _transcript_from_asr,
+    )
     from video_dsl.runtime.dsl_validator import (
         DEFAULT_MAX_SCENES,
         max_scenes_for_template,
@@ -107,6 +113,10 @@ try:
     from registry_loader import (
         load_registry_data as _load_registry_data,
         require_visible_template as _get_template,
+        declares_sources as _declares_sources,
+        load_contract as _load_contract,
+        template_for_source as _template_for_source,
+        api_base_url as _api_base_url,
         TemplateStatusGatedError,
     )
     _HAS_REGISTRY_LOADER = True
@@ -662,6 +672,88 @@ def _resolve_countdown(template_config: dict | None) -> dict:
     return resolve_countdown_spec(template_config)
 
 
+def _stamp_presenter(dsl: dict, template_id: str | None, source: str | None, style: str | None = None) -> None:
+
+    if not (template_id and _HAS_REGISTRY_LOADER):
+        return
+    template_config = _load_template_config(template_id)
+    if not _declares_sources(template_config):
+        return
+    contract = _load_contract(template_config, source)
+    dsl["global"]["presenter"] = {"source": contract.get("source")}
+    dsl["global"]["configLock"] = contract.get("releaseId")
+    if style:
+        styles = (contract.get("templateConfig") or {}).get("styles") or []
+        options = [s.get("id") for s in styles]
+        if style not in options:
+            raise ValueError(f"unknown --template-style {style!r}; options: {', '.join(map(str, options))}")
+        dsl["global"]["templateStyle"] = style
+
+
+def _recognize_speech(video_url: str) -> dict:
+
+    base = _api_base_url()
+    body = json.dumps({"audioUrl": video_url, "enableWords": True}).encode("utf-8")
+    headers = {"Content-Type": "application/json", "X-Priv-Token": os.environ.get("PRIV_TOKEN", "")}
+    req = urllib.request.Request(f"{base}/tool/asr", data=body, headers=headers, method="POST")
+    with urllib.request.urlopen(req, timeout=330) as resp:
+        result = json.loads(resp.read().decode("utf-8"))
+    if result.get("code") != 0:
+        raise ValueError(f"speech recognition failed: {result.get('msg')}")
+    return result.get("data") or {}
+
+
+def _video_seconds(video_url: str) -> float:
+
+    try:
+        out = subprocess.run(
+            ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", video_url],
+            capture_output=True, text=True, timeout=60,
+        )
+        return float(out.stdout.strip() or 0)
+    except (OSError, ValueError, subprocess.TimeoutExpired):
+        return 0.0
+
+
+def _apply_real_footage(dsl: dict, video_url: str | None, scene_count: int | None) -> None:
+
+
+
+
+
+    if not video_url:
+        raise ValueError("--source real-footage needs --presenter-video")
+    sentences = _transcript_from_asr(_recognize_speech(video_url))
+    if not sentences:
+        raise ValueError("no speech was recognized in the video")
+    fps = dsl["global"].get("fps", 30)
+    skeleton = dsl["scenes"]
+    planned = _plan_speech_scenes(sentences, _video_seconds(video_url), scene_count or len(skeleton), fps)
+    scenes = []
+    for i, part in enumerate(planned):
+        last = i == len(planned) - 1
+        base = deepcopy(skeleton[-1] if last else skeleton[min(i, len(skeleton) - 2)])
+        base.pop("audio", None)
+        base.pop("visuals", None)
+        base["id"] = f"scene-{i + 1:02d}"
+        base["duration"] = (part["endFrame"] - part["startFrame"]) / fps
+        base["customPayload"] = {}
+        base["speech"] = {"startFrame": part["startFrame"], "endFrame": part["endFrame"]}
+        scenes.append(base)
+    asset_id = "presenter-video"
+    dsl["scenes"] = scenes
+    dsl["assets"] = [{"assetId": asset_id, "type": "video", "source": "existing", "status": "generated", "url": video_url}]
+    dsl["global"]["subtitle"]["enabled"] = True
+    dsl["global"].pop("narration", None)
+    dsl["global"]["presenter"].update(
+        {
+            "segments": [{"assetRef": asset_id, "sourceIn": 0, "at": 0}],
+            "transcript": sentences,
+        }
+    )
+    dsl["meta"]["targetDuration"] = round(planned[-1]["endFrame"] / fps)
+
+
 def _resolve_contract(template_config: dict | None) -> dict:
 
 
@@ -2149,21 +2241,43 @@ def _apply_avatar(
 
 
 
-    if not avatar_id:
-        return
+    if avatar_id:
+        _bind_presenter_asset(dsl, template_config, template_id, {"avatarId": avatar_id}, "--avatar-id")
+
+
+def _apply_lip_sync(
+    dsl: dict,
+    template_config: dict | None,
+    template_id: str | None,
+    video_url: str | None,
+    mode: str,
+) -> None:
+
+    if not video_url:
+        raise ValueError("--source lip-sync needs --presenter-video")
+    _bind_presenter_asset(dsl, template_config, template_id, {"videoUrl": video_url, "mode": mode}, "--presenter-video")
+
+
+def _bind_presenter_asset(
+    dsl: dict,
+    template_config: dict | None,
+    template_id: str | None,
+    payload: dict,
+    flag: str,
+) -> None:
     if not template_id or template_config is None:
-        raise ValueError("--avatar-id needs --template-id: the template decides where the presenter shows.")
+        raise ValueError(f"{flag} needs --template-id: the template decides where the presenter shows.")
     support = ((template_config.get("capabilities") or {}).get("digitalHuman") or {}).get("support")
     if support not in ("optional", "required"):
-        raise ValueError(f"--avatar-id: template {template_id!r} does not declare capabilities.digitalHuman")
+        raise ValueError(f"{flag}: template {template_id!r} does not declare capabilities.digitalHuman")
     scenes = [
         s for s in dsl.get("scenes", [])
         if _slot_extracts(template_config, str(s.get("purpose")), "visuals.avatar.assetRef")
     ]
     if not scenes:
-        raise ValueError(f"--avatar-id: no scene of template {template_id!r} reads visuals.avatar.assetRef")
+        raise ValueError(f"{flag}: no scene of template {template_id!r} reads visuals.avatar.assetRef")
     if not any((s.get("audio") or {}).get("narration") for s in scenes):
-        raise ValueError("--avatar-id: the presenter is driven by the narration, and this script has none")
+        raise ValueError(f"{flag}: the presenter is driven by the narration, and this script has none")
     for scene in scenes:
         visuals = scene.get("visuals") if isinstance(scene.get("visuals"), dict) else {}
         scene["visuals"] = {**visuals, "avatar": {"assetRef": AVATAR_ASSET_ID}}
@@ -2172,7 +2286,7 @@ def _apply_avatar(
         "type": "avatar",
         "source": "gen-digital-human",
         "status": "planned",
-        "payload": {"avatarId": avatar_id},
+        "payload": payload,
     })
     _lock_scene_path(dsl, "visuals.avatar.assetRef")
 
@@ -2644,6 +2758,8 @@ def build_dsl(
         payload_writes: PayloadWrites | None = None,
         countdown_ending: bool = False,
         narration: str = "auto",
+        presenter_source: str | None = None,
+        presenter_video: str | None = None,
 ) -> dict:
 
 
@@ -2674,6 +2790,8 @@ def build_dsl(
 
 
     template_config = _load_template_config(template_id) if template_id else None
+    if _HAS_REGISTRY_LOADER:
+        template_config = _template_for_source(template_config, presenter_source)
     contract = _resolve_contract(template_config)
 
 
@@ -2766,6 +2884,7 @@ def build_dsl(
 
 
     provided_slots = [s.strip() for s in (scene_images or []) if s and s.strip()]
+    _reject_video_scene_images(provided_slots, primary_visual_type, template_config)
     scene_plans = _plan_skeleton_scenes(
         contract, countdown, beat_count, topic, duration, scene_count, output_language,
         effective_headline, countdown_ending, provided_slots, has_visual,
@@ -2894,6 +3013,218 @@ def build_dsl(
     return dsl
 
 
+
+BROLL_MODES = ("auto", "ai-image", "ai-video", "none")
+
+BROLL_AI_VIDEO_MAX = 3
+
+BROLL_AI_VIDEO_SECONDS = 4
+BROLL_VIDEO_MODEL = "seedance-mini"
+_BROLL_LAST_SCENES = ("opening", "cta", "ending")
+_VIDEO_SUFFIXES = (".mp4", ".mov", ".webm", ".avi", ".m4v")
+_SKELETON_PROMPT_HEAD = re.compile(r"\s*\[(?:骨架待填充|skeleton placeholder)\][^)）]*[)）]")
+
+
+def _style_broll_mode(template_config: dict | None, source: str | None, style_id: str | None) -> str:
+
+    if not _HAS_REGISTRY_LOADER or not _declares_sources(template_config):
+        return "ai-image"
+    config = _load_contract(template_config, source).get("templateConfig") or {}
+    wanted = style_id or config.get("defaultStyle")
+    style = next((s for s in config.get("styles") or [] if s.get("id") == wanted), {})
+    return style.get("brollDefault") or "ai-image"
+
+
+def _reject_video_scene_images(slots: list[str], visual_type: str, template_config: dict | None) -> None:
+
+    if visual_type != "image":
+        return
+    videos = [s for s in slots if urlsplit(s).path.lower().endswith(_VIDEO_SUFFIXES)]
+    if not videos:
+        return
+    binding = ((template_config or {}).get("capabilities") or {}).get("mediaBinding") or {}
+    hint = "pass them as --broll-asset" if binding.get("assetRefKey") else "this template takes images here"
+    raise ValueError(f"--scene-images got video URLs ({', '.join(videos)}): {hint}")
+
+
+def _broll_targets(scenes: list[dict]) -> list[dict]:
+
+    return sorted(scenes, key=lambda scene: scene.get("purpose") in _BROLL_LAST_SCENES)
+
+
+def _scene_seconds(scene: dict) -> float:
+    return float(scene.get("duration") or 0)
+
+
+def _fit_broll_item(item: dict, scene: dict) -> None:
+
+    length = _scene_seconds(scene)
+    span = min(float(item.get("durationSec") or 0), length)
+    item["durationSec"] = span
+    if float(item.get("atSec") or 0) + span > length:
+        item["atSec"] = round((length - span) / 2, 3)
+
+
+def _scene_speech_text(dsl: dict, scene: dict) -> str:
+
+    speech = scene.get("speech")
+    transcript = (dsl["global"].get("presenter") or {}).get("transcript")
+    if not speech or not transcript:
+        return ""
+    fps = dsl["global"].get("fps", 30)
+    start, end = speech["startFrame"] * 1000 / fps, speech["endFrame"] * 1000 / fps
+    return "".join(s["text"] for s in transcript if start <= (s["startMs"] + s["endMs"]) / 2 < end)
+
+
+def _ai_image_broll(asset_id: str, scene: dict, topic: str, style: str, text: str, guide: dict, stub: str | None) -> dict:
+    if stub:
+        return {"assetId": asset_id, "type": "image", "source": "existing", "status": "generated", "url": stub}
+    result = generate_image_prompt(scene.get("purpose", "point"), topic, style, narration_text=text, style_guide=guide)
+    payload = {
+        "prompt": result["prompt"],
+        "model": os.environ.get("DEFAULT_IMAGE_MODEL", "doubao/doubao-seedream-5-0-260128"),
+        "ratio": guide.get("ratio") or "1:1",
+    }
+    for key in ("negativePrompt", "guidanceScale"):
+        if key in result:
+            payload[key] = result[key]
+    return {"assetId": asset_id, "type": "image", "source": "gen-image", "status": "planned", "payload": payload}
+
+
+def _video_subject(image_prompt: str, scene_text: str, topic: str) -> str:
+
+
+
+
+    placeholder = _SKELETON_PROMPT_HEAD.match(image_prompt)
+    return placeholder.group(0) if placeholder else (scene_text or topic)
+
+
+def _ai_video_broll(asset_id: str, subject: str, ratio: str, stub: str | None) -> dict:
+    if stub:
+        return {"assetId": asset_id, "type": "video", "source": "existing", "status": "generated", "url": stub}
+    return {
+        "assetId": asset_id,
+        "type": "video",
+        "source": "gen-video",
+        "status": "planned",
+        "payload": {
+            "prompt": f"{subject}，空镜，画面里没有人说话，没有文字，镜头缓慢运动",
+            "model": os.environ.get("DEFAULT_VIDEO_MODEL", BROLL_VIDEO_MODEL),
+            "ratio": ratio,
+            "duration": BROLL_AI_VIDEO_SECONDS,
+        },
+    }
+
+
+def _library_broll(asset_id_index: int, url: str) -> tuple[dict, dict]:
+
+    if not urlsplit(url).path.lower().endswith(_VIDEO_SUFFIXES):
+        asset_id = f"img-broll-{asset_id_index:02d}"
+        return {"assetId": asset_id, "type": "image", "source": "existing", "status": "generated", "url": url}, {}
+    asset_id = f"video-broll-{asset_id_index:02d}"
+    asset = {"assetId": asset_id, "type": "video", "source": "existing", "status": "generated", "url": url}
+    seconds = _probe_sequence_video_duration(url)
+    return asset, {"type": "video", **({"sourceDurationSec": round(float(seconds), 3)} if seconds else {})}
+
+
+def _apply_brolls(
+        dsl: dict,
+        template_config: dict | None,
+        template_id: str | None,
+        urls: list[str] | None,
+        mode: str | None,
+        *,
+        variant: str | None = None,
+        stub_image_url: str | None = None,
+        stub_video_url: str | None = None,
+) -> None:
+
+
+
+
+
+    urls = [u.strip() for u in (urls or []) if u and u.strip()]
+    if not urls and not mode:
+        return
+    binding = ((template_config or {}).get("capabilities") or {}).get("mediaBinding") or {}
+    ref_key, items_path = binding.get("assetRefKey"), binding.get("itemsPath")
+    if not ref_key or not items_path:
+        raise ValueError(f"--broll-asset / --broll-mode: template {template_id!r} has no per-scene media")
+    parts = items_path.split(".")
+    guide = resolve_style_guide(template_config, variant or "") if _HAS_ENHANCER else {}
+    meta = dsl["meta"]
+    assets = dsl["assets"]
+    ai_videos = 0
+
+    def items_of(scene: dict) -> list[dict]:
+        items = _get_path(scene.get("customPayload") or {}, parts)
+        return items if isinstance(items, list) else []
+
+    def set_items(scene: dict, items: list[dict]) -> None:
+        _write_payload(scene.setdefault("customPayload", {}), [(parts, items)])
+
+    def drop_asset(asset_id: str | None) -> None:
+        assets[:] = [a for a in assets if a.get("assetId") != asset_id]
+
+    def new_item(scene: dict) -> dict:
+        item = deepcopy(binding.get("item") or {})
+        set_items(scene, [*items_of(scene), item])
+        return item
+
+    def place_library(scene: dict, index: int, url: str, item: dict) -> None:
+        drop_asset(item.get(ref_key))
+        asset, fields = _library_broll(index, url)
+        assets.append(asset)
+        item.update({ref_key: asset["assetId"], **fields})
+        seconds = fields.get("sourceDurationSec")
+        if seconds:
+            item["durationSec"] = min(float(item.get("durationSec") or seconds), seconds)
+        _fit_broll_item(item, scene)
+
+    targets = _broll_targets(dsl["scenes"])
+    for index, scene in enumerate(targets):
+        items = items_of(scene)
+        item = items[0] if items else None
+        if index < len(urls):
+            place_library(scene, index + 1, urls[index], item or new_item(scene))
+            continue
+        if mode == "none":
+            if item:
+                drop_asset(item.get(ref_key))
+                set_items(scene, items[1:])
+            continue
+        if mode not in ("ai-image", "ai-video"):
+            continue
+        if item is None:
+            asset_id = f"img-{scene['id']}"
+            assets.append(_ai_image_broll(
+                asset_id, scene, meta.get("topic", ""), meta.get("style", ""),
+                _scene_speech_text(dsl, scene), guide, stub_image_url,
+            ))
+            item = new_item(scene)
+            item[ref_key] = asset_id
+            _fit_broll_item(item, scene)
+        if mode == "ai-video" and ai_videos < BROLL_AI_VIDEO_MAX:
+            image = next(a for a in assets if a.get("assetId") == item.get(ref_key))
+            subject = _video_subject(
+                (image.get("payload") or {}).get("prompt") or "", _scene_speech_text(dsl, scene), meta.get("topic", ""),
+            )
+            video = _ai_video_broll(f"video-{scene['id']}", subject, dsl["global"].get("aspectRatio", "9:16"), stub_video_url)
+            drop_asset(image["assetId"])
+            assets.append(video)
+            item.update({ref_key: video["assetId"], "type": "video"})
+            item["durationSec"] = min(float(item.get("durationSec") or BROLL_AI_VIDEO_SECONDS), BROLL_AI_VIDEO_SECONDS)
+            _fit_broll_item(item, scene)
+            ai_videos += 1
+
+
+    for offset, url in enumerate(urls[len(targets):]):
+        index = len(targets) + offset + 1
+        scene = targets[offset % len(targets)]
+        place_library(scene, index, url, new_item(scene))
+
+
 def _build_arg_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description="Video script generator — produces Video DSL v1alpha1 JSON",
@@ -3008,6 +3339,35 @@ Examples:
     parser.add_argument("--avatar-id", default=None, help="Digital-human avatar id (dh_xxx) for templates declaring capabilities.digitalHuman: adds one gen-digital-human asset referenced by every scene's visuals.avatar.assetRef (one continuous presenter driven by the narration). Placement stays the template default unless set with --payload digitalHuman.<field>=...")
     parser.add_argument("--ending-payload", action="append", default=None, metavar="KEY=VALUE", help="Like --payload, but written only to the ending scene (e.g. <object>.<field>=<closing line>). On countdown templates it also appends the ending scene after #1. Repeatable.")
     parser.add_argument("--allow-ai-video", action="store_true", help="Allow AI-generated video assets")
+    parser.add_argument(
+        "--source", default=None, dest="presenter_source",
+        help="Presenter source, for templates that declare capabilities.sources "
+        "(digital-human, lip-sync, real-footage). Defaults to the template's defaultSource.",
+    )
+    parser.add_argument(
+        "--presenter-video", default=None,
+        help="URL of the user's talking-head video. Required with --source real-footage; "
+        "used with --source lip-sync as the video to re-voice.",
+    )
+    parser.add_argument(
+        "--lipsync-mode", default="speed", choices=("speed", "precision"),
+        help="Lip-sync quality for --source lip-sync. precision costs about 4x as much.",
+    )
+    parser.add_argument(
+        "--broll-asset", action="append", default=None,
+        help="Repeatable. URL of an image or video the user picked from their asset library as a b-roll shot. "
+        "For templates declaring capabilities.mediaBinding with assetRefKey, on every presenter source. "
+        "All of them go into the DSL; anchor each to the sentence it matches (atText) from its description.",
+    )
+    parser.add_argument(
+        "--broll-mode", default=None, choices=BROLL_MODES,
+        help="Scenes without a --broll-asset: auto (the visual style's default), ai-image (gen-image), ai-video (gen-video on at most "
+        f"{BROLL_AI_VIDEO_MAX} scenes, the rest stay images) or none. Omitted keeps the skeleton as it is.",
+    )
+    parser.add_argument(
+        "--template-style", default=None,
+        help="Visual style for templates that ship several (see templateConfig.styles in the template).",
+    )
     parser.add_argument("--template-id", default=None, help="Template id. The template owns outputLanguage and may declare a defaultVoiceId.")
     parser.add_argument(
         "--template-source", default=None, choices=TEMPLATE_SOURCES,
@@ -3313,7 +3673,12 @@ def main():
             payload_writes=payload_writes,
             countdown_ending=bool(args.ending_payload),
             narration=args.narration,
+            presenter_source=args.presenter_source,
+            presenter_video=args.presenter_video,
         )
+        _stamp_presenter(dsl, args.template_id, args.presenter_source, args.template_style)
+        if args.presenter_source == "real-footage":
+            _apply_real_footage(dsl, args.presenter_video, args.scenes)
     except ValueError as exc:
 
 
@@ -3333,6 +3698,16 @@ def main():
         _apply_scene_payload(dsl, look_config, args.template_id, args.payload, args.motion)
         _apply_ending_payload(dsl, look_config, args.template_id, args.ending_payload)
         _apply_avatar(dsl, look_config, args.template_id, args.avatar_id)
+        if args.presenter_source == "lip-sync":
+            _apply_lip_sync(dsl, look_config, args.template_id, args.presenter_video, args.lipsync_mode)
+        if args.broll_asset or args.broll_mode:
+            broll_mode = args.broll_mode
+            if broll_mode == "auto":
+                broll_mode = _style_broll_mode(look_config, args.presenter_source, args.template_style)
+            _apply_brolls(
+                dsl, look_config, args.template_id, args.broll_asset, broll_mode,
+                variant=args.variant, stub_image_url=stub_image_url, stub_video_url=stub_video_url,
+            )
     except ValueError as exc:
         parser.error(str(exc))
 

package/skills/gen-script/skill.json

@@ -64,6 +64,35 @@
         "type": "string",
         "description": "Media fill mode for templates declaring capabilities.mediaBinding.options.fit; one of that option's enum in the template's customPayloadSchema (its descriptions explain each value). Beats the preset's own fit. **Must** pass when the user picked it: `显示方式: <名称>（<id>）` → pass `<id>`; `auto` keeps the template / preset default."
       },
+      "source": {
+        "type": "string",
+        "description": "Presenter source, for templates that declare capabilities.sources: digital-human, lip-sync or real-footage. Omit to use the template's default."
+      },
+      "template_style": {
+        "type": "string",
+        "description": "Visual style for templates that ship several, e.g. talking-head-studio: douyin-bold, news-anchor, live-ecommerce."
+      },
+      "presenter_video": {
+        "type": "string",
+        "description": "URL of the user's talking-head video. Required with source real-footage or lip-sync."
+      },
+      "lipsync_mode": {
+        "type": "string",
+        "enum": ["speed", "precision"],
+        "description": "Lip-sync quality for source lip-sync, default speed. precision costs about 4x as much."
+      },
+      "broll_asset": {
+        "type": "array",
+        "items": {
+          "type": "string"
+        },
+        "description": "URLs of images or videos the user picked as b-roll from their asset library (the replicate card's 空镜素材 asset lines), on any source. Not scene_images: scene_images treats every entry as an image."
+      },
+      "broll_mode": {
+        "type": "string",
+        "enum": ["auto", "ai-image", "ai-video", "none"],
+        "description": "Scenes without a broll_asset (the replicate card's 其余画面): auto (the visual style's default), ai-image, ai-video (at most 3 scenes, the rest stay images) or none."
+      },
       "narration": {
         "type": "string",
         "enum": ["auto", "on", "off"],
@@ -205,6 +234,10 @@
       "background_preset",
       "sequence_preset",
       "sequence_fit",
+      "source",
+      "template_style",
+      "presenter_video",
+      "lipsync_mode",
       "narration",
       "background_asset",
       "payload",
@@ -222,6 +255,8 @@
       "subheadline",
       "carousel_items",
       "scene_images",
+      "broll_asset",
+      "broll_mode",
       "caption_lines",
       "stub_image_url",
       "stub_video_url",