@lalalic/markcut 2.3.0 → 2.4.1

package/AGENTS.md

@@ -39,7 +39,7 @@ See [docs/markdown-strict-descriptive.md](docs/markdown-strict-descriptive.md) f
 | `folder` | Internal container (series/parallel) |
 | `image` | Still photo with fit mode |
 | `video` | Video clip with startFrom/endAt trimming, playbackRate |
-| `audio` | Soundtrack/SFX with foreground ducking, loop |
+| `audio` | Soundtrack/SFX with foreground ducking |
 | `component` | External React component by componentName + props |
 | `effect` | CSS keyframe animation wrapper (fadeIn, zoomIn, bounceIn, etc.) |
 | `include` | Embed external video JSON (file, URL, or data URI) |
@@ -47,7 +47,7 @@ See [docs/markdown-strict-descriptive.md](docs/markdown-strict-descriptive.md) f
 ## CLI
 
 ```bash
-markcut render <file.json|.md> [--aspect 16x9|9x16|1x1|all] [--output path]
+markcut render <file.json|.md> [--output path]
 markcut preview <file.json|.md> [--edit] [--label] [--port 3001]
 ```
 
@@ -134,8 +134,8 @@ All types share base fields from `BaseShape`: `id`, `name`, `title`, `descriptio
 | `folder` | Group children | `isSeries`, `transition`, `children[]` |
 | `scene` | Storyboard node (alias for folder) | `name`, `description`, `script`, `children[]` |
 | `image` | Still photo | `src`, `fit` (contain/cover/fill) |
-| `video` | Video clip | `src`, `volume`, `playbackRate`, `loop` |
-| `audio` | Soundtrack/SFX | `src`, `volume`, `foreground` (ducks parent), `loop` |
+| `video` | Video clip | `src`, `volume`, `playbackRate` |
+| `audio` | Soundtrack/SFX | `src`, `volume`, `foreground` (ducks parent) |
 | `subtitle` | Text overlay | `src` (text/VTT) or `cues[]`, `fontSize`, `style` |
 | `component` | React component | `componentName`, `props`, `src` (remote ESM) |
 | `effect` | CSS animation wrapper | `animation`, `customKeyframes`, `children[]` |
@@ -176,7 +176,7 @@ All types share base fields from `BaseShape`: `id`, `name`, `title`, `descriptio
 ### CLI Usage
 
 ```bash
-node src/render/cli.mjs render <file.json> [--aspect 16x9|9x16|1x1|all] [--output path]
+node src/render/cli.mjs render <file.json> [--output path]
 node src/render/cli.mjs render --template <id> --data <data.json>
 node src/render/cli.mjs preview <file.json> [--edit] [--label] [--port 3001]
 node src/render/cli.mjs templates

package/README.md

@@ -4,7 +4,7 @@ Write a storyboard in markdown, get a rendered video with TTS narration and subt
 
 ```bash
 # Render a storyboard to MP4
-npx markcut render storyboard.md --aspect 9x16
+npx markcut render storyboard.md
 
 # Preview with live edit (edit .md file, player auto-reloads)
 npx markcut preview storyboard.md --edit

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lalalic/markcut",
-  "version": "2.3.0",
+  "version": "2.4.1",
   "description": "Markdown-to-video engine. Describe scenes in markdown, get a rendered video.",
   "bin": {
     "markcut": "bin/markcut"

package/skills/markcut/SKILL.md

@@ -83,4 +83,8 @@ some common issues (photo or video can't be displayed, audio missing), take belo
   - audio cut off
   - video cut off
   - subtitle mismatch
-  - sync issues between audio, video, and subtitles
+  - sync issues between audio, video, and subtitles
+- don't set duration for script or stream's duration depending on audio script
+  - markcut resolver will automatically calculate the duration based on the audio script length
+- **don't** rm `.markcut` directory, which served as cache for all generated content. cache will auto update according to the content change. rm `.markcut` will cause all content to be regenerated, which is time consuming and wasteful.
+- put all manual assets in `assets` folder, such as bgm, logo, watermark, etc. don't put them in `.markcut` folder, which is auto generated and will be deleted when `markcut clean` command is run.

package/skills/markcut/docs/markdown-descriptive.md

@@ -6,7 +6,7 @@ Complete reference for LLM-driven video generation. The parser uses **remark** (
 
 A markdown document compiled into a renderable scene tree.
 
-- Top heading `# <name>` (heading text ignored, just marks document root; `# video`, `# sub-video`, `# anything` all work)
+- Top heading `# video` marks the document root. (Only `# video` is treated as the main section — `# anything_else` creates a **variant** section for alternate configs, see [Variants](#variants-language--platform--any-override).)
 - Optional YAML frontmatter block `---\n...\n---\n` at the very top
 - Root config line: `width:<n> height:<n> fps:<n> layout:<mode>` (key:value pairs on the line after `# video`)
 - Scenes via `##`/`###`/`####` headings
@@ -165,7 +165,6 @@ For compatibility, `import { Name } from "spec"` also works and produces the sam
 | `foreground` | bool; ducks parent video audio while playing | audio |
 | `spots` | number[] beat timestamps | rhythm |
 | `fit` | `contain\|cover\|fill` | image |
-| `loop` | int >1 | audio |
 | `playbackRate` | number | video |
 | `jsx` | usage JSX expression (`"<ComA value={42} />"`); compiled at runtime with registered imports in scope | component |
 | `effects` | `[name, name(params...)]` e.g. `[fadeIn, bounceIn(1, ease-out, 2)]` | any leaf/container | Apply animations directly — no wrapper node needed |
@@ -226,7 +225,7 @@ When: moving footage. Required: `src` + (`duration` or `endAt`).
 
 When: voiceover, BGM, SFX, TTS narration. Required: `src` + (`duration` or `endAt`).
 
-`- audio src:bgm.mp3 duration:6 volume:0.4 loop:2`
+`- audio src:bgm.mp3 duration:6 volume:0.4 isBackground`
 
 ### Narration / TTS
 
@@ -760,16 +759,16 @@ markcut verify courseware.md
 # best practices
 ~~~ example - avoid audio cut
 - parallel
-  - image|video background:true
+  - image|video isBackground:true
   - audio|script 
 ~~~
 
 ~~~ example - global streams
 #video
-- audio background:true loop:true src:bgm.mp3
-- component background:true loop:true jsx:"<Logo />" style:"position:fixed;top:10px;left:10px;width:100px;height:100px;"
+- audio isBackground:true src:bgm.mp3
+- component isBackground:true jsx:"<Logo />" style:"position:fixed;top:10px;left:10px;width:100px;height:100px;"
 - parallel title:"lip sync" style:"position:fixed;bottom:100px;right:100px;width:100px;height:100px;"
-  - video src:background.mp4 loop:true background:true
+  - video src:background.mp4 isBackground:true
   - video src:lip_sync.mp4
 ~~~
 
@@ -803,3 +802,13 @@ And now it's <span class="emoji-dead">dead</span>.
 ```
 ~~~
 
+~~~ example - sync vision and audio, taking care of transition timing, dynamic audio duration
+#video
+layout:series transition:fade transitionTime:1.2
+
+## hello layout:parallel
+  - image src:vision.jpg isBackground:true  # set vision as background let script/audio play in front determine the duration of the scene
+  - script "..."                            # don't set duration for script/audio, markcut resolver will determine the duration of the scene based on the final audio length
+    start:1.2                               # let vision play for 1.2s before the script/audio start, so that the transition effect can be completed
+
+~~~

package/src/config.mjs

@@ -20,7 +20,7 @@ export const args=(function parseArgs(argv) {
     "--tti": "tti",
     "--ttv": "ttv"
   };
-  const args = { command: "", file: "", output: "", forceNew: false, verbose: false, label: false, edit: false, noBrowser: false, chat: false, port: 3001, compile: false, cli: false, showClis: false, scriptOutputDir: "", mediaOutputDir: "", variant: [], cliOverrides: {} };
+  const args = { command: "", file: "", output: "", forceNew: false, verbose: false, dev: false, label: false, edit: false, noBrowser: false, chat: false, port: 3001, compile: false, cli: false, showClis: false, scriptOutputDir: "", mediaOutputDir: "", variant: [], cliOverrides: {} };
   let i = 2;
   if (argv[i]) args.command = argv[i++];
   if (argv[i] && !argv[i].startsWith("--")) args.file = argv[i++];
@@ -34,6 +34,7 @@ export const args=(function parseArgs(argv) {
     else if (flag === "--compile") args.compile = true;
     else if (flag === "--force-new") args.forceNew = true;
     else if (flag === "--verbose") args.verbose = true;
+    else if (flag === "--dev") args.dev = true;
     else if (flag === "--label") args.label = true;
     else if (flag === "--edit") args.edit = true;
     else if (flag === "--no-browser") args.noBrowser = true;

package/src/descriptive/compiler.ts

@@ -527,7 +527,6 @@ function compileLeaf(node: Exclude<DescriptiveNode, DescriptiveContainer | Descr
         src: node.src,
         volume: node.volume ?? 1,
         foreground: node.foreground,
-        loop: node.loop,
         speaker: node.speaker,
       };
       return { stream, duration: end };
@@ -1182,8 +1181,13 @@ export function resolveVariantOverrides(
 export function compileDescriptiveRoot(input: DescriptiveRoot, options: CompileOptions = {}): Root {
   const root: DescriptiveRoot = typeof input === "string" ? JSON.parse(input) : input;
 
-  const resolved = resolveTransition(root.layout ? root.transition ?? root.layout : root.transition, root.transitionTime);
-  const rootKind: "series" | "parallel" | "transitionSeries" = root.layout === "parallel" ? "parallel" : "series";
+  const resolved = resolveTransition(
+    root.layout === "transitionSeries" ? (root.transition ?? "fade") : root.transition,
+    root.transitionTime,
+  );
+  const rootKind: "series" | "parallel" | "transitionSeries" =
+    root.layout === "parallel" ? "parallel" :
+    root.layout === "transitionSeries" ? "transitionSeries" : "series";
 
   const googleMapsApiKey = options.googleMapsApiKey ?? "";
   const ctx: CompileContext = {
@@ -1215,8 +1219,8 @@ export function compileDescriptiveRoot(input: DescriptiveRoot, options: CompileO
     stylesheet: input.stylesheet,
     subtitle: input.subtitle,
     isSeries: rootKind !== "parallel",
-    transition: rootKind === "transitionSeries" ? resolved.name : undefined,
-    transitionTime: rootKind === "transitionSeries" ? resolved.time : 0.5,
+    transition: root.transition || root.layout === "transitionSeries" ? resolved.name : undefined,
+    transitionTime: root.transition || root.layout === "transitionSeries" ? resolved.time : 0.5,
     children: children.map((c) => c.stream),
     durationInSeconds: duration,
   };

package/src/descriptive/markdown.ts

@@ -221,7 +221,6 @@ function parseNodeLine(content: string, lineNum?: number): DescriptiveNode {
         endAt: attrs.endAt as any,
         volume: attrs.volume as any,
         foreground: attrs.foreground as any,
-        loop: attrs.loop as any,
         instruction: attrs.instruction as any,
         script: attrs.script as any,
         visible: attrs.visible as any,
@@ -633,6 +632,9 @@ function applyRootAttrs(root: DescriptiveRoot, attrs: Record<string, unknown>):
       case "metadata":
         root.metadata = String(v);
         break;
+      case "seed":
+        root.seed = Number(v);
+        break;
       case "stylesheet":
         root.stylesheet = String(v);
         break;

package/src/descriptive/resolve.ts

@@ -412,11 +412,17 @@ export async function resolveScripts(
     allScriptNodes.push({ node, id });
   });
 
-  if (allScriptNodes.length > 0) {
-    console.log(`  🔊 TTS: generating ${allScriptNodes.length} script${allScriptNodes.length > 1 ? "s" : ""}...`);
+  const totalScripts = allScriptNodes.length;
+  let scriptsDone = 0;
+  let cacheHits = 0;
+  const ttsStart = Date.now();
+
+  if (totalScripts > 0) {
+    console.log(`  🔊 TTS: generating ${totalScripts} script${totalScripts > 1 ? "s" : ""}...`);
   }
 
   for (const { node, id } of allScriptNodes) {
+    scriptsDone++;
     // TTS CLI from root config only
     let ttsCli = clone.tts ?? options.ttsCli ?? DEFAULT_TTS_CLI;
 
@@ -440,13 +446,15 @@ export async function resolveScripts(
     const label = `${speakerLabel}${firstWords(node.script, 8)}`;
     if (cached) {
       generated = cached;
+      cacheHits++;
       console.log(`  ✓ TTS: ${label} (cached)`);
     } else {
+      console.log(`  🔊 TTS (${scriptsDone}/${totalScripts}): ${label}...`);
       generated = generateTTS(node.script, audioPath, ttsCli);
       if (generated) {
         updateCache(cache, `tts:${cacheKey}`, cacheKey, generated);
         cacheDirty = true;
-        console.log(`  ✓ TTS: ${label}`);
+        console.log(`  ✓ TTS (${scriptsDone}/${totalScripts}): ${label}`);
       } else {
         console.warn(`  ⚠ TTS produced no audio for "${label}". Audio will have no source. Check root.tts config.`);
       }
@@ -459,10 +467,11 @@ export async function resolveScripts(
   }
 
   if (cacheDirty) writeCacheManifest(options.outputDir, cache);
-  if (allScriptNodes.length > 0) {
+  if (totalScripts > 0) {
     try { writeFileSync(join(options.outputDir, ".cache.json"), JSON.stringify(cache, null, 2), "utf-8"); } catch {}
+    const ttsElapsed = Math.round((Date.now() - ttsStart) / 1000);
     const unique = new Set(allScriptNodes.filter(s => s.node.src).map(s => s.node.src)).size;
-    console.log(`  ✅ TTS: ${unique} unique audio file${unique > 1 ? "s" : ""} (${allScriptNodes.length} node${allScriptNodes.length > 1 ? "s" : ""})`);
+    console.log(`  ✅ TTS: ${unique} unique audio${unique > 1 ? "s" : ""} (${scriptsDone} nodes) in ${ttsElapsed}s (${cacheHits} cached)`);
   }
   return clone;
 }
@@ -713,27 +722,38 @@ export async function resolveGeneratedMedia(
   walkDown(clone as any, (node) => {
     if ((node.type !== "image" && node.type !== "video")) return;
     if (!node.prompt || typeof node.prompt !== "string") return;
-    // Skip if src is already set to a real path (prompt is just metadata).
-    // "auto" is a placeholder meaning "auto-generate via TTI/TTV".
     if (node.src && node.src !== "auto") return;
     const id = node.id ?? `${node.type}-${genNodes.length}`;
     genNodes.push({ node, id, type: node.type as "image" | "video", prompt: node.prompt });
   });
 
+  const total = genNodes.length;
+  let done = 0;
+  let cacheHits = 0;
+  const startTime = Date.now();
+
+  function progressLine() {
+    const elapsed = (Date.now() - startTime) / 1000;
+    const rate = done / Math.max(elapsed, 0.1);
+    const remaining = total - done;
+    const eta = rate > 0 ? Math.round(remaining / rate) : "?";
+    const etaStr = eta === "?" ? "" : `, ~${eta}s remaining`;
+    return `  ${done}/${total} generated (${cacheHits} cached)${etaStr}`;
+  }
+
+  if (total > 0) {
+    console.log(`  🎨 Generating ${total} media item${total > 1 ? "s" : ""}...`);
+  }
+
   for (const { node, id, type, prompt } of genNodes) {
+    done++;
     const ext = type === "image" ? "png" : "mp4";
 
-    // Resolve TTI/TTV config: root-level config overrides CLI defaults
     const cli = type === "image"
       ? (clone.tti ?? options.ttiCli ?? DEFAULT_TTI_CLI)
       : (clone.ttv ?? options.ttvCli ?? DEFAULT_TTV_CLI);
 
-    // Use root.seed for reproducible generation (applied when CLI template has {seed}).
-    // seed is part of the cache key so different seeds produce different cached outputs.
     const seed = options.seed;
-
-    // Content-addressed filename: hash of prompt + CLI + type + seed, so identical
-    // prompts with different seeds produce distinct cached media.
     const cacheKey = computeCacheKey({ prompt, cli, type, seed });
     const outputPath = join(options.outputDir, `${cacheKey}.${ext}`);
 
@@ -743,12 +763,13 @@ export async function resolveGeneratedMedia(
 
     if (cached) {
       node.src = resolvePath(cached);
-      console.log(`  ✓ ${label}: ${labelText} (cached)`);
+      cacheHits++;
+      if (done % 5 === 0 || done === total) console.log(progressLine());
       continue;
     }
 
     try {
-      console.log(`  🔊 ${label}: ${labelText}...`);
+      console.log(`  🔊 ${label} (${done}/${total}): ${labelText}...`);
       const ttiCmd = clone.tti ?? options.ttiCli ?? DEFAULT_TTI_CLI;
       const result = type === "image"
         ? generateTTI(prompt, outputPath, cli, seed)
@@ -757,7 +778,8 @@ export async function resolveGeneratedMedia(
         node.src = outputPath;
         updateCache(cache, `gen:${cacheKey}`, cacheKey, outputPath);
         cacheDirty = true;
-        console.log(`  ✓ ${label}: ${labelText}`);
+        console.log(`  ✓ ${label} (${done}/${total}): ${labelText}`);
+        if (done % 3 === 0 || done === total) console.log(progressLine());
       } else {
         const hint = cli.includes("echo")
           ? `No ${label} tool installed. The default CLI just echoes a message — set root.${type === "image" ? "tti" : "ttv"} to a real generation command.`
@@ -772,6 +794,11 @@ export async function resolveGeneratedMedia(
     }
   }
 
+  if (total > 0) {
+    const elapsed = Math.round((Date.now() - startTime) / 1000);
+    const mediaType = genNodes[0]?.type === "video" ? "TTV" : "TTI";
+    console.log(`  ✅ ${mediaType} complete: ${done} items in ${elapsed}s (${cacheHits} cached)`);
+  }
   if (cacheDirty) writeCacheManifest(options.outputDir, cache);
   return clone;
 }
@@ -986,7 +1013,8 @@ export async function resolveAll(
 
   // Step 3: Generate images/videos from prompts before probing durations.
   // Use root.seed if set; otherwise inherit from options (for includes).
-  const generationSeed = result.seed ?? options.seed;
+  // Auto-generate a random seed so {seed} in the CLI template always gets a value.
+  const generationSeed = result.seed ?? options.seed ?? Math.floor(Math.random() * 2_147_483_647);
   if (options.mediaOutputDir) {
     result = await resolveGeneratedMedia(result, {
       outputDir: options.mediaOutputDir,

package/src/player/browser.tsx

@@ -136,14 +136,16 @@ function PlayerApp() {
       .catch(() => {});
   }, []);
 
-  // Update active scene from currentTime
+  // Update active scene from currentTime.
+  // Iterate backward so the latest scene whose start <= currentTime wins —
+  // during transition overlaps, the newer scene takes precedence.
   React.useEffect(() => {
     const scenes = (window as any).__scenes;
     if (!scenes) return;
     let found = "";
-    for (const s of scenes) {
-      if (currentTime >= s.start && currentTime < s.end) {
-        found = s.name || "";
+    for (let i = scenes.length - 1; i >= 0; i--) {
+      if (currentTime >= scenes[i].start) {
+        found = scenes[i].name || "";
         break;
       }
     }
@@ -213,14 +215,28 @@ function PlayerApp() {
   }, [ready, data, startAt, fps]);
 
   // ── onFrameUpdate: track current time ───────────────────────────────
+  // Remotion Player 4.x dispatches `frameupdate` events via the player ref's
+  // EventTarget API — there is no `onFrameUpdate` prop. Subscribe in an effect.
   const handleFrameUpdate = React.useCallback((frame: number) => {
     currentFrameRef.current = frame;
-    // Throttle state updates to ~2fps for scene tracking
+    // Throttle state updates lightly for scene tracking
     setCurrentTime(prev => {
-      const newTime = frame / (data?.fps ?? 30);
-      return Math.abs(newTime - prev) > 0.5 ? newTime : prev;
+      const newTime = frame / fps;
+      return Math.abs(newTime - prev) > 0.1 ? newTime : prev;
     });
-  }, [data?.fps]);
+  }, [fps]);
+
+  // Subscribe to frameupdate events on the player ref (after it mounts)
+  React.useEffect(() => {
+    const p = playerRef.current;
+    if (!p || typeof (p as any).addEventListener !== "function") return;
+    const listener = (e: any) => {
+      const frame = e?.detail?.frame;
+      if (typeof frame === "number") handleFrameUpdate(frame);
+    };
+    (p as any).addEventListener("frameupdate", listener);
+    return () => (p as any).removeEventListener("frameupdate", listener);
+  }, [ready, data, handleFrameUpdate]);
 
   // Keyboard shortcuts
   React.useEffect(() => {
@@ -233,14 +249,17 @@ function PlayerApp() {
     function seekRelative(deltaSec: number) {
       const p = playerRef.current;
       if (!p) return;
-      const frame = p.getCurrentFrame() + Math.round(deltaSec * fps);
-      p.seekTo(Math.max(0, frame));
+      const frame = Math.max(0, p.getCurrentFrame() + Math.round(deltaSec * fps));
+      p.seekTo(frame);
+      setCurrentTime(frame / fps);
     }
 
     function seekPercent(pct: number) {
       const p = playerRef.current;
       if (!p) return;
-      p.seekTo(Math.round(pct * durationInFrames));
+      const frame = Math.round(pct * durationInFrames);
+      p.seekTo(frame);
+      setCurrentTime(frame / fps);
     }
 
     function showHelp() {
@@ -439,7 +458,6 @@ function PlayerApp() {
           clickToPlay={false}
           doubleClickToFullscreen={true}
           autoPlay={autoPlay}
-          onFrameUpdate={handleFrameUpdate}
         />
       </div>
 
@@ -448,8 +466,9 @@ function PlayerApp() {
         currentTime={currentTime}
         onSeek={(t) => {
           if (playerRef.current) {
-            const frame = Math.round(t * (data?.fps ?? 30));
+            const frame = Math.round(t * fps);
             playerRef.current.seekTo(frame);
+            setCurrentTime(t);
           }
         }}
       />