mulmocast 2.4.1 → 2.4.3

package/assets/html/tailwind_animated.html

@@ -55,7 +55,7 @@
 
     // === MulmoAnimation Helper Class ===
 
-    const TRANSFORM_PROPS = { translateX: 'px', translateY: 'px', scale: '', rotate: 'deg' };
+    const TRANSFORM_PROPS = { translateX: 'px', translateY: 'px', scale: '', rotate: 'deg', rotateX: 'deg', rotateY: 'deg', rotateZ: 'deg' };
     const SVG_PROPS = ['r', 'cx', 'cy', 'x', 'y', 'x1', 'y1', 'x2', 'y2', 'rx', 'ry',
                        'width', 'height', 'stroke-width', 'stroke-dashoffset', 'stroke-dasharray', 'opacity'];
 
@@ -176,7 +176,7 @@
 
         if (entry.kind === 'animate') {
           const startFrame = (opts.start || 0) * fps;
-          const endFrame = (opts.end || 0) * fps;
+          const endFrame = (opts.end === 'auto' ? window.__MULMO.totalFrames / fps : (opts.end || 0)) * fps;
           const progress = Math.max(0, Math.min(1, endFrame === startFrame ? 1 : (frame - startFrame) / (endFrame - startFrame)));
           const el = document.querySelector(entry.selector);
           this._applyProps(el, entry.props, progress, easingFn);
@@ -196,7 +196,7 @@
 
         } else if (entry.kind === 'typewriter') {
           const twStart = (opts.start || 0) * fps;
-          const twEnd = (opts.end || 0) * fps;
+          const twEnd = (opts.end === 'auto' ? window.__MULMO.totalFrames / fps : (opts.end || 0)) * fps;
           const twProgress = Math.max(0, Math.min(1, twEnd === twStart ? 1 : (frame - twStart) / (twEnd - twStart)));
           const charCount = Math.floor(twProgress * entry.text.length);
           const twEl = document.querySelector(entry.selector);
@@ -204,7 +204,7 @@
 
         } else if (entry.kind === 'counter') {
           const cStart = (opts.start || 0) * fps;
-          const cEnd = (opts.end || 0) * fps;
+          const cEnd = (opts.end === 'auto' ? window.__MULMO.totalFrames / fps : (opts.end || 0)) * fps;
           const cProgress = Math.max(0, Math.min(1, cEnd === cStart ? 1 : (frame - cStart) / (cEnd - cStart)));
           const cVal = entry.range[0] + easingFn(cProgress) * (entry.range[1] - entry.range[0]);
           const decimals = opts.decimals || 0;
@@ -214,7 +214,7 @@
 
         } else if (entry.kind === 'codeReveal') {
           const crStart = (opts.start || 0) * fps;
-          const crEnd = (opts.end || 0) * fps;
+          const crEnd = (opts.end === 'auto' ? window.__MULMO.totalFrames / fps : (opts.end || 0)) * fps;
           const crProgress = Math.max(0, Math.min(1, crEnd === crStart ? 1 : (frame - crStart) / (crEnd - crStart)));
           const lineCount = Math.floor(crProgress * entry.lines.length);
           const crEl = document.querySelector(entry.selector);

package/lib/actions/image_agents.js

@@ -123,7 +123,9 @@ export const imagePluginAgent = async (namedInputs) => {
     const effectiveImagePath = isAnimatedHtml ? getBeatAnimatedVideoPath(context, index) : imagePath;
     try {
         MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, true);
-        const processorParams = { beat, context, imagePath: effectiveImagePath, imageRefs, ...htmlStyle(context, beat) };
+        const studioBeat = context.studio.beats[index];
+        const beatDuration = beat.duration ?? studioBeat?.duration;
+        const processorParams = { beat, context, imagePath: effectiveImagePath, imageRefs, beatDuration, ...htmlStyle(context, beat) };
         await plugin.process(processorParams);
         MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, false);
     }

package/lib/actions/movie.d.ts

@@ -1,7 +1,7 @@
 import { MulmoStudioContext, MulmoBeat, MulmoTransition, MulmoCanvasDimension, MulmoFillOption, MulmoVideoFilter } from "../types/index.js";
 import { FfmpegContext } from "../utils/ffmpeg_utils.js";
 type VideoId = string | undefined;
-export declare const getVideoPart: (inputIndex: number, isMovie: boolean, duration: number, canvasInfo: MulmoCanvasDimension, fillOption: MulmoFillOption, speed: number, filters?: MulmoVideoFilter[]) => {
+export declare const getVideoPart: (inputIndex: number, isMovie: boolean, duration: number, canvasInfo: MulmoCanvasDimension, fillOption: MulmoFillOption, speed: number, filters?: MulmoVideoFilter[], frameCount?: number) => {
     videoId: string;
     videoPart: string;
 };

package/lib/actions/movie.js

@@ -8,7 +8,8 @@ import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 import { convertVideoFilterToFFmpeg } from "../utils/video_filter.js";
 // const isMac = process.platform === "darwin";
 const videoCodec = "libx264"; // "h264_videotoolbox" (macOS only) is too noisy
-export const getVideoPart = (inputIndex, isMovie, duration, canvasInfo, fillOption, speed, filters) => {
+const VIDEO_FPS = 30;
+export const getVideoPart = (inputIndex, isMovie, duration, canvasInfo, fillOption, speed, filters, frameCount) => {
     const videoId = `v${inputIndex}`;
     const videoFilters = [];
     // Handle different media types
@@ -21,8 +22,20 @@ export const getVideoPart = (inputIndex, isMovie, duration, canvasInfo, fillOpti
     else {
         videoFilters.push("loop=loop=-1:size=1:start=0");
     }
-    // Common filters for all media types
-    videoFilters.push(`trim=duration=${originalDuration}`, "fps=30");
+    // Normalize framerate first so trim=end_frame counts frames at VIDEO_FPS,
+    // regardless of the input's native framerate.
+    videoFilters.push(`fps=${VIDEO_FPS}`);
+    // Use frame-exact trimming when frameCount is provided to prevent cumulative drift
+    // between video and audio tracks. trim=duration=X rounds up to next frame boundary,
+    // causing ~0.03s extra per beat that accumulates over many beats.
+    if (frameCount !== undefined && frameCount > 0) {
+        // Account for speed: setpts compresses timestamps, so we need more input frames
+        const inputFrameCount = Math.max(1, Math.round(frameCount * speed));
+        videoFilters.push(`trim=end_frame=${inputFrameCount}`);
+    }
+    else {
+        videoFilters.push(`trim=duration=${originalDuration}`);
+    }
     // Apply speed if specified
     if (speed === 1.0) {
         videoFilters.push("setpts=PTS-STARTPTS");
@@ -79,7 +92,7 @@ const getOutputOption = (audioId, videoId) => {
         "4M", // Reduced buffer size
         "-maxrate",
         "3M", // Reduced from 7M to 3M
-        "-r 30", // Set frame rate
+        `-r ${VIDEO_FPS}`, // Set frame rate
         "-pix_fmt yuv420p", // Set pixel format for better compatibility
         "-c:a aac", // Audio codec
         "-b:a 128k", // Audio bitrate
@@ -284,7 +297,7 @@ const getClampedTransitionDuration = (transitionDuration, prevBeatDuration, curr
     return Math.min(transitionDuration, maxDuration);
 };
 export const getTransitionFrameDurations = (context, index) => {
-    const minFrame = 1 / 30; // 30fpsを想定。最小1フレーム
+    const minFrame = 1 / VIDEO_FPS;
     const beats = context.studio.beats;
     const scriptBeats = context.studio.script.beats;
     const getTransitionDuration = (transition, prevBeatIndex, currentBeatIndex) => {
@@ -320,22 +333,22 @@ export const addSplitAndExtractFrames = (ffmpegContext, videoId, firstDuration,
     if (needFirst) {
         // Create static frame using nullsrc as base for proper framerate/timebase
         // Note: setpts must NOT be used here as it loses framerate metadata needed by xfade
-        ffmpegContext.filterComplex.push(`nullsrc=size=${canvasInfo.width}x${canvasInfo.height}:duration=${firstDuration}:rate=30[${videoId}_first_null]`);
+        ffmpegContext.filterComplex.push(`nullsrc=size=${canvasInfo.width}x${canvasInfo.height}:duration=${firstDuration}:rate=${VIDEO_FPS}[${videoId}_first_null]`);
         ffmpegContext.filterComplex.push(`[${videoId}_first_src]select='eq(n,0)',scale=${canvasInfo.width}:${canvasInfo.height}[${videoId}_first_frame]`);
-        ffmpegContext.filterComplex.push(`[${videoId}_first_null][${videoId}_first_frame]overlay=format=auto,fps=30[${videoId}_first]`);
+        ffmpegContext.filterComplex.push(`[${videoId}_first_null][${videoId}_first_frame]overlay=format=auto,fps=${VIDEO_FPS}[${videoId}_first]`);
     }
     if (needLast) {
         if (isMovie) {
             // Movie beats: extract actual last frame
-            ffmpegContext.filterComplex.push(`nullsrc=size=${canvasInfo.width}x${canvasInfo.height}:duration=${lastDuration}:rate=30[${videoId}_last_null]`);
+            ffmpegContext.filterComplex.push(`nullsrc=size=${canvasInfo.width}x${canvasInfo.height}:duration=${lastDuration}:rate=${VIDEO_FPS}[${videoId}_last_null]`);
             ffmpegContext.filterComplex.push(`[${videoId}_last_src]reverse,select='eq(n,0)',reverse,scale=${canvasInfo.width}:${canvasInfo.height}[${videoId}_last_frame]`);
-            ffmpegContext.filterComplex.push(`[${videoId}_last_null][${videoId}_last_frame]overlay=format=auto,fps=30[${videoId}_last]`);
+            ffmpegContext.filterComplex.push(`[${videoId}_last_null][${videoId}_last_frame]overlay=format=auto,fps=${VIDEO_FPS}[${videoId}_last]`);
         }
         else {
             // Image beats: all frames are identical, so just select one
-            ffmpegContext.filterComplex.push(`nullsrc=size=${canvasInfo.width}x${canvasInfo.height}:duration=${lastDuration}:rate=30[${videoId}_last_null]`);
+            ffmpegContext.filterComplex.push(`nullsrc=size=${canvasInfo.width}x${canvasInfo.height}:duration=${lastDuration}:rate=${VIDEO_FPS}[${videoId}_last_null]`);
             ffmpegContext.filterComplex.push(`[${videoId}_last_src]select='eq(n,0)',scale=${canvasInfo.width}:${canvasInfo.height}[${videoId}_last_frame]`);
-            ffmpegContext.filterComplex.push(`[${videoId}_last_null][${videoId}_last_frame]overlay=format=auto,fps=30[${videoId}_last]`);
+            ffmpegContext.filterComplex.push(`[${videoId}_last_null][${videoId}_last_frame]overlay=format=auto,fps=${VIDEO_FPS}[${videoId}_last]`);
         }
     }
 };
@@ -367,6 +380,7 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
     const needsFirstFrame = getNeedFirstFrame(context);
     // Check which beats need _last (for any transition on next beat - they all need previous beat's last frame)
     const needsLastFrame = getNeedLastFrame(context);
+    let cumulativeFrames = 0;
     context.studio.beats.reduce((timestamp, studioBeat, index) => {
         const beat = context.studio.script.beats[index];
         if (beat.image?.type === "voice_over") {
@@ -377,13 +391,19 @@ export const createVideo = async (audioArtifactFilePath, outputVideoPath, contex
         const sourceFile = isTest ? "/test/dummy.mp4" : validateBeatSource(studioBeat, index);
         // The movie duration is bigger in case of voice-over.
         const duration = Math.max(studioBeat.duration + getExtraPadding(context, index), studioBeat.movieDuration ?? 0);
+        // Use cumulative frame tracking to prevent audio-video drift from frame quantization.
+        // trim=duration=X rounds up to the next frame boundary (~0.03s per beat at 30fps),
+        // causing cumulative drift. Instead, compute exact frame counts per beat.
+        const targetEndFrame = Math.round((timestamp + duration) * VIDEO_FPS);
+        const frameCount = targetEndFrame - cumulativeFrames;
+        cumulativeFrames = targetEndFrame;
         const inputIndex = FfmpegContextAddInput(ffmpegContext, sourceFile);
         const isMovie = !!(studioBeat.lipSyncFile ||
             studioBeat.movieFile ||
             MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat) === "movie");
         const speed = beat.movieParams?.speed ?? 1.0;
         const filters = beat.movieParams?.filters;
-        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, getFillOption(context, beat), speed, filters);
+        const { videoId, videoPart } = getVideoPart(inputIndex, isMovie, duration, canvasInfo, getFillOption(context, beat), speed, filters, frameCount);
         ffmpegContext.filterComplex.push(videoPart);
         // for transition
         const needFirst = needsFirstFrame[index]; // This beat has slidein

package/lib/types/type.d.ts

@@ -84,6 +84,7 @@ export type ImageProcessorParams = {
     textSlideStyle: string;
     canvasSize: MulmoCanvasDimension;
     imageRefs?: Record<string, string>;
+    beatDuration?: number;
 };
 export type PDFMode = (typeof pdf_modes)[number];
 export type PDFSize = (typeof pdf_sizes)[number];

package/lib/utils/image_plugins/html_tailwind.js

@@ -34,9 +34,9 @@ const processHtmlTailwindAnimated = async (params) => {
     const animConfig = getAnimationConfig(params);
     if (!animConfig)
         return;
-    const duration = beat.duration;
+    const duration = params.beatDuration ?? beat.duration;
     if (duration === undefined) {
-        throw new Error("html_tailwind animation requires explicit beat.duration. Set duration in the beat definition.");
+        throw new Error("html_tailwind animation requires beat.duration or audio-derived duration. Set duration in the beat or ensure audio is generated first.");
     }
     const fps = animConfig.fps;
     const totalFrames = Math.floor(duration * fps);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.4.1",
+  "version": "2.4.3",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",

package/scripts/test/test_html_animation.json

@@ -546,6 +546,83 @@
         "animation": { "fps": 24 }
       }
     },
+    {
+      "id": "demo_3d_card_flip",
+      "speaker": "Presenter",
+      "duration": 3,
+      "image": {
+        "type": "html_tailwind",
+        "html": [
+          "<div class='h-full flex items-center justify-center bg-gradient-to-br from-slate-900 to-indigo-950'>",
+          "  <div style='perspective:1000px'>",
+          "    <div id='card' class='relative' style='width:340px;height:200px;transform-style:preserve-3d'>",
+          "      <div class='absolute inset-0 rounded-2xl flex flex-col items-center justify-center' style='backface-visibility:hidden;background:linear-gradient(135deg,#3b82f6,#06b6d4);box-shadow:0 20px 60px rgba(6,182,212,0.3)'>",
+          "        <p class='text-white text-3xl font-bold tracking-wide'>MulmoCast</p>",
+          "        <p class='text-blue-200 text-sm mt-2 tracking-wider'>FRONT SIDE</p>",
+          "      </div>",
+          "      <div class='absolute inset-0 rounded-2xl flex flex-col items-center justify-center' style='backface-visibility:hidden;transform:rotateY(180deg);background:linear-gradient(135deg,#8b5cf6,#ec4899);box-shadow:0 20px 60px rgba(139,92,246,0.3)'>",
+          "        <p class='text-white text-3xl font-bold tracking-wide'>AI-Native</p>",
+          "        <p class='text-purple-200 text-sm mt-2 tracking-wider'>BACK SIDE</p>",
+          "      </div>",
+          "    </div>",
+          "  </div>",
+          "</div>"
+        ],
+        "script": [
+          "const animation = new MulmoAnimation();",
+          "animation.animate('#card', { rotateY: [0, 180] }, { start: 0.5, end: 2.5, easing: 'easeInOut' });"
+        ],
+        "animation": true
+      }
+    },
+    {
+      "id": "demo_3d_title_reveal",
+      "speaker": "Presenter",
+      "duration": 3,
+      "image": {
+        "type": "html_tailwind",
+        "html": [
+          "<div class='h-full flex flex-col items-center justify-center bg-black' style='perspective:800px'>",
+          "  <h1 id='title' class='text-7xl font-bold tracking-wider' style='opacity:0;font-family:Impact,sans-serif;color:white;text-shadow:0 0 40px rgba(6,182,212,0.5)'>CINEMATIC</h1>",
+          "  <div id='line' class='h-0.5 mt-6 rounded' style='width:0;background:linear-gradient(90deg,transparent,#06b6d4,transparent)'></div>",
+          "  <p id='sub' class='text-lg mt-6 tracking-[0.4em]' style='opacity:0;color:#64748b;font-family:monospace'>3D PERSPECTIVE REVEAL</p>",
+          "</div>"
+        ],
+        "script": [
+          "const animation = new MulmoAnimation();",
+          "animation.animate('#title', { opacity: [0, 1], rotateX: [90, 0] }, { start: 0.2, end: 1.2, easing: 'easeOut' });",
+          "animation.animate('#line', { width: [0, 400, 'px'] }, { start: 1.0, end: 1.8, easing: 'easeOut' });",
+          "animation.animate('#sub', { opacity: [0, 1] }, { start: 1.5, end: 2.2, easing: 'easeOut' });"
+        ],
+        "animation": true
+      }
+    },
+    {
+      "id": "demo_split_reveal",
+      "speaker": "Presenter",
+      "duration": 3,
+      "image": {
+        "type": "html_tailwind",
+        "html": [
+          "<div class='h-full flex bg-black overflow-hidden'>",
+          "  <div id='left' class='flex-1 flex items-center justify-center' style='background:linear-gradient(135deg,#1e3a5f,#0f172a);opacity:0'>",
+          "    <p class='text-6xl font-bold text-white' style='font-family:Georgia,serif'>Create</p>",
+          "  </div>",
+          "  <div id='divider' class='w-1' style='background:linear-gradient(to bottom,transparent,#06b6d4,transparent);opacity:0'></div>",
+          "  <div id='right' class='flex-1 flex items-center justify-center' style='background:linear-gradient(225deg,#4c1d95,#0f172a);opacity:0'>",
+          "    <p class='text-6xl font-bold text-white' style='font-family:Georgia,serif'>Inspire</p>",
+          "  </div>",
+          "</div>"
+        ],
+        "script": [
+          "const animation = new MulmoAnimation();",
+          "animation.animate('#left', { translateX: [-640, 0], opacity: [0, 1] }, { start: 0, end: 1.0, easing: 'easeOut' });",
+          "animation.animate('#right', { translateX: [640, 0], opacity: [0, 1] }, { start: 0.3, end: 1.3, easing: 'easeOut' });",
+          "animation.animate('#divider', { opacity: [0, 1] }, { start: 1.2, end: 1.8 });"
+        ],
+        "animation": true
+      }
+    },
     {
       "speaker": "Presenter",
       "duration": 2,