npm - mulmocast - Versions diffs - 2.3.2 → 2.4.0 - Mend

mulmocast 2.3.2 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (20) hide show

package/assets/html/tailwind.html +1 -0
package/assets/html/tailwind_animated.html +259 -0
package/lib/actions/image_agents.js +23 -2
package/lib/actions/images.js +2 -2
package/lib/agents/combine_audio_files_agent.js +5 -0
package/lib/methods/mulmo_beat.d.ts +7 -0
package/lib/methods/mulmo_beat.js +18 -0
package/lib/types/schema.d.ts +23 -0
package/lib/types/schema.js +10 -0
package/lib/utils/context.d.ts +8 -0
package/lib/utils/ffmpeg_utils.d.ts +10 -0
package/lib/utils/ffmpeg_utils.js +27 -0
package/lib/utils/file.d.ts +1 -0
package/lib/utils/file.js +6 -0
package/lib/utils/html_render.d.ts +11 -0
package/lib/utils/html_render.js +84 -34
package/lib/utils/image_plugins/html_tailwind.js +78 -6
package/package.json +3 -3
package/scripts/test/test_html_animation.json +563 -0
package/scripts/test/test_vocab_animation.json +226 -0

package/assets/html/tailwind.html CHANGED Viewed

@@ -43,5 +43,6 @@
 </head>
 <body class="bg-white text-gray-800 h-full flex flex-col">
   ${html_body}
+  ${user_script}
 </body>
 </html>

package/assets/html/tailwind_animated.html ADDED Viewed

@@ -0,0 +1,259 @@
+<!DOCTYPE html>
+<html lang="en" class="h-full">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <script src="https://cdn.tailwindcss.com"></script>
+  <style>
+    /* Disable all CSS animations/transitions for deterministic frame-based rendering */
+    *, *::before, *::after {
+      animation-play-state: paused !important;
+      transition: none !important;
+    }
+    ${custom_style}
+  </style>
+</head>
+<body class="bg-white text-gray-800 h-full flex flex-col">
+  ${html_body}
+  <script>
+    // === MulmoCast Animation Helpers ===
+    /**
+     * Easing functions for non-linear interpolation.
+     */
+    const Easing = {
+      linear: (t) => t,
+      easeIn: (t) => t * t,
+      easeOut: (t) => 1 - (1 - t) * (1 - t),
+      easeInOut: (t) => t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2,
+    };
+    /**
+     * Interpolation with clamping and optional easing.
+     *
+     * @param {number} value - Current value (typically frame number)
+     * @param {Object} opts - { input: { inMin, inMax }, output: { outMin, outMax }, easing?: string | function }
+     * @returns {number} Interpolated and clamped value
+     *
+     * @example
+     *   interpolate(frame, { input: { inMin: 0, inMax: 30 }, output: { outMin: 0, outMax: 1 } })
+     *   interpolate(frame, { input: { inMin: 0, inMax: 30 }, output: { outMin: 0, outMax: 1 }, easing: 'easeOut' })
+     */
+    function interpolate(value, opts) {
+      const { inMin, inMax } = opts.input;
+      const { outMin, outMax } = opts.output;
+      if (inMax === inMin) {
+        return outMin;
+      }
+      const easing = !opts.easing ? Easing.linear
+        : typeof opts.easing === 'function' ? opts.easing
+        : Easing[opts.easing] || Easing.linear;
+      const progress = Math.max(0, Math.min(1, (value - inMin) / (inMax - inMin)));
+      return outMin + easing(progress) * (outMax - outMin);
+    }
+    // === MulmoAnimation Helper Class ===
+    const TRANSFORM_PROPS = { translateX: 'px', translateY: 'px', scale: '', rotate: 'deg' };
+    const SVG_PROPS = ['r', 'cx', 'cy', 'x', 'y', 'x1', 'y1', 'x2', 'y2', 'rx', 'ry',
+                       'width', 'height', 'stroke-width', 'stroke-dashoffset', 'stroke-dasharray', 'opacity'];
+    function MulmoAnimation() {
+      this._entries = [];
+    }
+    /**
+     * Register a property animation on a single element.
+     * @param {string} selector - CSS selector (e.g. '#title')
+     * @param {Object} props - { opacity: [0, 1], translateY: [30, 0], width: [0, 80, '%'] }
+     * @param {Object} opts - { start, end, easing }  (start/end in seconds)
+     */
+    MulmoAnimation.prototype.animate = function(selector, props, opts) {
+      this._entries.push({ kind: 'animate', selector, props, opts: opts || {} });
+      return this;
+    };
+    /**
+     * Stagger animation across numbered elements.
+     * Selector must contain {i} placeholder (e.g. '#item{i}').
+     * @param {string} selector - e.g. '#item{i}'
+     * @param {number} count - number of elements (0-indexed)
+     * @param {Object} props - same as animate()
+     * @param {Object} opts - { start, stagger, duration, easing }
+     */
+    MulmoAnimation.prototype.stagger = function(selector, count, props, opts) {
+      this._entries.push({ kind: 'stagger', selector, count, props, opts: opts || {} });
+      return this;
+    };
+    /**
+     * Typewriter effect — reveal text character by character.
+     * @param {string} selector - target element selector
+     * @param {string} text - full text to reveal
+     * @param {Object} opts - { start, end }
+     */
+    MulmoAnimation.prototype.typewriter = function(selector, text, opts) {
+      this._entries.push({ kind: 'typewriter', selector, text, opts: opts || {} });
+      return this;
+    };
+    /**
+     * Animated counter — interpolate a number and display with optional prefix/suffix.
+     * @param {string} selector - target element selector
+     * @param {[number, number]} range - [from, to]
+     * @param {Object} opts - { start, end, prefix, suffix, decimals }
+     */
+    MulmoAnimation.prototype.counter = function(selector, range, opts) {
+      this._entries.push({ kind: 'counter', selector, range, opts: opts || {} });
+      return this;
+    };
+    /**
+     * Code reveal — show lines of code one by one (line-level typewriter).
+     * @param {string} selector - target element selector
+     * @param {string[]} lines - array of code lines
+     * @param {Object} opts - { start, end }
+     */
+    MulmoAnimation.prototype.codeReveal = function(selector, lines, opts) {
+      this._entries.push({ kind: 'codeReveal', selector, lines, opts: opts || {} });
+      return this;
+    };
+    /**
+     * Blink — periodic show/hide toggle (e.g. cursor blinking).
+     * @param {string} selector - target element selector
+     * @param {Object} opts - { interval } (half-cycle seconds, default 0.5)
+     */
+    MulmoAnimation.prototype.blink = function(selector, opts) {
+      this._entries.push({ kind: 'blink', selector, opts: opts || {} });
+      return this;
+    };
+    /** Resolve easing name string or function to an easing function */
+    MulmoAnimation.prototype._resolveEasing = function(e) {
+      if (!e) return Easing.linear;
+      if (typeof e === 'function') return e;
+      return Easing[e] || Easing.linear;
+    };
+    /** Apply props to element at a given progress (0-1) with easing */
+    MulmoAnimation.prototype._applyProps = function(el, props, progress, easingFn) {
+      if (!el) return;
+      const transforms = [];
+      Object.keys(props).forEach((prop) => {
+        const spec = props[prop];
+        const from = spec[0], to = spec[1];
+        const unit = (spec.length > 2) ? spec[2] : null;
+        const val = from + easingFn(progress) * (to - from);
+        if (TRANSFORM_PROPS.hasOwnProperty(prop)) {
+          const tUnit = unit || TRANSFORM_PROPS[prop];
+          transforms.push(prop === 'scale' ? 'scale(' + val + ')' : prop + '(' + val + tUnit + ')');
+        } else if (el instanceof SVGElement && SVG_PROPS.indexOf(prop) !== -1) {
+          el.setAttribute(prop, val);
+        } else if (prop === 'opacity') {
+          el.style.opacity = val;
+        } else {
+          const cssUnit = unit || 'px';
+          el.style[prop] = val + cssUnit;
+        }
+      });
+      if (transforms.length > 0) {
+        el.style.transform = transforms.join(' ');
+      }
+    };
+    /**
+     * Update all registered animations for the given frame.
+     * @param {number} frame - current frame number
+     * @param {number} fps - frames per second
+     */
+    MulmoAnimation.prototype.update = function(frame, fps) {
+      this._entries.forEach((entry) => {
+        const opts = entry.opts;
+        const easingFn = this._resolveEasing(opts.easing);
+        if (entry.kind === 'animate') {
+          const startFrame = (opts.start || 0) * fps;
+          const endFrame = (opts.end || 0) * fps;
+          const progress = Math.max(0, Math.min(1, endFrame === startFrame ? 1 : (frame - startFrame) / (endFrame - startFrame)));
+          const el = document.querySelector(entry.selector);
+          this._applyProps(el, entry.props, progress, easingFn);
+        } else if (entry.kind === 'stagger') {
+          const baseStart = (opts.start || 0) * fps;
+          const staggerDelay = (opts.stagger || 0.2) * fps;
+          const dur = (opts.duration || 0.5) * fps;
+          for (let j = 0; j < entry.count; j++) {
+            const sel = entry.selector.replace(/\{i\}/g, j);
+            const sEl = document.querySelector(sel);
+            const sStart = baseStart + j * staggerDelay;
+            const sEnd = sStart + dur;
+            const sProgress = Math.max(0, Math.min(1, sEnd === sStart ? 1 : (frame - sStart) / (sEnd - sStart)));
+            this._applyProps(sEl, entry.props, sProgress, easingFn);
+          }
+        } else if (entry.kind === 'typewriter') {
+          const twStart = (opts.start || 0) * fps;
+          const twEnd = (opts.end || 0) * fps;
+          const twProgress = Math.max(0, Math.min(1, twEnd === twStart ? 1 : (frame - twStart) / (twEnd - twStart)));
+          const charCount = Math.floor(twProgress * entry.text.length);
+          const twEl = document.querySelector(entry.selector);
+          if (twEl) twEl.textContent = entry.text.substring(0, charCount);
+        } else if (entry.kind === 'counter') {
+          const cStart = (opts.start || 0) * fps;
+          const cEnd = (opts.end || 0) * fps;
+          const cProgress = Math.max(0, Math.min(1, cEnd === cStart ? 1 : (frame - cStart) / (cEnd - cStart)));
+          const cVal = entry.range[0] + easingFn(cProgress) * (entry.range[1] - entry.range[0]);
+          const decimals = opts.decimals || 0;
+          const display = (opts.prefix || '') + cVal.toFixed(decimals) + (opts.suffix || '');
+          const cEl = document.querySelector(entry.selector);
+          if (cEl) cEl.textContent = display;
+        } else if (entry.kind === 'codeReveal') {
+          const crStart = (opts.start || 0) * fps;
+          const crEnd = (opts.end || 0) * fps;
+          const crProgress = Math.max(0, Math.min(1, crEnd === crStart ? 1 : (frame - crStart) / (crEnd - crStart)));
+          const lineCount = Math.floor(crProgress * entry.lines.length);
+          const crEl = document.querySelector(entry.selector);
+          if (crEl) crEl.textContent = entry.lines.slice(0, lineCount).join('\n');
+        } else if (entry.kind === 'blink') {
+          const interval_s = opts.interval || 0.5;
+          const blinkEl = document.querySelector(entry.selector);
+          if (blinkEl) {
+            const cycle = (frame / fps) / interval_s;
+            blinkEl.style.opacity = (Math.floor(cycle) % 2 === 0) ? 1 : 0;
+          }
+        }
+      });
+    };
+    // === MulmoCast Frame State (updated by Puppeteer per frame) ===
+    window.__MULMO = {
+      frame: 0,
+      totalFrames: ${totalFrames},
+      fps: ${fps},
+    };
+  </script>
+  ${user_script}
+  <script>
+    // Auto-render: if MulmoAnimation is used but render() is not defined, generate it
+    if (typeof render !== 'function' && typeof animation !== 'undefined' && animation instanceof MulmoAnimation) {
+      window.render = function(frame, totalFrames, fps) { animation.update(frame, fps); };
+    }
+    // Initial render (frame 0)
+    if (typeof render === 'function') {
+      const result = render(0, window.__MULMO.totalFrames, window.__MULMO.fps);
+      if (result && typeof result.then === 'function') {
+        result.catch(console.error);
+      }
+    }
+  </script>
+</body>
+</html>

package/lib/actions/image_agents.js CHANGED Viewed

@@ -1,6 +1,6 @@
 import { GraphAILogger } from "graphai";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMethods, MulmoMediaSourceMethods } from "../methods/index.js";
-import { getBeatPngImagePath, getBeatMoviePaths, getAudioFilePath, getGroupedAudioFilePath } from "../utils/file.js";
+import { getBeatPngImagePath, getBeatMoviePaths, getBeatAnimatedVideoPath, getAudioFilePath, getGroupedAudioFilePath } from "../utils/file.js";
 import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
 import { renderHTMLToImage } from "../utils/html_render.js";
 import { beatId } from "../utils/utils.js";
@@ -72,6 +72,24 @@ export const imagePreprocessAgent = async (namedInputs) => {
         const markdown = plugin.markdown ? plugin.markdown({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
         const html = plugin.html ? await plugin.html({ beat, context, imagePath, ...htmlStyle(context, beat) }) : undefined;
         const isTypeMovie = beat.image.type === "movie";
+        const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
+        // animation and moviePrompt cannot be used together
+        if (isAnimatedHtml && beat.moviePrompt) {
+            throw new Error("html_tailwind animation and moviePrompt cannot be used together on the same beat. Use either animation or moviePrompt, not both.");
+        }
+        if (isAnimatedHtml) {
+            const animatedVideoPath = getBeatAnimatedVideoPath(context, index);
+            // ImagePluginPreprocessAgentResponse
+            return {
+                ...returnValue,
+                imagePath, // for thumbnail extraction
+                movieFile: animatedVideoPath, // .mp4 path for the pipeline
+                imageFromMovie: true, // triggers extractImageFromMovie
+                referenceImageForMovie: pluginPath,
+                markdown,
+                html,
+            };
+        }
         // undefined prompt indicates that image generation is not needed
         // ImagePluginPreprocessAgentResponse
         return {
@@ -99,9 +117,12 @@ export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index, imageRefs } = namedInputs;
     const { imagePath } = getBeatPngImagePath(context, index);
     const plugin = MulmoBeatMethods.getPlugin(beat);
+    // For animated html_tailwind, use the .mp4 path so the plugin writes video there
+    const isAnimatedHtml = MulmoBeatMethods.isAnimatedHtmlTailwind(beat);
+    const effectiveImagePath = isAnimatedHtml ? getBeatAnimatedVideoPath(context, index) : imagePath;
     try {
         MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, true);
-        const processorParams = { beat, context, imagePath, imageRefs, ...htmlStyle(context, beat) };
+        const processorParams = { beat, context, imagePath: effectiveImagePath, imageRefs, ...htmlStyle(context, beat) };
         await plugin.process(processorParams);
         MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, false);
     }

package/lib/actions/images.js CHANGED Viewed

@@ -190,7 +190,7 @@ export const beat_graph_data = {
                 return await extractImageFromMovie(namedInputs.movieFile, namedInputs.imageFile);
             },
             inputs: {
-                onComplete: [":movieGenerator"], // to wait for movieGenerator to finish
+                onComplete: [":movieGenerator", ":imagePlugin"], // :imagePlugin for animated html_tailwind video generation
                 imageFile: ":preprocessor.imagePath",
                 movieFile: ":preprocessor.movieFile",
             },
@@ -219,7 +219,7 @@ export const beat_graph_data = {
                 }
             },
             inputs: {
-                onComplete: [":movieGenerator", ":htmlImageGenerator", ":soundEffectGenerator"],
+                onComplete: [":movieGenerator", ":htmlImageGenerator", ":soundEffectGenerator", ":imagePlugin"], // :imagePlugin for animated html_tailwind video generation
                 movieFile: ":preprocessor.movieFile",
                 imageFile: ":preprocessor.imagePath",
                 soundEffectFile: ":preprocessor.soundEffectFile",

package/lib/agents/combine_audio_files_agent.js CHANGED Viewed

@@ -2,6 +2,7 @@ import { assert, GraphAILogger } from "graphai";
 import { silent60secPath, isFile } from "../utils/file.js";
 import { FfmpegContextInit, FfmpegContextGenerateOutput, FfmpegContextInputFormattedAudio, ffmpegGetMediaDuration, } from "../utils/ffmpeg_utils.js";
 import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
+import { MulmoBeatMethods } from "../methods/index.js";
 import { userAssert } from "../utils/utils.js";
 import { getAudioInputIdsError } from "../utils/error_cause.js";
 const getMovieDuration = async (context, beat) => {
@@ -13,6 +14,10 @@ const getMovieDuration = async (context, beat) => {
             return { duration: duration / speed, hasAudio };
         }
     }
+    // Animated html_tailwind beats with explicit duration act as movie-like for voice_over grouping
+    if (MulmoBeatMethods.isAnimatedHtmlTailwind(beat) && beat.duration !== undefined) {
+        return { duration: beat.duration, hasAudio: false };
+    }
     return { duration: 0, hasAudio: false };
 };
 export const getPadding = (context, beat, index) => {

package/lib/methods/mulmo_beat.d.ts CHANGED Viewed

@@ -1,5 +1,11 @@
 import { MulmoBeat } from "../types/index.js";
+type AnimationConfig = {
+    fps?: number;
+};
 export declare const MulmoBeatMethods: {
+    isAnimationEnabled: (animation: unknown) => animation is true | AnimationConfig;
+    isAnimationObject: (animation: unknown) => animation is AnimationConfig;
+    isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
     getHtmlPrompt(beat: MulmoBeat): string | undefined;
     getPlugin(beat: MulmoBeat): {
         imageType: string;
@@ -10,3 +16,4 @@ export declare const MulmoBeatMethods: {
     };
     getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
 };
+export {};

package/lib/methods/mulmo_beat.js CHANGED Viewed

@@ -1,5 +1,23 @@
 import { findImagePlugin } from "../utils/image_plugins/index.js";
+/** Type guard: checks if animation value is an object config like { fps: 30 } */
+const isAnimationObject = (animation) => {
+    return typeof animation === "object" && animation !== null && !Array.isArray(animation);
+};
+/** Check if a value is a valid animation config (true or non-array object) */
+const isAnimationEnabled = (animation) => {
+    return animation === true || isAnimationObject(animation);
+};
+/** Check if a beat has html_tailwind animation enabled */
+const isAnimatedHtmlTailwind = (beat) => {
+    if (!beat.image || beat.image.type !== "html_tailwind")
+        return false;
+    const animation = beat.image.animation;
+    return isAnimationEnabled(animation);
+};
 export const MulmoBeatMethods = {
+    isAnimationEnabled,
+    isAnimationObject,
+    isAnimatedHtmlTailwind,
     getHtmlPrompt(beat) {
         if (beat?.htmlPrompt?.data) {
             return beat.htmlPrompt.prompt + "\n\n[data]\n" + JSON.stringify(beat.htmlPrompt.data, null, 2);

package/lib/types/schema.d.ts CHANGED Viewed

@@ -371,9 +371,16 @@ export declare const mulmoMermaidMediaSchema: z.ZodObject<{
         opacity: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>]>>>;
 }, z.core.$strict>;
+export declare const htmlTailwindAnimationSchema: z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
+    fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+}, z.core.$strip>]>;
 export declare const mulmoHtmlTailwindMediaSchema: z.ZodObject<{
     type: z.ZodLiteral<"html_tailwind">;
     html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
+    script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+    animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
+        fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    }, z.core.$strip>]>>;
 }, z.core.$strict>;
 export declare const mulmoBeatReferenceMediaSchema: z.ZodObject<{
     type: z.ZodLiteral<"beat">;
@@ -542,6 +549,10 @@ export declare const mulmoImageAssetSchema: z.ZodUnion<readonly [z.ZodObject<{
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodLiteral<"html_tailwind">;
     html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
+    script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+    animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
+        fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    }, z.core.$strip>]>>;
 }, z.core.$strict>, z.ZodObject<{
     type: z.ZodLiteral<"beat">;
     id: z.ZodOptional<z.ZodString>;
@@ -3591,6 +3602,10 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodLiteral<"html_tailwind">;
         html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
+        script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+        animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
+            fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        }, z.core.$strip>]>>;
     }, z.core.$strict>, z.ZodObject<{
         type: z.ZodLiteral<"beat">;
         id: z.ZodOptional<z.ZodString>;
@@ -7382,6 +7397,10 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         }, z.core.$strict>, z.ZodObject<{
             type: z.ZodLiteral<"html_tailwind">;
             html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
+            script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+            animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
+                fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+            }, z.core.$strip>]>>;
         }, z.core.$strict>, z.ZodObject<{
             type: z.ZodLiteral<"beat">;
             id: z.ZodOptional<z.ZodString>;
@@ -10793,6 +10812,10 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"html_tailwind">;
                 html: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
+                script: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>>;
+                animation: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodObject<{
+                    fps: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+                }, z.core.$strip>]>>;
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"beat">;
                 id: z.ZodOptional<z.ZodString>;

package/lib/types/schema.js CHANGED Viewed

@@ -193,10 +193,20 @@ export const mulmoMermaidMediaSchema = z
     backgroundImage: backgroundImageSchema,
 })
     .strict();
+export const htmlTailwindAnimationSchema = z.union([
+    z.literal(true),
+    z.object({
+        fps: z.number().min(1).max(60).optional().default(30),
+    }),
+]);
 export const mulmoHtmlTailwindMediaSchema = z
     .object({
     type: z.literal("html_tailwind"),
     html: stringOrStringArray,
+    script: stringOrStringArray.optional().describe("JavaScript code for the beat. Injected as a <script> tag after html. Use for render() function etc."),
+    animation: htmlTailwindAnimationSchema
+        .optional()
+        .describe("Enable frame-based animation (Remotion-style). true for defaults (30fps), or { fps: N } for custom frame rate."),
 })
     .strict();
 export const mulmoBeatReferenceMediaSchema = z

package/lib/utils/context.d.ts CHANGED Viewed

@@ -1722,6 +1722,10 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             } | {
                 type: "html_tailwind";
                 html: string | string[];
+                script?: string | string[] | undefined;
+                animation?: true | {
+                    fps: number;
+                } | undefined;
             } | {
                 type: "beat";
                 id?: string | undefined;
@@ -3810,6 +3814,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | {
                     type: "html_tailwind";
                     html: string | string[];
+                    script?: string | string[] | undefined;
+                    animation?: true | {
+                        fps: number;
+                    } | undefined;
                 } | {
                     type: "beat";
                     id?: string | undefined;

package/lib/utils/ffmpeg_utils.d.ts CHANGED Viewed

@@ -11,6 +11,16 @@ export declare const FfmpegContextAddInput: (context: FfmpegContext, input: stri
 export declare const FfmpegContextPushFormattedAudio: (context: FfmpegContext, sourceId: string, outputId: string, duration?: number | undefined) => void;
 export declare const FfmpegContextInputFormattedAudio: (context: FfmpegContext, input: string, duration?: number | undefined, inputOptions?: string[]) => string;
 export declare const FfmpegContextGenerateOutput: (context: FfmpegContext, output: string, options?: string[]) => Promise<number>;
+/** Round up odd dimensions to even (required by libx264 yuv420p) */
+export declare const normalizeEvenDimensions: (width: number, height: number) => {
+    width: number;
+    height: number;
+};
+/**
+ * Convert a sequence of PNG frames into a video file.
+ * Expects files named frame_00000.png, frame_00001.png, etc. in framesDir.
+ */
+export declare const framesToVideo: (framesDir: string, outputPath: string, fps: number, width: number, height: number) => Promise<void>;
 export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<{
     duration: number;
     hasAudio: boolean;

package/lib/utils/ffmpeg_utils.js CHANGED Viewed

@@ -62,6 +62,33 @@ export const FfmpegContextGenerateOutput = (context, output, options = []) => {
             .run();
     });
 };
+/** Round up odd dimensions to even (required by libx264 yuv420p) */
+export const normalizeEvenDimensions = (width, height) => {
+    return {
+        width: width % 2 === 0 ? width : width + 1,
+        height: height % 2 === 0 ? height : height + 1,
+    };
+};
+/**
+ * Convert a sequence of PNG frames into a video file.
+ * Expects files named frame_00000.png, frame_00001.png, etc. in framesDir.
+ */
+export const framesToVideo = (framesDir, outputPath, fps, width, height) => {
+    const safe = normalizeEvenDimensions(width, height);
+    if (safe.width !== width || safe.height !== height) {
+        GraphAILogger.info(`framesToVideo: adjusted ${width}x${height} → ${safe.width}x${safe.height} (libx264 yuv420p requires even dimensions)`);
+    }
+    return new Promise((resolve, reject) => {
+        ffmpeg()
+            .input(`${framesDir}/frame_%05d.png`)
+            .inputOptions(["-framerate", String(fps)])
+            .outputOptions(["-c:v", "libx264", "-pix_fmt", "yuv420p", "-r", String(fps), "-vf", `scale=${safe.width}:${safe.height}`])
+            .output(outputPath)
+            .on("end", () => resolve())
+            .on("error", (err) => reject(err))
+            .run();
+    });
+};
 export const ffmpegGetMediaDuration = (filePath) => {
     return new Promise((resolve, reject) => {
         // Only check file existence for local paths, not URLs

package/lib/utils/file.d.ts CHANGED Viewed

@@ -30,6 +30,7 @@ export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: n
     imagePath: string;
     htmlImageFile: string;
 };
+export declare const getBeatAnimatedVideoPath: (context: MulmoStudioContext, index: number) => string;
 export declare const getBeatMoviePaths: (context: MulmoStudioContext, index: number) => {
     movieFile: string;
     soundEffectFile: string;

package/lib/utils/file.js CHANGED Viewed

@@ -99,6 +99,12 @@ export const getBeatPngImagePath = (context, index) => {
     const htmlImageFile = `${imageProjectDirPath}/${filename}_html.png`;
     return { imagePath, htmlImageFile };
 };
+export const getBeatAnimatedVideoPath = (context, index) => {
+    const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    const beat = context.studio.script.beats[index];
+    const filename = beat?.id ? `${beat.id}` : `${index}${imageSuffix}`;
+    return `${imageProjectDirPath}/${filename}_animated.mp4`;
+};
 export const getBeatMoviePaths = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
     const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.

package/lib/utils/html_render.d.ts CHANGED Viewed

@@ -1,3 +1,14 @@
 export declare const renderHTMLToImage: (html: string, outputPath: string, width: number, height: number, isMermaid?: boolean, omitBackground?: boolean) => Promise<void>;
+/**
+ * Render an animated HTML page frame-by-frame using Puppeteer.
+ *
+ * For each frame:
+ *   1. Update window.__MULMO.frame
+ *   2. Call render(frame, totalFrames, fps) — awaits if it returns a Promise
+ *   3. Take a screenshot
+ *
+ * The user-defined render() function may be sync or async.
+ */
+export declare const renderHTMLToFrames: (html: string, outputDir: string, width: number, height: number, totalFrames: number, fps: number) => Promise<string[]>;
 export declare const renderMarkdownToImage: (markdown: string, style: string, outputPath: string, width: number, height: number) => Promise<void>;
 export declare const interpolate: (template: string, data: Record<string, string>) => string;