mulmocast 2.6.0 → 2.6.2

package/lib/actions/image_agents.d.ts

@@ -60,6 +60,7 @@ export declare const imagePluginAgent: (namedInputs: {
     beat: MulmoBeat;
     index: number;
     imageRefs?: Record<string, string>;
+    movieRefs?: Record<string, string>;
 }) => Promise<void>;
 export declare const htmlImageGeneratorAgent: (namedInputs: {
     file: string;

package/lib/actions/image_agents.js

@@ -115,7 +115,7 @@ export const imagePreprocessAgent = async (namedInputs) => {
     return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages };
 };
 export const imagePluginAgent = async (namedInputs) => {
-    const { context, beat, index, imageRefs } = namedInputs;
+    const { context, beat, index, imageRefs, movieRefs } = namedInputs;
     const { imagePath } = getBeatPngImagePath(context, index);
     const plugin = MulmoBeatMethods.getPlugin(beat);
     // For animated html_tailwind, use the .mp4 path so the plugin writes video there
@@ -125,7 +125,7 @@ export const imagePluginAgent = async (namedInputs) => {
         MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, true);
         const studioBeat = context.studio.beats[index];
         const beatDuration = beat.duration ?? studioBeat?.duration;
-        const processorParams = { beat, context, imagePath: effectiveImagePath, imageRefs, beatDuration, ...htmlStyle(context, beat) };
+        const processorParams = { beat, context, imagePath: effectiveImagePath, imageRefs, movieRefs, beatDuration, ...htmlStyle(context, beat) };
         await plugin.process(processorParams);
         MulmoStudioContextMethods.setBeatSessionState(context, "image", index, beat.id, false);
     }

package/lib/actions/image_references.d.ts

@@ -6,4 +6,10 @@ export declare const generateReferenceImage: (inputs: {
     image: MulmoImagePromptMedia;
     force?: boolean;
 }) => Promise<string>;
+export type MediaRefs = {
+    imageRefs: Record<string, string>;
+    movieRefs: Record<string, string>;
+};
+export declare const getMediaRefs: (context: MulmoStudioContext) => Promise<MediaRefs>;
+/** @deprecated Use getMediaRefs instead */
 export declare const getImageRefs: (context: MulmoStudioContext) => Promise<Record<string, string>>;

package/lib/actions/image_references.js

@@ -51,12 +51,13 @@ export const generateReferenceImage = async (inputs) => {
         });
     }
 };
-export const getImageRefs = async (context) => {
+export const getMediaRefs = async (context) => {
     const images = context.presentationStyle.imageParams?.images;
     if (!images) {
-        return {};
+        return { imageRefs: {}, movieRefs: {} };
     }
     const imageRefs = {};
+    const movieRefs = {};
     await Promise.all(Object.keys(images)
         .sort()
         .map(async (key, index) => {
@@ -67,6 +68,17 @@ export const getImageRefs = async (context) => {
         else if (image.type === "image") {
             imageRefs[key] = await MulmoMediaSourceMethods.imageReference(image.source, context, key);
         }
+        else if (image.type === "movie") {
+            movieRefs[key] = await resolveMovieReference(image, context, key);
+        }
     }));
+    return { imageRefs, movieRefs };
+};
+const resolveMovieReference = async (movie, context, key) => {
+    return MulmoMediaSourceMethods.imageReference(movie.source, context, key);
+};
+/** @deprecated Use getMediaRefs instead */
+export const getImageRefs = async (context) => {
+    const { imageRefs } = await getMediaRefs(context);
     return imageRefs;
 };

package/lib/actions/images.d.ts

@@ -7,6 +7,7 @@ export declare const beat_graph_data: {
         context: {};
         htmlImageAgentInfo: {};
         imageRefs: {};
+        movieRefs: {};
         beat: {};
         __mapIndex: {};
         forceMovie: {
@@ -154,6 +155,7 @@ export declare const beat_graph_data: {
                 beat: string;
                 index: string;
                 imageRefs: string;
+                movieRefs: string;
             };
         };
         imagePlugin: {
@@ -164,12 +166,14 @@ export declare const beat_graph_data: {
                 beat: import("../types/type.js").MulmoBeat;
                 index: number;
                 imageRefs?: Record<string, string>;
+                movieRefs?: Record<string, string>;
             }) => Promise<void>;
             inputs: {
                 context: string;
                 beat: string;
                 index: string;
                 imageRefs: string;
+                movieRefs: string;
                 onComplete: string[];
             };
         };

package/lib/actions/images.js

@@ -14,7 +14,7 @@ import { fileCacheAgentFilter } from "../utils/filters.js";
 import { settings2GraphAIConfig } from "../utils/utils.js";
 import { audioCheckerError } from "../utils/error_cause.js";
 import { extractImageFromMovie, ffmpegGetMediaDuration, trimMusic } from "../utils/ffmpeg_utils.js";
-import { getImageRefs } from "./image_references.js";
+import { getMediaRefs } from "./image_references.js";
 import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent } from "./image_agents.js";
 const vanillaAgents = vanilla.default ?? vanilla;
 const imageAgents = {
@@ -52,6 +52,7 @@ export const beat_graph_data = {
         context: {},
         htmlImageAgentInfo: {},
         imageRefs: {},
+        movieRefs: {},
         beat: {},
         __mapIndex: {},
         forceMovie: { value: false },
@@ -66,6 +67,7 @@ export const beat_graph_data = {
                 beat: ":beat",
                 index: ":__mapIndex",
                 imageRefs: ":imageRefs",
+                movieRefs: ":movieRefs",
             },
         },
         imagePlugin: {
@@ -77,6 +79,7 @@ export const beat_graph_data = {
                 beat: ":beat",
                 index: ":__mapIndex",
                 imageRefs: ":imageRefs",
+                movieRefs: ":movieRefs",
                 onComplete: [":preprocessor"],
             },
         },
@@ -334,6 +337,7 @@ export const images_graph_data = {
         htmlImageAgentInfo: {},
         outputStudioFilePath: {},
         imageRefs: {},
+        movieRefs: {},
         map: {
             agent: "mapAgent",
             inputs: {
@@ -341,6 +345,7 @@ export const images_graph_data = {
                 context: ":context",
                 htmlImageAgentInfo: ":htmlImageAgentInfo",
                 imageRefs: ":imageRefs",
+                movieRefs: ":movieRefs",
             },
             isResult: true,
             params: {
@@ -436,13 +441,14 @@ const prepareGenerateImages = async (context) => {
     mkdir(imageProjectDirPath);
     const provider = MulmoPresentationStyleMethods.getText2ImageProvider(context.presentationStyle.imageParams?.provider);
     const htmlImageAgentInfo = MulmoPresentationStyleMethods.getHtmlImageAgentInfo(context.presentationStyle);
-    const imageRefs = await getImageRefs(context);
+    const { imageRefs, movieRefs } = await getMediaRefs(context);
     GraphAILogger.info(`text2image: provider=${provider} model=${context.presentationStyle.imageParams?.model}`);
     const injections = {
         context,
         htmlImageAgentInfo,
         outputStudioFilePath: getOutputStudioFilePath(outDirPath, fileName),
         imageRefs,
+        movieRefs,
     };
     return injections;
 };

package/lib/agents/combine_audio_files_agent.js

@@ -44,10 +44,11 @@ const getMediaDurationsOfAllBeats = (context) => {
         const beat = context.studio.script.beats[index];
         const { duration: movieDuration, hasAudio: hasMovieAudio } = await getMovieDuration(context, beat);
         const audioDuration = studioBeat.audioFile ? (await ffmpegGetMediaDuration(studioBeat.audioFile)).duration : 0;
+        const hasMoviePrompt = Boolean(beat.moviePrompt);
         return {
             movieDuration,
             audioDuration,
-            hasMedia: movieDuration + audioDuration > 0,
+            hasMedia: movieDuration + audioDuration > 0 || hasMoviePrompt,
             silenceDuration: 0,
             hasMovieAudio,
         };

package/lib/types/schema.d.ts

@@ -276,6 +276,19 @@ export declare const mulmoImageMediaSchema: z.ZodObject<{
         path: z.ZodString;
     }, z.core.$strict>], "kind">;
 }, z.core.$strict>;
+export declare const mulmoMovieMediaSchema: z.ZodObject<{
+    type: z.ZodLiteral<"movie">;
+    source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodURL;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"base64">;
+        data: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+    }, z.core.$strict>], "kind">;
+}, z.core.$strict>;
 export declare const mulmoTextSlideMediaSchema: z.ZodObject<{
     type: z.ZodLiteral<"textSlide">;
     slide: z.ZodObject<{
@@ -2984,6 +2997,18 @@ export declare const mulmoImageParamsImagesValueSchema: z.ZodUnion<readonly [z.Z
         width: z.ZodNumber;
         height: z.ZodNumber;
     }, z.core.$strict>>;
+}, z.core.$strict>, z.ZodObject<{
+    type: z.ZodLiteral<"movie">;
+    source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodURL;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"base64">;
+        data: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+    }, z.core.$strict>], "kind">;
 }, z.core.$strict>]>;
 export declare const mulmoImageParamsImagesSchema: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
     type: z.ZodLiteral<"image">;
@@ -3004,6 +3029,18 @@ export declare const mulmoImageParamsImagesSchema: z.ZodRecord<z.ZodString, z.Zo
         width: z.ZodNumber;
         height: z.ZodNumber;
     }, z.core.$strict>>;
+}, z.core.$strict>, z.ZodObject<{
+    type: z.ZodLiteral<"movie">;
+    source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodURL;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"base64">;
+        data: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+    }, z.core.$strict>], "kind">;
 }, z.core.$strict>]>>;
 export declare const mulmoFillOptionSchema: z.ZodObject<{
     style: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
@@ -3076,6 +3113,18 @@ export declare const mulmoImageParamsSchema: z.ZodObject<{
             width: z.ZodNumber;
             height: z.ZodNumber;
         }, z.core.$strict>>;
+    }, z.core.$strict>, z.ZodObject<{
+        type: z.ZodLiteral<"movie">;
+        source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodURL;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"base64">;
+            data: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+        }, z.core.$strict>], "kind">;
     }, z.core.$strict>]>>>;
     backgroundImage: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
         source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -6429,6 +6478,18 @@ export declare const mulmoPresentationStyleSchema: z.ZodObject<{
                 width: z.ZodNumber;
                 height: z.ZodNumber;
             }, z.core.$strict>>;
+        }, z.core.$strict>, z.ZodObject<{
+            type: z.ZodLiteral<"movie">;
+            source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"url">;
+                url: z.ZodURL;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"base64">;
+                data: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"path">;
+                path: z.ZodString;
+            }, z.core.$strict>], "kind">;
         }, z.core.$strict>]>>>;
         backgroundImage: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
             source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -6889,6 +6950,18 @@ export declare const mulmoScriptSchema: z.ZodObject<{
                 width: z.ZodNumber;
                 height: z.ZodNumber;
             }, z.core.$strict>>;
+        }, z.core.$strict>, z.ZodObject<{
+            type: z.ZodLiteral<"movie">;
+            source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"url">;
+                url: z.ZodURL;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"base64">;
+                data: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"path">;
+                path: z.ZodString;
+            }, z.core.$strict>], "kind">;
         }, z.core.$strict>]>>>;
         backgroundImage: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
             source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -10312,6 +10385,18 @@ export declare const mulmoStudioSchema: z.ZodObject<{
                     width: z.ZodNumber;
                     height: z.ZodNumber;
                 }, z.core.$strict>>;
+            }, z.core.$strict>, z.ZodObject<{
+                type: z.ZodLiteral<"movie">;
+                source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"base64">;
+                    data: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>], "kind">;
             }, z.core.$strict>]>>>;
             backgroundImage: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -13671,6 +13756,18 @@ export declare const mulmoPromptTemplateSchema: z.ZodObject<{
                     width: z.ZodNumber;
                     height: z.ZodNumber;
                 }, z.core.$strict>>;
+            }, z.core.$strict>, z.ZodObject<{
+                type: z.ZodLiteral<"movie">;
+                source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"base64">;
+                    data: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>], "kind">;
             }, z.core.$strict>]>>>;
             backgroundImage: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -14125,6 +14222,18 @@ export declare const mulmoPromptTemplateFileSchema: z.ZodObject<{
                     width: z.ZodNumber;
                     height: z.ZodNumber;
                 }, z.core.$strict>>;
+            }, z.core.$strict>, z.ZodObject<{
+                type: z.ZodLiteral<"movie">;
+                source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"base64">;
+                    data: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>], "kind">;
             }, z.core.$strict>]>>>;
             backgroundImage: z.ZodOptional<z.ZodNullable<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
                 source: z.ZodDiscriminatedUnion<[z.ZodObject<{

package/lib/types/schema.js

@@ -142,7 +142,7 @@ const mulmoSvgMediaSchema = z
     source: mediaSourceSchema,
 })
     .strict();
-const mulmoMovieMediaSchema = z
+export const mulmoMovieMediaSchema = z
     .object({
     type: z.literal("movie"),
     source: mediaSourceSchema,
@@ -337,7 +337,7 @@ export const mulmoImagePromptMediaSchema = z
     canvasSize: z.object({ width: z.number(), height: z.number() }).strict().optional(),
 })
     .strict();
-export const mulmoImageParamsImagesValueSchema = z.union([mulmoImageMediaSchema, mulmoImagePromptMediaSchema]);
+export const mulmoImageParamsImagesValueSchema = z.union([mulmoImageMediaSchema, mulmoImagePromptMediaSchema, mulmoMovieMediaSchema]);
 export const mulmoImageParamsImagesSchema = z.record(imageIdSchema, mulmoImageParamsImagesValueSchema);
 export const mulmoFillOptionSchema = z
     .object({

package/lib/types/type.d.ts

@@ -1,5 +1,5 @@
 import { type CallbackFunction } from "graphai";
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoMovieMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "./const.js";
 import type { LLM } from "./provider2agent.js";
 import { z } from "zod";
@@ -55,6 +55,7 @@ export type MulmoImageAsset = z.infer<typeof mulmoImageAssetSchema>;
 export type MulmoTextSlideMedia = z.infer<typeof mulmoTextSlideMediaSchema>;
 export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
+export type MulmoMovieMedia = z.infer<typeof mulmoMovieMediaSchema>;
 export type MulmoChartMedia = z.infer<typeof mulmoChartMediaSchema>;
 export type MulmoMermaidMedia = z.infer<typeof mulmoMermaidMediaSchema>;
 export type MulmoSessionState = z.infer<typeof mulmoSessionStateSchema>;
@@ -84,6 +85,7 @@ export type ImageProcessorParams = {
     textSlideStyle: string;
     canvasSize: MulmoCanvasDimension;
     imageRefs?: Record<string, string>;
+    movieRefs?: Record<string, string>;
     beatDuration?: number;
 };
 export type PDFMode = (typeof pdf_modes)[number];

package/lib/utils/context.d.ts

@@ -66,6 +66,18 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                     kind: "path";
                     path: string;
                 };
+            } | {
+                type: "movie";
+                source: {
+                    kind: "url";
+                    url: string;
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    kind: "path";
+                    path: string;
+                };
             } | {
                 type: "imagePrompt";
                 prompt: string;
@@ -1599,6 +1611,18 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                     kind: "path";
                     path: string;
                 };
+            } | {
+                type: "movie";
+                source: {
+                    kind: "url";
+                    url: string;
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    kind: "path";
+                    path: string;
+                };
             } | {
                 type: "markdown";
                 markdown: string | string[] | ({
@@ -1653,18 +1677,6 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                     kind: "path";
                     path: string;
                 };
-            } | {
-                type: "movie";
-                source: {
-                    kind: "url";
-                    url: string;
-                } | {
-                    kind: "base64";
-                    data: string;
-                } | {
-                    kind: "path";
-                    path: string;
-                };
             } | {
                 type: "textSlide";
                 slide: {
@@ -2166,6 +2178,18 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         kind: "path";
                         path: string;
                     };
+                } | {
+                    type: "movie";
+                    source: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    };
                 } | {
                     type: "imagePrompt";
                     prompt: string;
@@ -3699,6 +3723,18 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         kind: "path";
                         path: string;
                     };
+                } | {
+                    type: "movie";
+                    source: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    };
                 } | {
                     type: "markdown";
                     markdown: string | string[] | ({
@@ -3753,18 +3789,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         kind: "path";
                         path: string;
                     };
-                } | {
-                    type: "movie";
-                    source: {
-                        kind: "url";
-                        url: string;
-                    } | {
-                        kind: "base64";
-                        data: string;
-                    } | {
-                        kind: "path";
-                        path: string;
-                    };
                 } | {
                     type: "textSlide";
                     slide: {
@@ -4273,6 +4297,18 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     kind: "path";
                     path: string;
                 };
+            } | {
+                type: "movie";
+                source: {
+                    kind: "url";
+                    url: string;
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    kind: "path";
+                    path: string;
+                };
             } | {
                 type: "imagePrompt";
                 prompt: string;

package/lib/utils/html_render.js

@@ -4,7 +4,58 @@ import nodePath from "node:path";
 import crypto from "node:crypto";
 import { marked } from "marked";
 import puppeteer from "puppeteer";
+import { GraphAILogger } from "graphai";
 const isCI = process.env.CI === "true";
+const VIDEO_LOAD_TIMEOUT_MS = 15000;
+const VIDEO_SEEK_TIMEOUT_MS = 3000;
+/** Wait for all <video> elements on the page to be ready for playback */
+const waitForVideosReady = async (page) => {
+    const hasVideos = await page.evaluate(() => document.querySelectorAll("video").length > 0);
+    if (!hasVideos)
+        return;
+    GraphAILogger.info("Waiting for video elements to load...");
+    await page.evaluate((timeout_ms) => {
+        const videos = Array.from(document.querySelectorAll("video"));
+        const pending = videos.filter((v) => v.readyState < 3);
+        if (pending.length === 0)
+            return Promise.resolve();
+        /* eslint-disable sonarjs/no-nested-functions -- inside page.evaluate serialization boundary */
+        return new Promise((resolve) => {
+            let remaining = pending.length;
+            const timer = setTimeout(() => resolve(), timeout_ms);
+            pending.forEach((v) => v.addEventListener("canplaythrough", () => {
+                remaining--;
+                if (remaining <= 0) {
+                    clearTimeout(timer);
+                    resolve();
+                }
+            }, { once: true }));
+        });
+        /* eslint-enable sonarjs/no-nested-functions */
+    }, VIDEO_LOAD_TIMEOUT_MS);
+};
+/** Seek all <video> elements to the specified frame time and wait for seek to complete */
+const syncVideosToFrame = async (page, frameIndex, fps) => {
+    const time = frameIndex / fps;
+    await page.evaluate((seekTime, seekTimeout) => {
+        const videos = Array.from(document.querySelectorAll("video"));
+        if (videos.length === 0)
+            return;
+        videos.forEach((v) => {
+            v.pause();
+            v.currentTime = seekTime;
+        });
+        /* eslint-disable sonarjs/no-nested-functions -- unavoidable inside page.evaluate serialization boundary */
+        return Promise.all(videos.map((v) => new Promise((r) => {
+            const timer = setTimeout(() => r(), seekTimeout);
+            v.addEventListener("seeked", () => {
+                clearTimeout(timer);
+                r();
+            }, { once: true });
+        })));
+        /* eslint-enable sonarjs/no-nested-functions */
+    }, time, VIDEO_SEEK_TIMEOUT_MS);
+};
 /** Scale the page content so it fits inside the viewport without overflow */
 const scaleContentToFit = async (page, viewportWidth, viewportHeight) => {
     await page.evaluate(({ targetWidth, targetHeight }) => {
@@ -115,9 +166,11 @@ export const renderHTMLToFrames = async (html, outputDir, width, height, totalFr
         await page.addStyleTag({ content: "html{height:100%;margin:0;padding:0;overflow:hidden}" });
         // Scale content to fit viewport (same logic as renderHTMLToImage)
         await scaleContentToFit(page, width, height);
+        await waitForVideosReady(page);
         const framePaths = [];
         for (let frame = 0; frame < totalFrames; frame++) {
             // Update frame state and call render() — await in case it returns a Promise
+            // Update frame state and call render()
             await page.evaluate(async ({ frameIndex, totalFrameCount, framesPerSecond }) => {
                 const mulmoWindow = window;
                 mulmoWindow.__MULMO.frame = frameIndex;
@@ -125,6 +178,8 @@ export const renderHTMLToFrames = async (html, outputDir, width, height, totalFr
                     await mulmoWindow.render(frameIndex, totalFrameCount, framesPerSecond);
                 }
             }, { frameIndex: frame, totalFrameCount: totalFrames, framesPerSecond: fps });
+            // Sync all <video> elements to the current frame time
+            await syncVideosToFrame(page, frame, fps);
             const framePath = nodePath.join(outputDir, `frame_${String(frame).padStart(5, "0")}.png`);
             await page.screenshot({ path: framePath });
             framePaths.push(framePath);
@@ -151,6 +206,16 @@ export const renderHTMLToVideo = async (html, videoPath, width, height, totalFra
         await page.setViewport({ width, height });
         await page.addStyleTag({ content: "html{height:100%;margin:0;padding:0;overflow:hidden}" });
         await scaleContentToFit(page, width, height);
+        await waitForVideosReady(page);
+        // Reset all videos to start and begin playback
+        await page.evaluate(() => {
+            const videos = Array.from(document.querySelectorAll("video"));
+            return Promise.all(videos.map((v) => {
+                v.muted = true;
+                v.currentTime = 0;
+                return v.play().catch(() => { });
+            }));
+        });
         const recorder = await page.screencast({
             path: videoPath,
             format: "mp4",

package/lib/utils/image_plugins/html_tailwind.d.ts

@@ -5,6 +5,11 @@ export declare const imageType = "html_tailwind";
  * e.g., src="image:bg_office" → src="file:///abs/path/to/bg_office.png"
  */
 export declare const resolveImageRefs: (html: string, imageRefs: Record<string, string>) => string;
+/**
+ * Resolve movie:name references to file:// absolute paths using movieRefs.
+ * e.g., src="movie:office_pan" → src="file:///abs/path/to/office_pan.mp4"
+ */
+export declare const resolveMovieRefs: (html: string, movieRefs: Record<string, string>) => string;
 /**
  * Resolve relative paths in src attributes to file:// absolute paths.
  * Paths starting with http://, https://, file://, data:, image:, or / are left unchanged.

package/lib/utils/image_plugins/html_tailwind.js

@@ -20,6 +20,19 @@ export const resolveImageRefs = (html, imageRefs) => {
         return `${prefix}${quote}file://${resolvedPath}${quote}`;
     });
 };
+/**
+ * Resolve movie:name references to file:// absolute paths using movieRefs.
+ * e.g., src="movie:office_pan" → src="file:///abs/path/to/office_pan.mp4"
+ */
+export const resolveMovieRefs = (html, movieRefs) => {
+    return html.replace(/(\bsrc\s*=\s*)(["'])movie:([^"']+)\2/gi, (match, prefix, quote, name) => {
+        const resolvedPath = movieRefs[name];
+        if (!resolvedPath) {
+            return match;
+        }
+        return `${prefix}${quote}file://${resolvedPath}${quote}`;
+    });
+};
 /**
  * Resolve relative paths in src attributes to file:// absolute paths.
  * Paths starting with http://, https://, file://, data:, image:, or / are left unchanged.
@@ -100,8 +113,9 @@ const processHtmlTailwindAnimated = async (params) => {
         fps: String(fps),
         custom_style: "",
     });
-    const resolvedRefs = resolveImageRefs(rawHtmlData, params.imageRefs ?? {});
-    const htmlData = resolveRelativeImagePaths(resolvedRefs, context.fileDirs.mulmoFileDirPath);
+    const resolvedImageRefs = resolveImageRefs(rawHtmlData, params.imageRefs ?? {});
+    const resolvedAllRefs = resolveMovieRefs(resolvedImageRefs, params.movieRefs ?? {});
+    const htmlData = resolveRelativeImagePaths(resolvedAllRefs, context.fileDirs.mulmoFileDirPath);
     // imagePath is set to the .mp4 path by imagePluginAgent for animated beats
     const videoPath = imagePath;
     if (animConfig.movie) {
@@ -133,8 +147,9 @@ const processHtmlTailwindStatic = async (params) => {
         html_body: html,
         user_script: buildUserScript(script),
     });
-    const resolvedRefs = resolveImageRefs(rawHtmlData, params.imageRefs ?? {});
-    const htmlData = resolveRelativeImagePaths(resolvedRefs, context.fileDirs.mulmoFileDirPath);
+    const resolvedImageRefs = resolveImageRefs(rawHtmlData, params.imageRefs ?? {});
+    const resolvedAllRefs = resolveMovieRefs(resolvedImageRefs, params.movieRefs ?? {});
+    const htmlData = resolveRelativeImagePaths(resolvedAllRefs, context.fileDirs.mulmoFileDirPath);
     await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
     return imagePath;
 };

package/lib/utils/image_plugins/markdown.js

@@ -3,6 +3,7 @@ import { renderHTMLToImage, interpolate } from "../html_render.js";
 import { parrotingImagePath } from "./utils.js";
 import { resolveCombinedStyle } from "./bg_image_util.js";
 import { generateLayoutHtml, layoutToMarkdown, toMarkdownString, parseMarkdown } from "./markdown_layout.js";
+import { resolveImageRefs, resolveMovieRefs } from "./html_tailwind.js";
 import { isObject } from "graphai";
 export const imageType = "markdown";
 // Type guard for object (data) format
@@ -60,7 +61,9 @@ const processMarkdown = async (params) => {
     const { beat, imagePath, canvasSize } = params;
     if (!beat.image || beat.image.type !== imageType)
         return;
-    const html = await generateHtml(params);
+    const rawHtml = await generateHtml(params);
+    const resolvedImages = resolveImageRefs(rawHtml, params.imageRefs ?? {});
+    const html = resolveMovieRefs(resolvedImages, params.movieRefs ?? {});
     const hasMermaid = containsMermaid(beat.image.markdown);
     await renderHTMLToImage(html, imagePath, canvasSize.width, canvasSize.height, hasMermaid);
     return imagePath;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.6.0",
+  "version": "2.6.2",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",

package/scripts/test/test_markdown_image_refs.json

@@ -0,0 +1,59 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "lang": "en",
+  "canvasSize": { "width": 1920, "height": 1080 },
+  "title": "Markdown image:refs test",
+  "audioParams": {
+    "padding": 0,
+    "introPadding": 0,
+    "closingPadding": 0,
+    "outroPadding": 0
+  },
+  "imageParams": {
+    "images": {
+      "qaLandscape": {
+        "type": "image",
+        "source": {
+          "kind": "path",
+          "path": "images/qa_landscape.jpg"
+        }
+      },
+      "qaPortrait": {
+        "type": "image",
+        "source": {
+          "kind": "path",
+          "path": "images/qa_portrait.png"
+        }
+      }
+    }
+  },
+  "beats": [
+    {
+      "id": "markdown_text_image_ref",
+      "duration": 3,
+      "image": {
+        "type": "markdown",
+        "markdown": [
+          "# Markdown image:refs test",
+          "",
+          "![Landscape photo](image:qaLandscape)",
+          "",
+          "This image is resolved from `imageParams.images` via the `image:` scheme."
+        ]
+      }
+    },
+    {
+      "id": "markdown_layout_image_ref",
+      "duration": 3,
+      "image": {
+        "type": "markdown",
+        "markdown": {
+          "row-2": [
+            ["# Landscape", "![Landscape](image:qaLandscape)"],
+            ["# Portrait", "![Portrait](image:qaPortrait)"]
+          ]
+        }
+      }
+    }
+  ]
+}

package/scripts/test/test_movie_refs.json

@@ -0,0 +1,126 @@
+{
+  "$mulmocast": { "version": "1.1" },
+  "lang": "ja",
+  "canvasSize": { "width": 1080, "height": 1920 },
+  "title": "movie: スキームテスト",
+  "speechParams": {
+    "provider": "kotodama",
+    "speakers": {
+      "Presenter": { "provider": "kotodama", "voiceId": "jikkyo_baby" }
+    }
+  },
+  "imageParams": {
+    "images": {
+      "sample_video": {
+        "type": "movie",
+        "source": {
+          "kind": "path",
+          "path": "../../test/assets/hello.mp4"
+        }
+      }
+    }
+  },
+  "beats": [
+    {
+      "text": "Beat1。動画を全画面背景として表示するパターンです。テキストオーバーレイ付き。",
+      "speaker": "Presenter",
+      "image": {
+        "type": "html_tailwind",
+        "html": [
+          "<div class='h-full w-full overflow-hidden relative bg-black'>",
+          "  <div style='position:absolute;inset:0;overflow:hidden'>",
+          "    <video src='movie:sample_video' autoplay muted loop style='width:100%;height:100%;object-fit:cover;filter:brightness(0.7)'></video>",
+          "  </div>",
+          "  <div style='position:absolute;top:50%;left:40px;right:40px;transform:translateY(-50%);text-align:center'>",
+          "    <div style='display:inline-block;background:rgba(59,130,246,0.85);padding:12px 32px;border-radius:12px'>",
+          "      <span style='color:white;font-size:80px;font-weight:900'>全画面背景</span>",
+          "    </div>",
+          "    <div style='color:white;font-size:48px;font-weight:900;margin-top:20px;text-shadow:0 4px 16px rgba(0,0,0,0.9)'>動画をフルスクリーン表示</div>",
+          "  </div>",
+          "</div>"
+        ],
+        "animation": { "movie": true }
+      }
+    },
+    {
+      "text": "Beat2。画面の一部にだけ動画を表示するパターン。上半分に動画、下半分にテキストカードを置いています。",
+      "speaker": "Presenter",
+      "image": {
+        "type": "html_tailwind",
+        "html": [
+          "<div class='h-full w-full flex flex-col bg-gray-900'>",
+          "  <div style='flex:1;overflow:hidden;border-bottom:4px solid #3B82F6'>",
+          "    <video src='movie:sample_video' autoplay muted loop style='width:100%;height:100%;object-fit:cover'></video>",
+          "  </div>",
+          "  <div style='flex:1;display:flex;align-items:center;justify-content:center;padding:40px'>",
+          "    <div style='text-align:center'>",
+          "      <div style='color:#F59E0B;font-size:64px;font-weight:900;margin-bottom:16px'>上半分に動画</div>",
+          "      <div style='color:white;font-size:44px;line-height:1.5'>下半分はテキストカード。<br>動画は画面の一部にだけ表示。</div>",
+          "    </div>",
+          "  </div>",
+          "</div>"
+        ],
+        "animation": { "movie": true }
+      }
+    },
+    {
+      "text": "Beat3。角丸の小さなウィンドウで動画をピクチャーインピクチャー風に表示するパターンです。",
+      "speaker": "Presenter",
+      "image": {
+        "type": "html_tailwind",
+        "html": [
+          "<div class='h-full w-full relative' style='background:linear-gradient(135deg,#1E293B 0%,#0F172A 100%)'>",
+          "  <div style='position:absolute;top:80px;right:40px;width:400px;height:300px;border-radius:24px;overflow:hidden;border:3px solid rgba(255,255,255,0.3);box-shadow:0 8px 32px rgba(0,0,0,0.5)'>",
+          "    <video src='movie:sample_video' autoplay muted loop style='width:100%;height:100%;object-fit:cover'></video>",
+          "  </div>",
+          "  <div style='position:absolute;top:50%;left:40px;right:40px;transform:translateY(-50%);text-align:left'>",
+          "    <div style='color:#60A5FA;font-size:40px;font-weight:700;margin-bottom:12px'>PiP スタイル</div>",
+          "    <div style='color:white;font-size:72px;font-weight:900;line-height:1.2'>右上に小さな<br>動画ウィンドウ</div>",
+          "    <div style='color:rgba(255,255,255,0.6);font-size:36px;margin-top:20px'>角丸 + ボーダー + シャドウで浮遊感</div>",
+          "  </div>",
+          "</div>"
+        ],
+        "animation": { "movie": true }
+      }
+    },
+    {
+      "text": "Beat4。円形にマスクした動画を中央に配置するパターン。プロフィール動画やアバター風の見せ方です。",
+      "speaker": "Presenter",
+      "image": {
+        "type": "html_tailwind",
+        "html": [
+          "<div class='h-full w-full flex flex-col items-center justify-center' style='background:linear-gradient(180deg,#312E81 0%,#1E1B4B 100%)'>",
+          "  <div style='width:500px;height:500px;border-radius:50%;overflow:hidden;border:6px solid #A78BFA;box-shadow:0 0 60px rgba(167,139,250,0.4)'>",
+          "    <video src='movie:sample_video' autoplay muted loop style='width:100%;height:100%;object-fit:cover'></video>",
+          "  </div>",
+          "  <div style='color:white;font-size:64px;font-weight:900;margin-top:40px'>円形マスク</div>",
+          "  <div style='color:#C4B5FD;font-size:40px;margin-top:12px'>アバター / プロフィール風</div>",
+          "</div>"
+        ],
+        "animation": { "movie": true }
+      }
+    },
+    {
+      "text": "Beat5。左右分割で、左に動画、右にテキストを並べるスプリットレイアウトです。",
+      "speaker": "Presenter",
+      "image": {
+        "type": "html_tailwind",
+        "html": [
+          "<div class='h-full w-full flex flex-row bg-black'>",
+          "  <div style='flex:1;overflow:hidden'>",
+          "    <video src='movie:sample_video' autoplay muted loop style='width:100%;height:100%;object-fit:cover'></video>",
+          "  </div>",
+          "  <div style='flex:1;display:flex;align-items:center;justify-content:center;padding:32px;background:linear-gradient(180deg,#1E3A5F 0%,#0D1B2A 100%)'>",
+          "    <div style='text-align:center'>",
+          "      <div style='color:#38BDF8;font-size:56px;font-weight:900;margin-bottom:20px'>左右分割</div>",
+          "      <div style='color:white;font-size:40px;line-height:1.5'>左半分が動画<br>右半分がテキスト</div>",
+          "      <div style='margin-top:24px;width:120px;height:4px;background:#38BDF8;border-radius:2px;margin-left:auto;margin-right:auto'></div>",
+          "    </div>",
+          "  </div>",
+          "</div>"
+        ],
+        "animation": { "movie": true }
+      }
+    }
+  ]
+}