mulmocast 0.0.20 → 0.0.22

package/lib/actions/images.d.ts

@@ -31,6 +31,10 @@ export declare const imagePreprocessAgent: (namedInputs: {
     movieFile: string | undefined;
     imagePath: string | undefined;
     referenceImage: string | undefined;
+    htmlPrompt?: undefined;
+} | {
+    imagePath: string;
+    htmlPrompt: string;
 } | {
     imagePath: string;
     images: string[];
@@ -57,6 +61,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
         }> | undefined;
     };
     movieFile: string | undefined;
+    htmlPrompt?: undefined;
 } | {
     images: string[];
     imageParams: {
@@ -84,6 +89,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     imagePath: string;
     referenceImage: string;
     prompt: string;
+    htmlPrompt?: undefined;
 }>;
 export declare const imagePluginAgent: (namedInputs: {
     context: MulmoStudioContext;

package/lib/actions/images.js

@@ -3,6 +3,7 @@ import fs from "fs";
 import { GraphAI, GraphAILogger } from "graphai";
 import { TaskManager } from "graphai/lib/task_manager.js";
 import * as agents from "@graphai/vanilla";
+import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { getOutputStudioFilePath, getBeatPngImagePath, getBeatMoviePath, getReferenceImagePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
@@ -11,6 +12,7 @@ import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../met
 import { findImagePlugin } from "../utils/image_plugins/index.js";
 import { imagePrompt } from "../utils/prompt.js";
 import { defaultOpenAIImageModel } from "../utils/const.js";
+import { renderHTMLToImage } from "../utils/markdown.js";
 const vanillaAgents = agents.default ?? agents;
 dotenv.config();
 // const openai = new OpenAI();
@@ -39,6 +41,10 @@ export const imagePreprocessAgent = async (namedInputs) => {
         // undefined prompt indicates that image generation is not needed
         return { imagePath: path, referenceImage: path, ...returnValue };
     }
+    if (beat.htmlPrompt) {
+        const htmlPrompt = beat.htmlPrompt.prompt + (beat.htmlPrompt.data ? "\n\n data\n" + JSON.stringify(beat.htmlPrompt.data, null, 2) : "");
+        return { imagePath, htmlPrompt };
+    }
     // images for "edit_image"
     const images = (() => {
         const imageNames = beat.imageNames ?? Object.keys(imageRefs); // use all images if imageNames is not specified
@@ -69,6 +75,10 @@ export const imagePluginAgent = async (namedInputs) => {
         throw error;
     }
 };
+const htmlImageGeneratorAgent = async (namedInputs) => {
+    const { html, file, canvasSize } = namedInputs;
+    await renderHTMLToImage(html, file, canvasSize.width, canvasSize.height);
+};
 const beat_graph_data = {
     version: 0.5,
     concurrency: 4,
@@ -100,15 +110,43 @@ const beat_graph_data = {
                 onComplete: ":preprocessor",
             },
         },
+        htmlImageAgent: {
+            if: ":preprocessor.htmlPrompt",
+            defaultValue: {},
+            agent: "openAIAgent",
+            inputs: {
+                prompt: ":preprocessor.htmlPrompt",
+                system: [
+                    "Based on the provided information, create a single slide HTML page using Tailwind CSS.",
+                    "If charts are needed, use Chart.js to present them in a clean and visually appealing way.",
+                    "Include a balanced mix of comments, graphs, and illustrations to enhance visual impact.",
+                    "Output only the HTML code. Do not include any comments, explanations, or additional information outside the HTML.",
+                    "If data is provided, use it effectively to populate the slide.",
+                ],
+            },
+        },
+        htmlImageGenerator: {
+            if: ":preprocessor.htmlPrompt",
+            defaultValue: {},
+            agent: htmlImageGeneratorAgent,
+            // console: { before: true, after: true },
+            inputs: {
+                html: ":htmlImageAgent.text.codeBlock()",
+                canvasSize: ":context.presentationStyle.canvasSize",
+                file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
+                mulmoContext: ":context", // for fileCacheAgentFilter
+                index: ":__mapIndex", // for fileCacheAgentFilter
+                sessionType: "image", // for fileCacheAgentFilter
+            },
+        },
         imageGenerator: {
             if: ":preprocessor.prompt",
             agent: ":imageAgentInfo.agent",
-            retry: 3,
+            retry: 2,
             inputs: {
                 prompt: ":preprocessor.prompt",
                 images: ":preprocessor.images",
                 file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
-                text: ":preprocessor.prompt", // only for fileCacheAgentFilter
                 force: ":context.force", // only for fileCacheAgentFilter
                 mulmoContext: ":context", // for fileCacheAgentFilter
                 index: ":__mapIndex", // for fileCacheAgentFilter
@@ -157,7 +195,7 @@ const beat_graph_data = {
         output: {
             agent: "copyAgent",
             inputs: {
-                onComplete: ":imageFromMovie", // to wait for imageFromMovie to finish
+                onComplete: [":imageFromMovie", ":htmlImageGenerator"], // to wait for imageFromMovie to finish
                 imageFile: ":preprocessor.imagePath",
                 movieFile: ":preprocessor.movieFile",
             },
@@ -258,7 +296,7 @@ const graphOption = async (context) => {
         {
             name: "fileCacheAgentFilter",
             agent: fileCacheAgentFilter,
-            nodeIds: ["imageGenerator", "movieGenerator"],
+            nodeIds: ["imageGenerator", "movieGenerator", "htmlImageGenerator"],
         },
     ];
     const taskManager = new TaskManager(getConcurrency(context));
@@ -358,7 +396,7 @@ const getConcurrency = (context) => {
 const generateImages = async (context, callbacks) => {
     const options = await graphOption(context);
     const injections = await prepareGenerateImages(context);
-    const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
+    const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent, openAIAgent }, options);
     Object.keys(injections).forEach((key) => {
         graph.injectValue(key, injections[key]);
     });
@@ -385,7 +423,7 @@ export const images = async (context, callbacks) => {
 export const generateBeatImage = async (index, context, callbacks) => {
     const options = await graphOption(context);
     const injections = await prepareGenerateImages(context);
-    const graph = new GraphAI(beat_graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
+    const graph = new GraphAI(beat_graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent, openAIAgent }, options);
     Object.keys(injections).forEach((key) => {
         if ("outputStudioFilePath" !== key) {
             graph.injectValue(key, injections[key]);

package/lib/agents/add_bgm_agent.js

@@ -9,7 +9,7 @@ const addBGMAgent = async ({ namedInputs, params, }) => {
     const totalDuration = speechDuration + introPadding + outroPadding;
     GraphAILogger.log("totalDucation:", speechDuration, totalDuration);
     const ffmpegContext = FfmpegContextInit();
-    const musicInputIndex = FfmpegContextAddInput(ffmpegContext, musicFile);
+    const musicInputIndex = FfmpegContextAddInput(ffmpegContext, musicFile, ["-stream_loop", "-1"]);
     const voiceInputIndex = FfmpegContextAddInput(ffmpegContext, voiceFile);
     ffmpegContext.filterComplex.push(`[${musicInputIndex}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, volume=${context.presentationStyle.audioParams.bgmVolume}[music]`);
     ffmpegContext.filterComplex.push(`[${voiceInputIndex}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, volume=${context.presentationStyle.audioParams.audioVolume}, adelay=${introPadding * 1000}|${introPadding * 1000}[voice]`);

package/lib/agents/combine_audio_files_agent.js

@@ -35,7 +35,7 @@ const getMediaDurations = (context) => {
         return {
             movieDuration,
             audioDuration,
-            hasMadia: movieDuration + audioDuration > 0,
+            hasMedia: movieDuration + audioDuration > 0,
             silenceDuration: 0,
         };
     }));
@@ -70,7 +70,7 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
         if (audioDuration > 0) {
             // Check if the current beat has spilled over audio.
             const group = [index];
-            for (let i = index + 1; i < context.studio.beats.length && !mediaDurations[i].hasMadia; i++) {
+            for (let i = index + 1; i < context.studio.beats.length && !mediaDurations[i].hasMedia; i++) {
                 group.push(i);
             }
             if (group.length > 1) {

package/lib/agents/image_openai_agent.js

@@ -1,5 +1,6 @@
 import fs from "fs";
 import path from "path";
+import { GraphAILogger } from "graphai";
 import OpenAI, { toFile } from "openai";
 import { defaultOpenAIImageModel } from "../utils/const.js";
 // https://platform.openai.com/docs/guides/image-generation
@@ -42,17 +43,23 @@ export const imageOpenaiAgent = async ({ namedInputs, params }) => {
         imageOptions.moderation = moderation || "auto";
     }
     const response = await (async () => {
-        const targetSize = imageOptions.size;
-        if ((images ?? []).length > 0 && (targetSize === "1536x1024" || targetSize === "1024x1536" || targetSize === "1024x1024")) {
-            const imagelist = await Promise.all((images ?? []).map(async (file) => {
-                const ext = path.extname(file).toLowerCase();
-                const type = ext === ".jpg" || ext === ".jpeg" ? "image/jpeg" : "image/png";
-                return await toFile(fs.createReadStream(file), null, { type });
-            }));
-            return await openai.images.edit({ ...imageOptions, size: targetSize, image: imagelist });
+        try {
+            const targetSize = imageOptions.size;
+            if ((images ?? []).length > 0 && (targetSize === "1536x1024" || targetSize === "1024x1536" || targetSize === "1024x1024")) {
+                const imagelist = await Promise.all((images ?? []).map(async (file) => {
+                    const ext = path.extname(file).toLowerCase();
+                    const type = ext === ".jpg" || ext === ".jpeg" ? "image/jpeg" : "image/png";
+                    return await toFile(fs.createReadStream(file), null, { type });
+                }));
+                return await openai.images.edit({ ...imageOptions, size: targetSize, image: imagelist });
+            }
+            else {
+                return await openai.images.generate(imageOptions);
+            }
         }
-        else {
-            return await openai.images.generate(imageOptions);
+        catch (error) {
+            GraphAILogger.info("Failed to generate image:", error.message);
+            throw error;
         }
     })();
     if (!response.data) {

package/lib/agents/movie_google_agent.js

@@ -102,7 +102,7 @@ export const movieGoogleAgent = async ({ namedInputs, params, config }) => {
         throw new Error("ERROR: geneateImage returned undefined");
     }
     catch (error) {
-        GraphAILogger.info("Failed to generate movie:", error);
+        GraphAILogger.info("Failed to generate movie:", error.message);
         throw error;
     }
 };

package/lib/index.browser.d.ts

@@ -1,2 +1,3 @@
+export * from "./types/type.js";
 export * from "./types/schema.js";
 export * from "./agents/validate_schema_agent.js";

package/lib/index.browser.js

@@ -1,3 +1,4 @@
 // Entry point that exposes only APIs available for use in the browser
+export * from "./types/type.js";
 export * from "./types/schema.js";
 export * from "./agents/validate_schema_agent.js";

package/lib/methods/mulmo_studio_context.d.ts

@@ -1,17 +1,4 @@
-import { MulmoStudioContext } from "../types/index.js";
-type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
-type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie";
-type SessionProgressEvent = {
-    kind: "session";
-    sessionType: SessionType;
-    inSession: boolean;
-} | {
-    kind: "beat";
-    sessionType: BeatSessionType;
-    index: number;
-    inSession: boolean;
-};
-type SessionProgressCallback = (change: SessionProgressEvent) => void;
+import { BeatSessionType, MulmoStudioContext, SessionProgressCallback, SessionType } from "../types/index.js";
 export declare const addSessionProgressCallback: (cb: SessionProgressCallback) => void;
 export declare const removeSessionProgressCallback: (cb: SessionProgressCallback) => void;
 export declare const MulmoStudioContextMethods: {
@@ -25,4 +12,3 @@ export declare const MulmoStudioContextMethods: {
     setSessionState(context: MulmoStudioContext, sessionType: SessionType, value: boolean): void;
     setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType, index: number, value: boolean): void;
 };
-export {};

package/lib/types/schema.d.ts

@@ -1163,6 +1163,22 @@ export declare const audioParamsSchema: z.ZodObject<{
     bgmVolume?: number | undefined;
     audioVolume?: number | undefined;
 }>;
+export declare const htmlPromptParamsSchema: z.ZodObject<{
+    systemPrompt: z.ZodOptional<z.ZodDefault<z.ZodString>>;
+    prompt: z.ZodDefault<z.ZodString>;
+    data: z.ZodOptional<z.ZodAny>;
+    images: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+}, "strict", z.ZodTypeAny, {
+    prompt: string;
+    data?: any;
+    images?: Record<string, any> | undefined;
+    systemPrompt?: string | undefined;
+}, {
+    data?: any;
+    images?: Record<string, any> | undefined;
+    systemPrompt?: string | undefined;
+    prompt?: string | undefined;
+}>;
 export declare const mulmoBeatSchema: z.ZodObject<{
     speaker: z.ZodDefault<z.ZodString>;
     text: z.ZodDefault<z.ZodString>;
@@ -1823,6 +1839,22 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     imageNames: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     imagePrompt: z.ZodOptional<z.ZodString>;
     moviePrompt: z.ZodOptional<z.ZodString>;
+    htmlPrompt: z.ZodOptional<z.ZodObject<{
+        systemPrompt: z.ZodOptional<z.ZodDefault<z.ZodString>>;
+        prompt: z.ZodDefault<z.ZodString>;
+        data: z.ZodOptional<z.ZodAny>;
+        images: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    }, "strict", z.ZodTypeAny, {
+        prompt: string;
+        data?: any;
+        images?: Record<string, any> | undefined;
+        systemPrompt?: string | undefined;
+    }, {
+        data?: any;
+        images?: Record<string, any> | undefined;
+        systemPrompt?: string | undefined;
+        prompt?: string | undefined;
+    }>>;
 }, "strict", z.ZodTypeAny, {
     text: string;
     speaker: string;
@@ -1984,6 +2016,12 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     imageNames?: string[] | undefined;
     imagePrompt?: string | undefined;
     moviePrompt?: string | undefined;
+    htmlPrompt?: {
+        prompt: string;
+        data?: any;
+        images?: Record<string, any> | undefined;
+        systemPrompt?: string | undefined;
+    } | undefined;
 }, {
     text?: string | undefined;
     duration?: number | undefined;
@@ -2145,6 +2183,12 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     imageNames?: string[] | undefined;
     imagePrompt?: string | undefined;
     moviePrompt?: string | undefined;
+    htmlPrompt?: {
+        data?: any;
+        images?: Record<string, any> | undefined;
+        systemPrompt?: string | undefined;
+        prompt?: string | undefined;
+    } | undefined;
 }>;
 export declare const mulmoCanvasDimensionSchema: z.ZodDefault<z.ZodObject<{
     width: z.ZodNumber;
@@ -3741,6 +3785,22 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         imagePrompt: z.ZodOptional<z.ZodString>;
         moviePrompt: z.ZodOptional<z.ZodString>;
+        htmlPrompt: z.ZodOptional<z.ZodObject<{
+            systemPrompt: z.ZodOptional<z.ZodDefault<z.ZodString>>;
+            prompt: z.ZodDefault<z.ZodString>;
+            data: z.ZodOptional<z.ZodAny>;
+            images: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+        }, "strict", z.ZodTypeAny, {
+            prompt: string;
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+        }, {
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+            prompt?: string | undefined;
+        }>>;
     }, "strict", z.ZodTypeAny, {
         text: string;
         speaker: string;
@@ -3902,6 +3962,12 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames?: string[] | undefined;
         imagePrompt?: string | undefined;
         moviePrompt?: string | undefined;
+        htmlPrompt?: {
+            prompt: string;
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+        } | undefined;
     }, {
         text?: string | undefined;
         duration?: number | undefined;
@@ -4063,6 +4129,12 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames?: string[] | undefined;
         imagePrompt?: string | undefined;
         moviePrompt?: string | undefined;
+        htmlPrompt?: {
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+            prompt?: string | undefined;
+        } | undefined;
     }>, "many">;
     imagePath: z.ZodOptional<z.ZodString>;
     __test_invalid__: z.ZodOptional<z.ZodBoolean>;
@@ -4269,6 +4341,12 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames?: string[] | undefined;
         imagePrompt?: string | undefined;
         moviePrompt?: string | undefined;
+        htmlPrompt?: {
+            prompt: string;
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+        } | undefined;
     }[];
     lang?: string | undefined;
     title?: string | undefined;
@@ -4480,6 +4558,12 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames?: string[] | undefined;
         imagePrompt?: string | undefined;
         moviePrompt?: string | undefined;
+        htmlPrompt?: {
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+            prompt?: string | undefined;
+        } | undefined;
     }[];
     lang?: string | undefined;
     title?: string | undefined;
@@ -5731,6 +5815,22 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
             imagePrompt: z.ZodOptional<z.ZodString>;
             moviePrompt: z.ZodOptional<z.ZodString>;
+            htmlPrompt: z.ZodOptional<z.ZodObject<{
+                systemPrompt: z.ZodOptional<z.ZodDefault<z.ZodString>>;
+                prompt: z.ZodDefault<z.ZodString>;
+                data: z.ZodOptional<z.ZodAny>;
+                images: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+            }, "strict", z.ZodTypeAny, {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            }, {
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+                prompt?: string | undefined;
+            }>>;
         }, "strict", z.ZodTypeAny, {
             text: string;
             speaker: string;
@@ -5892,6 +5992,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
         }, {
             text?: string | undefined;
             duration?: number | undefined;
@@ -6053,6 +6159,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+                prompt?: string | undefined;
+            } | undefined;
         }>, "many">;
         imagePath: z.ZodOptional<z.ZodString>;
         __test_invalid__: z.ZodOptional<z.ZodBoolean>;
@@ -6259,6 +6371,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
@@ -6470,6 +6588,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+                prompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
@@ -6788,6 +6912,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
@@ -7010,6 +7140,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+                prompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
@@ -7569,8 +7705,8 @@ export declare const mulmoScriptTemplateSchema: z.ZodObject<{
     }>>;
 }, "strict", z.ZodTypeAny, {
     title: string;
-    description: string;
     systemPrompt: string;
+    description: string;
     scriptName?: string | undefined;
     presentationStyle?: {
         audioParams: {
@@ -7650,8 +7786,8 @@ export declare const mulmoScriptTemplateSchema: z.ZodObject<{
     } | undefined;
 }, {
     title: string;
-    description: string;
     systemPrompt: string;
+    description: string;
     scriptName?: string | undefined;
     presentationStyle?: {
         $mulmocast: {
@@ -8206,9 +8342,9 @@ export declare const mulmoScriptTemplateFileSchema: z.ZodObject<{
     filename: z.ZodString;
 }, "strict", z.ZodTypeAny, {
     title: string;
+    systemPrompt: string;
     description: string;
     filename: string;
-    systemPrompt: string;
     scriptName?: string | undefined;
     presentationStyle?: {
         audioParams: {
@@ -8288,9 +8424,9 @@ export declare const mulmoScriptTemplateFileSchema: z.ZodObject<{
     } | undefined;
 }, {
     title: string;
+    systemPrompt: string;
     description: string;
     filename: string;
-    systemPrompt: string;
     scriptName?: string | undefined;
     presentationStyle?: {
         $mulmocast: {

package/lib/types/schema.js

@@ -171,6 +171,14 @@ export const audioParamsSchema = z
     audioVolume: z.number().default(1.0).describe("Volume of the audio"),
 })
     .strict();
+export const htmlPromptParamsSchema = z
+    .object({
+    systemPrompt: z.string().default("").optional(),
+    prompt: z.string().default(""),
+    data: z.any().optional(),
+    images: z.record(z.any()).optional(),
+})
+    .strict();
 export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.default("Presenter"),
@@ -187,6 +195,7 @@ export const mulmoBeatSchema = z
     imageNames: z.array(imageIdSchema).optional(), // list of image names to use for image generation. The default is all images in the imageParams.images.
     imagePrompt: z.string().optional(),
     moviePrompt: z.string().optional(),
+    htmlPrompt: htmlPromptParamsSchema.optional(),
 })
     .strict();
 export const mulmoCanvasDimensionSchema = z

package/lib/types/type.d.ts

@@ -79,3 +79,16 @@ export type Text2ImageAgentInfo = {
 };
 export type BeatMediaType = "movie" | "image";
 export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];
+export type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
+export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie";
+export type SessionProgressEvent = {
+    kind: "session";
+    sessionType: SessionType;
+    inSession: boolean;
+} | {
+    kind: "beat";
+    sessionType: BeatSessionType;
+    index: number;
+    inSession: boolean;
+};
+export type SessionProgressCallback = (change: SessionProgressEvent) => void;

package/lib/utils/ffmpeg_utils.d.ts

@@ -5,7 +5,7 @@ export type FfmpegContext = {
     filterComplex: string[];
 };
 export declare const FfmpegContextInit: () => FfmpegContext;
-export declare const FfmpegContextAddInput: (context: FfmpegContext, input: string) => number;
+export declare const FfmpegContextAddInput: (context: FfmpegContext, input: string, inputOptions?: string[]) => number;
 export declare const FfmpegContextPushFormattedAudio: (context: FfmpegContext, sourceId: string, outputId: string, duration?: number | undefined) => void;
 export declare const FfmpegContextInputFormattedAudio: (context: FfmpegContext, input: string, duration?: number | undefined) => string;
 export declare const FfmpegContextGenerateOutput: (context: FfmpegContext, output: string, options?: string[]) => Promise<number>;

package/lib/utils/ffmpeg_utils.js

@@ -7,8 +7,13 @@ export const FfmpegContextInit = () => {
         filterComplex: [],
     };
 };
-export const FfmpegContextAddInput = (context, input) => {
-    context.command.input(input);
+export const FfmpegContextAddInput = (context, input, inputOptions) => {
+    if (inputOptions) {
+        context.command.input(input).inputOptions(inputOptions);
+    }
+    else {
+        context.command.input(input);
+    }
     context.inputCount++;
     return context.inputCount - 1; // returned the index of the input
 };

package/lib/utils/image_plugins/image.d.ts

@@ -1,3 +1,3 @@
 export declare const imageType = "image";
-export declare const process: (params: import("../../index.js").ImageProcessorParams) => string | undefined;
-export declare const path: (params: import("../../index.js").ImageProcessorParams) => string | undefined;
+export declare const process: (params: import("../../index.browser.js").ImageProcessorParams) => string | undefined;
+export declare const path: (params: import("../../index.browser.js").ImageProcessorParams) => string | undefined;

package/lib/utils/image_plugins/movie.d.ts

@@ -1,3 +1,3 @@
 export declare const imageType = "movie";
-export declare const process: (params: import("../../index.js").ImageProcessorParams) => string | undefined;
-export declare const path: (params: import("../../index.js").ImageProcessorParams) => string | undefined;
+export declare const process: (params: import("../../index.browser.js").ImageProcessorParams) => string | undefined;
+export declare const path: (params: import("../../index.browser.js").ImageProcessorParams) => string | undefined;

package/lib/utils/preprocess.d.ts

@@ -211,6 +211,12 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.0.20",
+  "version": "0.0.22",
   "description": "",
   "type": "module",
   "main": "lib/index.js",