mulmocast 0.0.21 → 0.0.22

package/lib/actions/images.d.ts

@@ -31,6 +31,10 @@ export declare const imagePreprocessAgent: (namedInputs: {
     movieFile: string | undefined;
     imagePath: string | undefined;
     referenceImage: string | undefined;
+    htmlPrompt?: undefined;
+} | {
+    imagePath: string;
+    htmlPrompt: string;
 } | {
     imagePath: string;
     images: string[];
@@ -57,6 +61,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
         }> | undefined;
     };
     movieFile: string | undefined;
+    htmlPrompt?: undefined;
 } | {
     images: string[];
     imageParams: {
@@ -84,6 +89,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     imagePath: string;
     referenceImage: string;
     prompt: string;
+    htmlPrompt?: undefined;
 }>;
 export declare const imagePluginAgent: (namedInputs: {
     context: MulmoStudioContext;

package/lib/actions/images.js

@@ -3,6 +3,7 @@ import fs from "fs";
 import { GraphAI, GraphAILogger } from "graphai";
 import { TaskManager } from "graphai/lib/task_manager.js";
 import * as agents from "@graphai/vanilla";
+import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { getOutputStudioFilePath, getBeatPngImagePath, getBeatMoviePath, getReferenceImagePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
@@ -11,6 +12,7 @@ import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../met
 import { findImagePlugin } from "../utils/image_plugins/index.js";
 import { imagePrompt } from "../utils/prompt.js";
 import { defaultOpenAIImageModel } from "../utils/const.js";
+import { renderHTMLToImage } from "../utils/markdown.js";
 const vanillaAgents = agents.default ?? agents;
 dotenv.config();
 // const openai = new OpenAI();
@@ -39,6 +41,10 @@ export const imagePreprocessAgent = async (namedInputs) => {
         // undefined prompt indicates that image generation is not needed
         return { imagePath: path, referenceImage: path, ...returnValue };
     }
+    if (beat.htmlPrompt) {
+        const htmlPrompt = beat.htmlPrompt.prompt + (beat.htmlPrompt.data ? "\n\n data\n" + JSON.stringify(beat.htmlPrompt.data, null, 2) : "");
+        return { imagePath, htmlPrompt };
+    }
     // images for "edit_image"
     const images = (() => {
         const imageNames = beat.imageNames ?? Object.keys(imageRefs); // use all images if imageNames is not specified
@@ -69,6 +75,10 @@ export const imagePluginAgent = async (namedInputs) => {
         throw error;
     }
 };
+const htmlImageGeneratorAgent = async (namedInputs) => {
+    const { html, file, canvasSize } = namedInputs;
+    await renderHTMLToImage(html, file, canvasSize.width, canvasSize.height);
+};
 const beat_graph_data = {
     version: 0.5,
     concurrency: 4,
@@ -100,6 +110,35 @@ const beat_graph_data = {
                 onComplete: ":preprocessor",
             },
         },
+        htmlImageAgent: {
+            if: ":preprocessor.htmlPrompt",
+            defaultValue: {},
+            agent: "openAIAgent",
+            inputs: {
+                prompt: ":preprocessor.htmlPrompt",
+                system: [
+                    "Based on the provided information, create a single slide HTML page using Tailwind CSS.",
+                    "If charts are needed, use Chart.js to present them in a clean and visually appealing way.",
+                    "Include a balanced mix of comments, graphs, and illustrations to enhance visual impact.",
+                    "Output only the HTML code. Do not include any comments, explanations, or additional information outside the HTML.",
+                    "If data is provided, use it effectively to populate the slide.",
+                ],
+            },
+        },
+        htmlImageGenerator: {
+            if: ":preprocessor.htmlPrompt",
+            defaultValue: {},
+            agent: htmlImageGeneratorAgent,
+            // console: { before: true, after: true },
+            inputs: {
+                html: ":htmlImageAgent.text.codeBlock()",
+                canvasSize: ":context.presentationStyle.canvasSize",
+                file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
+                mulmoContext: ":context", // for fileCacheAgentFilter
+                index: ":__mapIndex", // for fileCacheAgentFilter
+                sessionType: "image", // for fileCacheAgentFilter
+            },
+        },
         imageGenerator: {
             if: ":preprocessor.prompt",
             agent: ":imageAgentInfo.agent",
@@ -108,7 +147,6 @@ const beat_graph_data = {
                 prompt: ":preprocessor.prompt",
                 images: ":preprocessor.images",
                 file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
-                text: ":preprocessor.prompt", // only for fileCacheAgentFilter
                 force: ":context.force", // only for fileCacheAgentFilter
                 mulmoContext: ":context", // for fileCacheAgentFilter
                 index: ":__mapIndex", // for fileCacheAgentFilter
@@ -157,7 +195,7 @@ const beat_graph_data = {
         output: {
             agent: "copyAgent",
             inputs: {
-                onComplete: ":imageFromMovie", // to wait for imageFromMovie to finish
+                onComplete: [":imageFromMovie", ":htmlImageGenerator"], // to wait for imageFromMovie to finish
                 imageFile: ":preprocessor.imagePath",
                 movieFile: ":preprocessor.movieFile",
             },
@@ -258,7 +296,7 @@ const graphOption = async (context) => {
         {
             name: "fileCacheAgentFilter",
             agent: fileCacheAgentFilter,
-            nodeIds: ["imageGenerator", "movieGenerator"],
+            nodeIds: ["imageGenerator", "movieGenerator", "htmlImageGenerator"],
         },
     ];
     const taskManager = new TaskManager(getConcurrency(context));
@@ -358,7 +396,7 @@ const getConcurrency = (context) => {
 const generateImages = async (context, callbacks) => {
     const options = await graphOption(context);
     const injections = await prepareGenerateImages(context);
-    const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
+    const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent, openAIAgent }, options);
     Object.keys(injections).forEach((key) => {
         graph.injectValue(key, injections[key]);
     });
@@ -385,7 +423,7 @@ export const images = async (context, callbacks) => {
 export const generateBeatImage = async (index, context, callbacks) => {
     const options = await graphOption(context);
     const injections = await prepareGenerateImages(context);
-    const graph = new GraphAI(beat_graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent }, options);
+    const graph = new GraphAI(beat_graph_data, { ...vanillaAgents, imageGoogleAgent, movieGoogleAgent, imageOpenaiAgent, mediaMockAgent, fileWriteAgent, openAIAgent }, options);
     Object.keys(injections).forEach((key) => {
         if ("outputStudioFilePath" !== key) {
             graph.injectValue(key, injections[key]);

package/lib/agents/add_bgm_agent.js

@@ -9,7 +9,7 @@ const addBGMAgent = async ({ namedInputs, params, }) => {
     const totalDuration = speechDuration + introPadding + outroPadding;
     GraphAILogger.log("totalDucation:", speechDuration, totalDuration);
     const ffmpegContext = FfmpegContextInit();
-    const musicInputIndex = FfmpegContextAddInput(ffmpegContext, musicFile);
+    const musicInputIndex = FfmpegContextAddInput(ffmpegContext, musicFile, ["-stream_loop", "-1"]);
     const voiceInputIndex = FfmpegContextAddInput(ffmpegContext, voiceFile);
     ffmpegContext.filterComplex.push(`[${musicInputIndex}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, volume=${context.presentationStyle.audioParams.bgmVolume}[music]`);
     ffmpegContext.filterComplex.push(`[${voiceInputIndex}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, volume=${context.presentationStyle.audioParams.audioVolume}, adelay=${introPadding * 1000}|${introPadding * 1000}[voice]`);

package/lib/types/schema.d.ts

@@ -1163,6 +1163,22 @@ export declare const audioParamsSchema: z.ZodObject<{
     bgmVolume?: number | undefined;
     audioVolume?: number | undefined;
 }>;
+export declare const htmlPromptParamsSchema: z.ZodObject<{
+    systemPrompt: z.ZodOptional<z.ZodDefault<z.ZodString>>;
+    prompt: z.ZodDefault<z.ZodString>;
+    data: z.ZodOptional<z.ZodAny>;
+    images: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+}, "strict", z.ZodTypeAny, {
+    prompt: string;
+    data?: any;
+    images?: Record<string, any> | undefined;
+    systemPrompt?: string | undefined;
+}, {
+    data?: any;
+    images?: Record<string, any> | undefined;
+    systemPrompt?: string | undefined;
+    prompt?: string | undefined;
+}>;
 export declare const mulmoBeatSchema: z.ZodObject<{
     speaker: z.ZodDefault<z.ZodString>;
     text: z.ZodDefault<z.ZodString>;
@@ -1823,6 +1839,22 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     imageNames: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     imagePrompt: z.ZodOptional<z.ZodString>;
     moviePrompt: z.ZodOptional<z.ZodString>;
+    htmlPrompt: z.ZodOptional<z.ZodObject<{
+        systemPrompt: z.ZodOptional<z.ZodDefault<z.ZodString>>;
+        prompt: z.ZodDefault<z.ZodString>;
+        data: z.ZodOptional<z.ZodAny>;
+        images: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    }, "strict", z.ZodTypeAny, {
+        prompt: string;
+        data?: any;
+        images?: Record<string, any> | undefined;
+        systemPrompt?: string | undefined;
+    }, {
+        data?: any;
+        images?: Record<string, any> | undefined;
+        systemPrompt?: string | undefined;
+        prompt?: string | undefined;
+    }>>;
 }, "strict", z.ZodTypeAny, {
     text: string;
     speaker: string;
@@ -1984,6 +2016,12 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     imageNames?: string[] | undefined;
     imagePrompt?: string | undefined;
     moviePrompt?: string | undefined;
+    htmlPrompt?: {
+        prompt: string;
+        data?: any;
+        images?: Record<string, any> | undefined;
+        systemPrompt?: string | undefined;
+    } | undefined;
 }, {
     text?: string | undefined;
     duration?: number | undefined;
@@ -2145,6 +2183,12 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     imageNames?: string[] | undefined;
     imagePrompt?: string | undefined;
     moviePrompt?: string | undefined;
+    htmlPrompt?: {
+        data?: any;
+        images?: Record<string, any> | undefined;
+        systemPrompt?: string | undefined;
+        prompt?: string | undefined;
+    } | undefined;
 }>;
 export declare const mulmoCanvasDimensionSchema: z.ZodDefault<z.ZodObject<{
     width: z.ZodNumber;
@@ -3741,6 +3785,22 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
         imagePrompt: z.ZodOptional<z.ZodString>;
         moviePrompt: z.ZodOptional<z.ZodString>;
+        htmlPrompt: z.ZodOptional<z.ZodObject<{
+            systemPrompt: z.ZodOptional<z.ZodDefault<z.ZodString>>;
+            prompt: z.ZodDefault<z.ZodString>;
+            data: z.ZodOptional<z.ZodAny>;
+            images: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+        }, "strict", z.ZodTypeAny, {
+            prompt: string;
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+        }, {
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+            prompt?: string | undefined;
+        }>>;
     }, "strict", z.ZodTypeAny, {
         text: string;
         speaker: string;
@@ -3902,6 +3962,12 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames?: string[] | undefined;
         imagePrompt?: string | undefined;
         moviePrompt?: string | undefined;
+        htmlPrompt?: {
+            prompt: string;
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+        } | undefined;
     }, {
         text?: string | undefined;
         duration?: number | undefined;
@@ -4063,6 +4129,12 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames?: string[] | undefined;
         imagePrompt?: string | undefined;
         moviePrompt?: string | undefined;
+        htmlPrompt?: {
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+            prompt?: string | undefined;
+        } | undefined;
     }>, "many">;
     imagePath: z.ZodOptional<z.ZodString>;
     __test_invalid__: z.ZodOptional<z.ZodBoolean>;
@@ -4269,6 +4341,12 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames?: string[] | undefined;
         imagePrompt?: string | undefined;
         moviePrompt?: string | undefined;
+        htmlPrompt?: {
+            prompt: string;
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+        } | undefined;
     }[];
     lang?: string | undefined;
     title?: string | undefined;
@@ -4480,6 +4558,12 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imageNames?: string[] | undefined;
         imagePrompt?: string | undefined;
         moviePrompt?: string | undefined;
+        htmlPrompt?: {
+            data?: any;
+            images?: Record<string, any> | undefined;
+            systemPrompt?: string | undefined;
+            prompt?: string | undefined;
+        } | undefined;
     }[];
     lang?: string | undefined;
     title?: string | undefined;
@@ -5731,6 +5815,22 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
             imagePrompt: z.ZodOptional<z.ZodString>;
             moviePrompt: z.ZodOptional<z.ZodString>;
+            htmlPrompt: z.ZodOptional<z.ZodObject<{
+                systemPrompt: z.ZodOptional<z.ZodDefault<z.ZodString>>;
+                prompt: z.ZodDefault<z.ZodString>;
+                data: z.ZodOptional<z.ZodAny>;
+                images: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+            }, "strict", z.ZodTypeAny, {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            }, {
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+                prompt?: string | undefined;
+            }>>;
         }, "strict", z.ZodTypeAny, {
             text: string;
             speaker: string;
@@ -5892,6 +5992,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
         }, {
             text?: string | undefined;
             duration?: number | undefined;
@@ -6053,6 +6159,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+                prompt?: string | undefined;
+            } | undefined;
         }>, "many">;
         imagePath: z.ZodOptional<z.ZodString>;
         __test_invalid__: z.ZodOptional<z.ZodBoolean>;
@@ -6259,6 +6371,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
@@ -6470,6 +6588,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+                prompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
@@ -6788,6 +6912,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
@@ -7010,6 +7140,12 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+                prompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
@@ -7569,8 +7705,8 @@ export declare const mulmoScriptTemplateSchema: z.ZodObject<{
     }>>;
 }, "strict", z.ZodTypeAny, {
     title: string;
-    description: string;
     systemPrompt: string;
+    description: string;
     scriptName?: string | undefined;
     presentationStyle?: {
         audioParams: {
@@ -7650,8 +7786,8 @@ export declare const mulmoScriptTemplateSchema: z.ZodObject<{
     } | undefined;
 }, {
     title: string;
-    description: string;
     systemPrompt: string;
+    description: string;
     scriptName?: string | undefined;
     presentationStyle?: {
         $mulmocast: {
@@ -8206,9 +8342,9 @@ export declare const mulmoScriptTemplateFileSchema: z.ZodObject<{
     filename: z.ZodString;
 }, "strict", z.ZodTypeAny, {
     title: string;
+    systemPrompt: string;
     description: string;
     filename: string;
-    systemPrompt: string;
     scriptName?: string | undefined;
     presentationStyle?: {
         audioParams: {
@@ -8288,9 +8424,9 @@ export declare const mulmoScriptTemplateFileSchema: z.ZodObject<{
     } | undefined;
 }, {
     title: string;
+    systemPrompt: string;
     description: string;
     filename: string;
-    systemPrompt: string;
     scriptName?: string | undefined;
     presentationStyle?: {
         $mulmocast: {

package/lib/types/schema.js

@@ -171,6 +171,14 @@ export const audioParamsSchema = z
     audioVolume: z.number().default(1.0).describe("Volume of the audio"),
 })
     .strict();
+export const htmlPromptParamsSchema = z
+    .object({
+    systemPrompt: z.string().default("").optional(),
+    prompt: z.string().default(""),
+    data: z.any().optional(),
+    images: z.record(z.any()).optional(),
+})
+    .strict();
 export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.default("Presenter"),
@@ -187,6 +195,7 @@ export const mulmoBeatSchema = z
     imageNames: z.array(imageIdSchema).optional(), // list of image names to use for image generation. The default is all images in the imageParams.images.
     imagePrompt: z.string().optional(),
     moviePrompt: z.string().optional(),
+    htmlPrompt: htmlPromptParamsSchema.optional(),
 })
     .strict();
 export const mulmoCanvasDimensionSchema = z

package/lib/utils/ffmpeg_utils.d.ts

@@ -5,7 +5,7 @@ export type FfmpegContext = {
     filterComplex: string[];
 };
 export declare const FfmpegContextInit: () => FfmpegContext;
-export declare const FfmpegContextAddInput: (context: FfmpegContext, input: string) => number;
+export declare const FfmpegContextAddInput: (context: FfmpegContext, input: string, inputOptions?: string[]) => number;
 export declare const FfmpegContextPushFormattedAudio: (context: FfmpegContext, sourceId: string, outputId: string, duration?: number | undefined) => void;
 export declare const FfmpegContextInputFormattedAudio: (context: FfmpegContext, input: string, duration?: number | undefined) => string;
 export declare const FfmpegContextGenerateOutput: (context: FfmpegContext, output: string, options?: string[]) => Promise<number>;

package/lib/utils/ffmpeg_utils.js

@@ -7,8 +7,13 @@ export const FfmpegContextInit = () => {
         filterComplex: [],
     };
 };
-export const FfmpegContextAddInput = (context, input) => {
-    context.command.input(input);
+export const FfmpegContextAddInput = (context, input, inputOptions) => {
+    if (inputOptions) {
+        context.command.input(input).inputOptions(inputOptions);
+    }
+    else {
+        context.command.input(input);
+    }
     context.inputCount++;
     return context.inputCount - 1; // returned the index of the input
 };

package/lib/utils/preprocess.d.ts

@@ -211,6 +211,12 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             imageNames?: string[] | undefined;
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.0.21",
+  "version": "0.0.22",
   "description": "",
   "type": "module",
   "main": "lib/index.js",