mulmocast 0.1.2 → 0.1.3

package/assets/templates/characters.json

@@ -0,0 +1,16 @@
+{
+  "title": "Story with multiple characters",
+  "description": "Template for story with multiple characters.",
+  "presentationStyle": {
+    "$mulmocast": {
+      "version": "1.0",
+      "credit": "closing"
+    },
+    "canvasSize": {
+      "width": 1536,
+      "height": 1024
+    }
+  },
+  "systemPrompt": "Generate a script for a the given story with multiple characters. Generate image prompts for each character, and make references to them in the beats. Use the JSON below as a template.",
+  "scriptName": "image_refs.json"
+}

package/assets/templates/html.json

@@ -0,0 +1,6 @@
+{
+  "title": "Business presentation in HTML",
+  "description": "Template for business presentation in HTML.",
+  "systemPrompt": "Generate a script for a business presentation of the given topic. Another LLM will generate actual slides from the prompt and data for each beat. Adding optional data would help it to generate more compelling slide. Mention the reference in one of beats, if it exists. The valid type of reference is 'article', 'paper', 'image', 'video', 'audio'. Use the JSON below as a template.",
+  "scriptName": "html.json"
+}

package/lib/actions/audio.js

@@ -91,11 +91,13 @@ const graph_tts = {
             agent: ":preprocessor.ttsAgent",
             inputs: {
                 text: ":preprocessor.text",
-                file: ":preprocessor.audioPath",
-                force: ":context.force",
-                mulmoContext: ":context", // for cache
-                index: ":__mapIndex", // for cache
-                sessionType: "audio", // for cache
+                cache: {
+                    force: [":context.force"],
+                    file: ":preprocessor.audioPath",
+                    index: ":__mapIndex",
+                    mulmoContext: ":context",
+                    sessionType: "audio",
+                },
                 params: {
                     voice: ":preprocessor.voiceId",
                     speed: ":preprocessor.speechOptions.speed",
@@ -239,7 +241,7 @@ export const audio = async (context, settings, callbacks) => {
         const outputStudioFilePath = getOutputStudioFilePath(outDirPath, fileName);
         mkdir(outDirPath);
         mkdir(audioSegmentDirPath);
-        const config = settings2GraphAIConfig(settings);
+        const config = settings2GraphAIConfig(settings, process.env);
         const taskManager = new TaskManager(getConcurrency(context));
         const graph = new GraphAI(graph_data, audioAgents, { agentFilters, taskManager, config });
         graph.injectValue("context", context);

package/lib/actions/image_agents.d.ts

@@ -0,0 +1,121 @@
+import { MulmoStudioContext, MulmoBeat, MulmoCanvasDimension } from "../types/index.js";
+export declare const imagePreprocessAgent: (namedInputs: {
+    context: MulmoStudioContext;
+    beat: MulmoBeat;
+    index: number;
+    imageRefs: Record<string, string>;
+}) => Promise<{
+    imagePath: string;
+    htmlPrompt: string | undefined;
+    htmlPath: string;
+    htmlImageSystemPrompt: string[];
+} | {
+    imagePath: string | undefined;
+    referenceImageForMovie: string | undefined;
+    imageParams: {
+        provider: "openai" | "google";
+        style?: string | undefined;
+        model?: string | undefined;
+        moderation?: string | undefined;
+        images?: Record<string, {
+            type: "image";
+            source: {
+                url: string;
+                kind: "url";
+            } | {
+                kind: "base64";
+                data: string;
+            } | {
+                text: string;
+                kind: "text";
+            } | {
+                path: string;
+                kind: "path";
+            };
+        } | {
+            type: "imagePrompt";
+            prompt: string;
+        }> | undefined;
+    };
+    movieFile: string | undefined;
+    htmlPrompt?: undefined;
+    htmlPath?: undefined;
+    htmlImageSystemPrompt?: undefined;
+} | {
+    imagePath: string;
+    imageFromMovie: boolean;
+    imageParams: {
+        provider: "openai" | "google";
+        style?: string | undefined;
+        model?: string | undefined;
+        moderation?: string | undefined;
+        images?: Record<string, {
+            type: "image";
+            source: {
+                url: string;
+                kind: "url";
+            } | {
+                kind: "base64";
+                data: string;
+            } | {
+                text: string;
+                kind: "text";
+            } | {
+                path: string;
+                kind: "path";
+            };
+        } | {
+            type: "imagePrompt";
+            prompt: string;
+        }> | undefined;
+    };
+    movieFile: string | undefined;
+    htmlPrompt?: undefined;
+    htmlPath?: undefined;
+    htmlImageSystemPrompt?: undefined;
+} | {
+    imagePath: string;
+    referenceImageForMovie: string;
+    imageAgentInfo: import("../types/type.js").Text2ImageAgentInfo;
+    prompt: string;
+    referenceImages: string[];
+    imageParams: {
+        provider: "openai" | "google";
+        style?: string | undefined;
+        model?: string | undefined;
+        moderation?: string | undefined;
+        images?: Record<string, {
+            type: "image";
+            source: {
+                url: string;
+                kind: "url";
+            } | {
+                kind: "base64";
+                data: string;
+            } | {
+                text: string;
+                kind: "text";
+            } | {
+                path: string;
+                kind: "path";
+            };
+        } | {
+            type: "imagePrompt";
+            prompt: string;
+        }> | undefined;
+    };
+    movieFile: string | undefined;
+    htmlPrompt?: undefined;
+    htmlPath?: undefined;
+    htmlImageSystemPrompt?: undefined;
+}>;
+export declare const imagePluginAgent: (namedInputs: {
+    context: MulmoStudioContext;
+    beat: MulmoBeat;
+    index: number;
+}) => Promise<void>;
+export declare const htmlImageGeneratorAgent: (namedInputs: {
+    file: string;
+    canvasSize: MulmoCanvasDimension;
+    htmlText: string;
+}) => Promise<void>;

package/lib/actions/image_agents.js

@@ -0,0 +1,56 @@
+import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMethods } from "../methods/index.js";
+import { getBeatPngImagePath, getBeatMoviePath } from "../utils/file.js";
+import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
+import { renderHTMLToImage } from "../utils/markdown.js";
+const htmlStyle = (context, beat) => {
+    return {
+        canvasSize: MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle),
+        textSlideStyle: MulmoPresentationStyleMethods.getTextSlideStyle(context.presentationStyle, beat),
+    };
+};
+export const imagePreprocessAgent = async (namedInputs) => {
+    const { context, beat, index, imageRefs } = namedInputs;
+    const imagePath = getBeatPngImagePath(context, index);
+    if (beat.htmlPrompt) {
+        const htmlPrompt = MulmoBeatMethods.getHtmlPrompt(beat);
+        const htmlPath = imagePath.replace(/\.[^/.]+$/, ".html");
+        return { imagePath, htmlPrompt, htmlPath, htmlImageSystemPrompt: htmlImageSystemPrompt(context.presentationStyle.canvasSize) };
+    }
+    const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle, beat);
+    const returnValue = {
+        imageParams: imageAgentInfo.imageParams,
+        movieFile: beat.moviePrompt ? getBeatMoviePath(context, index) : undefined,
+    };
+    if (beat.image) {
+        const plugin = MulmoBeatMethods.getPlugin(beat);
+        const pluginPath = plugin.path({ beat, context, imagePath, ...htmlStyle(context, beat) });
+        // undefined prompt indicates that image generation is not needed
+        return { ...returnValue, imagePath: pluginPath, referenceImageForMovie: pluginPath };
+    }
+    if (beat.moviePrompt && !beat.imagePrompt) {
+        return { ...returnValue, imagePath, imageFromMovie: true }; // no image prompt, only movie prompt
+    }
+    // referenceImages for "edit_image", openai agent.
+    const referenceImages = MulmoBeatMethods.getImageReferenceForImageGenerator(beat, imageRefs);
+    const prompt = imagePrompt(beat, imageAgentInfo.imageParams.style);
+    return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages };
+};
+export const imagePluginAgent = async (namedInputs) => {
+    const { context, beat, index } = namedInputs;
+    const imagePath = getBeatPngImagePath(context, index);
+    const plugin = MulmoBeatMethods.getPlugin(beat);
+    try {
+        MulmoStudioContextMethods.setBeatSessionState(context, "image", index, true);
+        const processorParams = { beat, context, imagePath, ...htmlStyle(context, beat) };
+        await plugin.process(processorParams);
+        MulmoStudioContextMethods.setBeatSessionState(context, "image", index, false);
+    }
+    catch (error) {
+        MulmoStudioContextMethods.setBeatSessionState(context, "image", index, false);
+        throw error;
+    }
+};
+export const htmlImageGeneratorAgent = async (namedInputs) => {
+    const { file, canvasSize, htmlText } = namedInputs;
+    await renderHTMLToImage(htmlText, file, canvasSize.width, canvasSize.height);
+};

package/lib/actions/image_references.d.ts

@@ -0,0 +1,9 @@
+import { MulmoStudioContext, MulmoImagePromptMedia } from "../types/index.js";
+export declare const generateReferenceImage: (inputs: {
+    context: MulmoStudioContext;
+    key: string;
+    index: number;
+    image: MulmoImagePromptMedia;
+    force?: boolean;
+}) => Promise<string>;
+export declare const getImageRefs: (context: MulmoStudioContext) => Promise<Record<string, string>>;

package/lib/actions/image_references.js

@@ -0,0 +1,79 @@
+import fs from "fs";
+import { GraphAI } from "graphai";
+import { getReferenceImagePath } from "../utils/file.js";
+import { getExtention } from "../utils/utils.js";
+import { graphOption } from "./images.js";
+import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
+import { imageGoogleAgent, imageOpenaiAgent } from "../agents/index.js";
+// public api
+// Application may call this function directly to generate reference image.
+export const generateReferenceImage = async (inputs) => {
+    const { context, key, index, image, force } = inputs;
+    const imagePath = getReferenceImagePath(context, key, "png");
+    // generate image
+    const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(context.presentationStyle);
+    const prompt = `${image.prompt}\n${imageAgentInfo.imageParams.style || ""}`;
+    const image_graph_data = {
+        version: 0.5,
+        nodes: {
+            imageGenerator: {
+                agent: imageAgentInfo.agent,
+                retry: 2,
+                inputs: {
+                    prompt,
+                    cache: {
+                        force: [context.force, force ?? false],
+                        file: imagePath,
+                        index,
+                        mulmoContext: context,
+                        sessionType: "imageReference",
+                    },
+                },
+                params: {
+                    model: imageAgentInfo.imageParams.model,
+                    canvasSize: context.presentationStyle.canvasSize,
+                },
+            },
+        },
+    };
+    const options = await graphOption(context);
+    const graph = new GraphAI(image_graph_data, { imageGoogleAgent, imageOpenaiAgent }, options);
+    await graph.run();
+    return imagePath;
+};
+const downLoadImage = async (context, key, url) => {
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Failed to download image: ${url}`);
+    }
+    const buffer = Buffer.from(await response.arrayBuffer());
+    // Detect file extension from Content-Type header or URL
+    const extension = getExtention(response.headers.get("content-type"), url);
+    const imagePath = getReferenceImagePath(context, key, extension);
+    await fs.promises.writeFile(imagePath, buffer);
+    return imagePath;
+};
+export const getImageRefs = async (context) => {
+    const images = context.presentationStyle.imageParams?.images;
+    if (!images) {
+        return {};
+    }
+    const imageRefs = {};
+    await Promise.all(Object.keys(images)
+        .sort()
+        .map(async (key, index) => {
+        const image = images[key];
+        if (image.type === "imagePrompt") {
+            imageRefs[key] = await generateReferenceImage({ context, key, index, image, force: false });
+        }
+        else if (image.type === "image") {
+            if (image.source.kind === "path") {
+                imageRefs[key] = MulmoStudioContextMethods.resolveAssetPath(context, image.source.path);
+            }
+            else if (image.source.kind === "url") {
+                imageRefs[key] = await downLoadImage(context, key, image.source.url);
+            }
+        }
+    }));
+    return imageRefs;
+};

package/lib/actions/images.d.ts

@@ -1,112 +1,12 @@
-import type { CallbackFunction } from "graphai";
-import { MulmoStudioContext, MulmoBeat } from "../types/index.js";
-export declare const imagePreprocessAgent: (namedInputs: {
-    context: MulmoStudioContext;
-    beat: MulmoBeat;
+import type { GraphOptions, CallbackFunction } from "graphai";
+import { MulmoStudioContext } from "../types/index.js";
+export declare const graphOption: (context: MulmoStudioContext, settings?: Record<string, string>) => Promise<GraphOptions>;
+export declare const images: (context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;
+export declare const generateBeatImage: (inputs: {
     index: number;
-    imageRefs: Record<string, string>;
-}) => Promise<{
-    imageParams: {
-        provider: "openai" | "google";
-        style?: string | undefined;
-        model?: string | undefined;
-        moderation?: string | undefined;
-        images?: Record<string, {
-            type: "image";
-            source: {
-                url: string;
-                kind: "url";
-            } | {
-                kind: "base64";
-                data: string;
-            } | {
-                text: string;
-                kind: "text";
-            } | {
-                path: string;
-                kind: "path";
-            };
-        }> | undefined;
-    };
-    movieFile: string | undefined;
-    imagePath: string | undefined;
-    referenceImage: string | undefined;
-    htmlPrompt?: undefined;
-    htmlPath?: undefined;
-    htmlImageSystemPrompt?: undefined;
-} | {
-    imagePath: string;
-    htmlPrompt: string;
-    htmlPath: string;
-    htmlImageSystemPrompt: string[];
-} | {
-    imagePath: string;
-    images: string[];
-    imageFromMovie: boolean;
-    imageParams: {
-        provider: "openai" | "google";
-        style?: string | undefined;
-        model?: string | undefined;
-        moderation?: string | undefined;
-        images?: Record<string, {
-            type: "image";
-            source: {
-                url: string;
-                kind: "url";
-            } | {
-                kind: "base64";
-                data: string;
-            } | {
-                text: string;
-                kind: "text";
-            } | {
-                path: string;
-                kind: "path";
-            };
-        }> | undefined;
-    };
-    movieFile: string | undefined;
-    htmlPrompt?: undefined;
-    htmlPath?: undefined;
-    htmlImageSystemPrompt?: undefined;
-} | {
-    images: string[];
-    imageParams: {
-        provider: "openai" | "google";
-        style?: string | undefined;
-        model?: string | undefined;
-        moderation?: string | undefined;
-        images?: Record<string, {
-            type: "image";
-            source: {
-                url: string;
-                kind: "url";
-            } | {
-                kind: "base64";
-                data: string;
-            } | {
-                text: string;
-                kind: "text";
-            } | {
-                path: string;
-                kind: "path";
-            };
-        }> | undefined;
-    };
-    movieFile: string | undefined;
-    imageAgentInfo: import("../types/type.js").Text2ImageAgentInfo;
-    imagePath: string;
-    referenceImage: string;
-    prompt: string;
-    htmlPrompt?: undefined;
-    htmlPath?: undefined;
-    htmlImageSystemPrompt?: undefined;
-}>;
-export declare const imagePluginAgent: (namedInputs: {
     context: MulmoStudioContext;
-    beat: MulmoBeat;
-    index: number;
+    settings?: Record<string, string>;
+    callbacks?: CallbackFunction[];
+    forceMovie?: boolean;
+    forceImage?: boolean;
 }) => Promise<void>;
-export declare const getImageRefs: (context: MulmoStudioContext) => Promise<Record<string, string>>;
-export declare const images: (context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<MulmoStudioContext>;
-export declare const generateBeatImage: (index: number, context: MulmoStudioContext, settings?: Record<string, string>, callbacks?: CallbackFunction[]) => Promise<void>;