mulmocast 1.2.26 → 1.2.28

package/assets/templates/documentary.json

@@ -0,0 +1,19 @@
+{
+  "title": "Documentary",
+  "description": "Template for documentary, biography, and other long form content.",
+  "systemPrompt": "Generate 30 to 40 beats. Another AI will generate image for each beat based on the text description of that beat. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
+  "presentationStyle": {
+    "$mulmocast": {
+      "version": "1.1",
+      "credit": "closing"
+    },
+    "imageParams": {
+      "style": "<style>Photo realistic and cinematic. Let the art convey the story and emotions without text. Use the image for the aspect ratio</style>"
+    },
+    "canvasSize": {
+      "width": 1536,
+      "height": 1024
+    }
+  },
+  "scriptName": "text_only_template.json"
+}

package/assets/templates/image_prompt.json

@@ -0,0 +1,16 @@
+{
+  "title": "Plain Image Prompt",
+  "description": "Template for image prompt (no style, no images).",
+  "systemPrompt": "Another AI will generate images for each beat based on the image prompt of that beat. Movie prompts must be written in English. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
+  "presentationStyle": {
+    "$mulmocast": {
+      "version": "1.1",
+      "credit": "closing"
+    },
+    "canvasSize": {
+      "width": 1536,
+      "height": 1024
+    }
+  },
+  "scriptName": "image_prompts_template.json"
+}

package/lib/actions/image_agents.d.ts

@@ -1,4 +1,4 @@
-import { MulmoStudioContext, MulmoBeat, MulmoCanvasDimension, MulmoImageParams } from "../types/index.js";
+import { MulmoStudioContext, MulmoBeat, MulmoCanvasDimension, MulmoImageParams, MulmoMovieParams } from "../types/index.js";
 export declare const imagePreprocessAgent: (namedInputs: {
     context: MulmoStudioContext;
     beat: MulmoBeat;
@@ -31,6 +31,10 @@ export declare const imagePreprocessAgent: (namedInputs: {
     duration?: number;
     audioFile?: string;
     beatDuration?: number;
+    movieAgentInfo?: {
+        agent: string;
+        movieParams: MulmoMovieParams;
+    };
     htmlPrompt?: undefined;
     htmlImageFile?: undefined;
     htmlPath?: undefined;
@@ -38,21 +42,6 @@ export declare const imagePreprocessAgent: (namedInputs: {
 } | {
     imagePath: string;
     imageFromMovie: boolean;
-    movieAgentInfo: {
-        agent: string;
-        movieParams: {
-            speed?: number | undefined;
-            provider?: string | undefined;
-            model?: string | undefined;
-            fillOption?: {
-                style: "aspectFit" | "aspectFill";
-            } | undefined;
-            transition?: {
-                type: "fade" | "slideout_left";
-                duration: number;
-            } | undefined;
-        };
-    };
     imageParams: MulmoImageParams;
     movieFile: string | undefined;
     soundEffectFile?: string;
@@ -71,6 +60,10 @@ export declare const imagePreprocessAgent: (namedInputs: {
     duration?: number;
     audioFile?: string;
     beatDuration?: number;
+    movieAgentInfo?: {
+        agent: string;
+        movieParams: MulmoMovieParams;
+    };
     htmlPrompt?: undefined;
     htmlImageFile?: undefined;
     htmlPath?: undefined;
@@ -81,21 +74,6 @@ export declare const imagePreprocessAgent: (namedInputs: {
     imageAgentInfo: import("../types/type.js").Text2ImageAgentInfo;
     prompt: string;
     referenceImages: string[];
-    movieAgentInfo: {
-        agent: string;
-        movieParams: {
-            speed?: number | undefined;
-            provider?: string | undefined;
-            model?: string | undefined;
-            fillOption?: {
-                style: "aspectFit" | "aspectFill";
-            } | undefined;
-            transition?: {
-                type: "fade" | "slideout_left";
-                duration: number;
-            } | undefined;
-        };
-    };
     imageParams: MulmoImageParams;
     movieFile: string | undefined;
     soundEffectFile?: string;
@@ -114,6 +92,10 @@ export declare const imagePreprocessAgent: (namedInputs: {
     duration?: number;
     audioFile?: string;
     beatDuration?: number;
+    movieAgentInfo?: {
+        agent: string;
+        movieParams: MulmoMovieParams;
+    };
     htmlPrompt?: undefined;
     htmlImageFile?: undefined;
     htmlPath?: undefined;

package/lib/actions/image_agents.js

@@ -2,7 +2,6 @@ import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMeth
 import { getBeatPngImagePath, getBeatMoviePaths, getAudioFilePath } from "../utils/file.js";
 import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
 import { renderHTMLToImage } from "../utils/markdown.js";
-import { GraphAILogger } from "graphai";
 import { beatId } from "../utils/utils.js";
 const htmlStyle = (context, beat) => {
     return {
@@ -57,21 +56,20 @@ export const imagePreprocessAgent = async (namedInputs) => {
             returnValue.audioFile = studioBeat?.audioFile;
         }
     }
+    returnValue.movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
     if (beat.image) {
         const plugin = MulmoBeatMethods.getPlugin(beat);
         const pluginPath = plugin.path({ beat, context, imagePath, ...htmlStyle(context, beat) });
         // undefined prompt indicates that image generation is not needed
         return { ...returnValue, imagePath: pluginPath, referenceImageForMovie: pluginPath };
     }
-    const movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
-    GraphAILogger.log(`movieParams: ${index}`, movieAgentInfo.movieParams, returnValue.soundEffectAgentInfo, "\n", beat.moviePrompt, beat.soundEffectPrompt);
     if (beat.moviePrompt && !beat.imagePrompt) {
-        return { ...returnValue, imagePath, imageFromMovie: true, movieAgentInfo }; // no image prompt, only movie prompt
+        return { ...returnValue, imagePath, imageFromMovie: true }; // no image prompt, only movie prompt
     }
     // referenceImages for "edit_image", openai agent.
     const referenceImages = MulmoBeatMethods.getImageReferenceForImageGenerator(beat, imageRefs);
     const prompt = imagePrompt(beat, imageAgentInfo.imageParams.style);
-    return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages, movieAgentInfo };
+    return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages };
 };
 export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index } = namedInputs;

package/lib/actions/image_references.js

@@ -4,7 +4,7 @@ import { getReferenceImagePath, resolveAssetPath } from "../utils/file.js";
 import { getExtention } from "../utils/utils.js";
 import { graphOption } from "./images.js";
 import { MulmoPresentationStyleMethods } from "../methods/index.js";
-import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent } from "../agents/index.js";
+import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent } from "../agents/index.js";
 // public api
 // Application may call this function directly to generate reference image.
 export const generateReferenceImage = async (inputs) => {
@@ -39,7 +39,7 @@ export const generateReferenceImage = async (inputs) => {
         },
     };
     const options = await graphOption(context);
-    const graph = new GraphAI(image_graph_data, { imageGenAIAgent, imageOpenaiAgent, mediaMockAgent }, options);
+    const graph = new GraphAI(image_graph_data, { imageGenAIAgent, imageOpenaiAgent, mediaMockAgent, imageReplicateAgent }, options);
     await graph.run();
     return imagePath;
 };

package/lib/actions/images.d.ts

@@ -54,6 +54,10 @@ export declare const beat_graph_data: {
                 duration?: number;
                 audioFile?: string;
                 beatDuration?: number;
+                movieAgentInfo?: {
+                    agent: string;
+                    movieParams: import("../types/type.js").MulmoMovieParams;
+                };
                 htmlPrompt?: undefined;
                 htmlImageFile?: undefined;
                 htmlPath?: undefined;
@@ -61,21 +65,6 @@ export declare const beat_graph_data: {
             } | {
                 imagePath: string;
                 imageFromMovie: boolean;
-                movieAgentInfo: {
-                    agent: string;
-                    movieParams: {
-                        speed?: number | undefined;
-                        provider?: string | undefined;
-                        model?: string | undefined;
-                        fillOption?: {
-                            style: "aspectFit" | "aspectFill";
-                        } | undefined;
-                        transition?: {
-                            type: "fade" | "slideout_left";
-                            duration: number;
-                        } | undefined;
-                    };
-                };
                 imageParams: MulmoImageParams;
                 movieFile: string | undefined;
                 soundEffectFile?: string;
@@ -94,6 +83,10 @@ export declare const beat_graph_data: {
                 duration?: number;
                 audioFile?: string;
                 beatDuration?: number;
+                movieAgentInfo?: {
+                    agent: string;
+                    movieParams: import("../types/type.js").MulmoMovieParams;
+                };
                 htmlPrompt?: undefined;
                 htmlImageFile?: undefined;
                 htmlPath?: undefined;
@@ -104,21 +97,6 @@ export declare const beat_graph_data: {
                 imageAgentInfo: import("../types/type.js").Text2ImageAgentInfo;
                 prompt: string;
                 referenceImages: string[];
-                movieAgentInfo: {
-                    agent: string;
-                    movieParams: {
-                        speed?: number | undefined;
-                        provider?: string | undefined;
-                        model?: string | undefined;
-                        fillOption?: {
-                            style: "aspectFit" | "aspectFill";
-                        } | undefined;
-                        transition?: {
-                            type: "fade" | "slideout_left";
-                            duration: number;
-                        } | undefined;
-                    };
-                };
                 imageParams: MulmoImageParams;
                 movieFile: string | undefined;
                 soundEffectFile?: string;
@@ -137,6 +115,10 @@ export declare const beat_graph_data: {
                 duration?: number;
                 audioFile?: string;
                 beatDuration?: number;
+                movieAgentInfo?: {
+                    agent: string;
+                    movieParams: import("../types/type.js").MulmoMovieParams;
+                };
                 htmlPrompt?: undefined;
                 htmlImageFile?: undefined;
                 htmlPath?: undefined;

package/lib/actions/images.js

@@ -5,7 +5,7 @@ import * as vanilla from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { imageGenAIAgent, imageOpenaiAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, } from "../agents/index.js";
+import { imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, } from "../agents/index.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
 import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
@@ -17,6 +17,7 @@ const vanillaAgents = vanilla.default ?? vanilla;
 const imageAgents = {
     imageGenAIAgent,
     imageOpenaiAgent,
+    imageReplicateAgent,
 };
 const movieAgents = {
     movieGenAIAgent,

package/lib/agents/image_replicate_agent.d.ts

@@ -0,0 +1,7 @@
+import { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { ReplicateImageAgentParams } from "../types/agent.js";
+import type { AgentBufferResult, ImageAgentInputs, AgentConfig } from "../types/agent.js";
+export type ReplicateImageAgentConfig = AgentConfig;
+export declare const imageReplicateAgent: AgentFunction<ReplicateImageAgentParams, AgentBufferResult, ImageAgentInputs, ReplicateImageAgentConfig>;
+declare const imageReplicateAgentInfo: AgentFunctionInfo;
+export default imageReplicateAgentInfo;

package/lib/agents/image_replicate_agent.js

@@ -0,0 +1,84 @@
+import { GraphAILogger } from "graphai";
+import Replicate from "replicate";
+import { getAspectRatio } from "./movie_replicate_agent.js";
+import { provider2ImageAgent } from "../utils/provider2agent.js";
+export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
+    const { prompt } = namedInputs;
+    const { canvasSize } = params;
+    const model = params.model ?? provider2ImageAgent.replicate.defaultModel;
+    const apiKey = config?.apiKey;
+    if (!apiKey) {
+        throw new Error("Replicate API key is required (REPLICATE_API_TOKEN)");
+    }
+    const replicate = new Replicate({
+        auth: apiKey,
+    });
+    const input = {
+        prompt,
+        width: canvasSize.width,
+        height: canvasSize.height,
+    };
+    if (model === "bytedance/seedream-4") {
+        input.size = "custom";
+        if (input.width < 1024) {
+            const ratio = 1024 / input.width;
+            input.width = 1024;
+            input.height = Math.round(input.height * ratio);
+        }
+        if (input.height < 1024) {
+            const ratio = 1024 / input.height;
+            input.width = Math.round(input.width * ratio);
+            input.height = 1024;
+        }
+    }
+    else if (model === "qwen/qwen-image") {
+        input.aspect_ratio = getAspectRatio(canvasSize);
+    }
+    // Add image if provided (for image-to-image generation)
+    /*
+    if (imagePath) {
+      const buffer = readFileSync(imagePath);
+      const base64Image = `data:image/png;base64,${buffer.toString("base64")}`;
+      const start_image = provider2MovieAgent.replicate.modelParams[model]?.start_image;
+      if (start_image === "first_frame_image" || start_image === "image" || start_image === "start_image") {
+        input[start_image] = base64Image;
+      } else if (start_image === undefined) {
+        throw new Error(`Model ${model} does not support image-to-video generation`);
+      } else {
+        input.image = base64Image;
+      }
+    }
+    */
+    try {
+        const output = await replicate.run(model, { input });
+        // Download the generated video
+        if (output && Array.isArray(output) && output.length > 0 && typeof output[0] === "object" && "url" in output[0]) {
+            const imageUrl = output[0].url();
+            const imageResponse = await fetch(imageUrl);
+            if (!imageResponse.ok) {
+                throw new Error(`Error downloading video: ${imageResponse.status} - ${imageResponse.statusText}`);
+            }
+            const arrayBuffer = await imageResponse.arrayBuffer();
+            const buffer = Buffer.from(arrayBuffer);
+            return { buffer };
+        }
+        throw new Error("ERROR: generateImage returned undefined");
+    }
+    catch (error) {
+        GraphAILogger.info("Replicate generation error:", error);
+        throw error;
+    }
+};
+const imageReplicateAgentInfo = {
+    name: "imageReplicateAgent",
+    agent: imageReplicateAgent,
+    mock: imageReplicateAgent,
+    samples: [],
+    description: "Replicate Image agent using FLUX and other models",
+    category: ["image"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["REPLICATE_API_TOKEN"],
+};
+export default imageReplicateAgentInfo;

package/lib/agents/index.d.ts

@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
 import combineAudioFilesAgent from "./combine_audio_files_agent.js";
 import imageGenAIAgent from "./image_genai_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
+import imageReplicateAgent from "./image_replicate_agent.js";
 import tavilySearchAgent from "./tavily_agent.js";
 import movieGenAIAgent from "./movie_genai_agent.js";
 import movieReplicateAgent from "./movie_replicate_agent.js";
@@ -18,4 +19,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/agents/index.js

@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
 import combineAudioFilesAgent from "./combine_audio_files_agent.js";
 import imageGenAIAgent from "./image_genai_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
+import imageReplicateAgent from "./image_replicate_agent.js";
 import tavilySearchAgent from "./tavily_agent.js";
 import movieGenAIAgent from "./movie_genai_agent.js";
 import movieReplicateAgent from "./movie_replicate_agent.js";
@@ -19,4 +20,4 @@ import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/data/promptTemplates.d.ts

@@ -39,10 +39,6 @@ export declare const promptTemplates: ({
             provider?: undefined;
             model?: undefined;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -61,6 +57,7 @@ export declare const promptTemplates: ({
                 Teacher?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;
@@ -182,10 +179,6 @@ export declare const promptTemplates: ({
             style?: undefined;
             model?: undefined;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -204,6 +197,7 @@ export declare const promptTemplates: ({
                 Teacher?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;
@@ -236,10 +230,6 @@ export declare const promptTemplates: ({
             provider?: undefined;
             model?: undefined;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -258,6 +248,7 @@ export declare const promptTemplates: ({
                 Teacher?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;
@@ -303,10 +294,6 @@ export declare const promptTemplates: ({
             provider?: undefined;
             model?: undefined;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -325,6 +312,7 @@ export declare const promptTemplates: ({
                 Teacher?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;
@@ -370,10 +358,6 @@ export declare const promptTemplates: ({
             provider: string;
             style: string;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -392,6 +376,7 @@ export declare const promptTemplates: ({
                 Teacher?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;
@@ -443,10 +428,6 @@ export declare const promptTemplates: ({
             provider?: undefined;
             model?: undefined;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -465,6 +446,7 @@ export declare const promptTemplates: ({
                 Teacher?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;
@@ -497,10 +479,6 @@ export declare const promptTemplates: ({
             provider?: undefined;
             model?: undefined;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -530,6 +508,7 @@ export declare const promptTemplates: ({
                 Presenter?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;
@@ -562,10 +541,6 @@ export declare const promptTemplates: ({
             provider?: undefined;
             model?: undefined;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -584,6 +559,7 @@ export declare const promptTemplates: ({
                 Teacher?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;
@@ -629,10 +605,6 @@ export declare const promptTemplates: ({
             provider?: undefined;
             model?: undefined;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -651,6 +623,7 @@ export declare const promptTemplates: ({
                 Teacher?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;
@@ -686,10 +659,6 @@ export declare const promptTemplates: ({
             provider?: undefined;
             model?: undefined;
         };
-        movieParams: {
-            provider: string;
-            model?: undefined;
-        };
         soundEffectParams: {
             provider: string;
         };
@@ -708,6 +677,7 @@ export declare const promptTemplates: ({
                 Teacher?: undefined;
             };
         };
+        movieParams?: undefined;
     };
     scriptName: string;
     systemPrompt: string;