mulmocast 1.2.27 → 1.2.28

package/assets/templates/documentary.json

@@ -0,0 +1,19 @@
+{
+  "title": "Documentary",
+  "description": "Template for documentary, biography, and other long form content.",
+  "systemPrompt": "Generate 30 to 40 beats. Another AI will generate image for each beat based on the text description of that beat. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
+  "presentationStyle": {
+    "$mulmocast": {
+      "version": "1.1",
+      "credit": "closing"
+    },
+    "imageParams": {
+      "style": "<style>Photo realistic and cinematic. Let the art convey the story and emotions without text. Use the image for the aspect ratio</style>"
+    },
+    "canvasSize": {
+      "width": 1536,
+      "height": 1024
+    }
+  },
+  "scriptName": "text_only_template.json"
+}

package/lib/actions/image_agents.d.ts

@@ -1,4 +1,4 @@
-import { MulmoStudioContext, MulmoBeat, MulmoCanvasDimension, MulmoImageParams } from "../types/index.js";
+import { MulmoStudioContext, MulmoBeat, MulmoCanvasDimension, MulmoImageParams, MulmoMovieParams } from "../types/index.js";
 export declare const imagePreprocessAgent: (namedInputs: {
     context: MulmoStudioContext;
     beat: MulmoBeat;
@@ -31,6 +31,10 @@ export declare const imagePreprocessAgent: (namedInputs: {
     duration?: number;
     audioFile?: string;
     beatDuration?: number;
+    movieAgentInfo?: {
+        agent: string;
+        movieParams: MulmoMovieParams;
+    };
     htmlPrompt?: undefined;
     htmlImageFile?: undefined;
     htmlPath?: undefined;
@@ -38,21 +42,6 @@ export declare const imagePreprocessAgent: (namedInputs: {
 } | {
     imagePath: string;
     imageFromMovie: boolean;
-    movieAgentInfo: {
-        agent: string;
-        movieParams: {
-            speed?: number | undefined;
-            provider?: string | undefined;
-            model?: string | undefined;
-            fillOption?: {
-                style: "aspectFit" | "aspectFill";
-            } | undefined;
-            transition?: {
-                type: "fade" | "slideout_left";
-                duration: number;
-            } | undefined;
-        };
-    };
     imageParams: MulmoImageParams;
     movieFile: string | undefined;
     soundEffectFile?: string;
@@ -71,6 +60,10 @@ export declare const imagePreprocessAgent: (namedInputs: {
     duration?: number;
     audioFile?: string;
     beatDuration?: number;
+    movieAgentInfo?: {
+        agent: string;
+        movieParams: MulmoMovieParams;
+    };
     htmlPrompt?: undefined;
     htmlImageFile?: undefined;
     htmlPath?: undefined;
@@ -81,21 +74,6 @@ export declare const imagePreprocessAgent: (namedInputs: {
     imageAgentInfo: import("../types/type.js").Text2ImageAgentInfo;
     prompt: string;
     referenceImages: string[];
-    movieAgentInfo: {
-        agent: string;
-        movieParams: {
-            speed?: number | undefined;
-            provider?: string | undefined;
-            model?: string | undefined;
-            fillOption?: {
-                style: "aspectFit" | "aspectFill";
-            } | undefined;
-            transition?: {
-                type: "fade" | "slideout_left";
-                duration: number;
-            } | undefined;
-        };
-    };
     imageParams: MulmoImageParams;
     movieFile: string | undefined;
     soundEffectFile?: string;
@@ -114,6 +92,10 @@ export declare const imagePreprocessAgent: (namedInputs: {
     duration?: number;
     audioFile?: string;
     beatDuration?: number;
+    movieAgentInfo?: {
+        agent: string;
+        movieParams: MulmoMovieParams;
+    };
     htmlPrompt?: undefined;
     htmlImageFile?: undefined;
     htmlPath?: undefined;

package/lib/actions/image_agents.js

@@ -2,7 +2,6 @@ import { MulmoPresentationStyleMethods, MulmoStudioContextMethods, MulmoBeatMeth
 import { getBeatPngImagePath, getBeatMoviePaths, getAudioFilePath } from "../utils/file.js";
 import { imagePrompt, htmlImageSystemPrompt } from "../utils/prompt.js";
 import { renderHTMLToImage } from "../utils/markdown.js";
-import { GraphAILogger } from "graphai";
 import { beatId } from "../utils/utils.js";
 const htmlStyle = (context, beat) => {
     return {
@@ -57,21 +56,20 @@ export const imagePreprocessAgent = async (namedInputs) => {
             returnValue.audioFile = studioBeat?.audioFile;
         }
     }
+    returnValue.movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
     if (beat.image) {
         const plugin = MulmoBeatMethods.getPlugin(beat);
         const pluginPath = plugin.path({ beat, context, imagePath, ...htmlStyle(context, beat) });
         // undefined prompt indicates that image generation is not needed
         return { ...returnValue, imagePath: pluginPath, referenceImageForMovie: pluginPath };
     }
-    const movieAgentInfo = MulmoPresentationStyleMethods.getMovieAgentInfo(context.presentationStyle, beat);
-    GraphAILogger.log(`movieParams: ${index}`, movieAgentInfo.movieParams, returnValue.soundEffectAgentInfo, "\n", beat.moviePrompt, beat.soundEffectPrompt);
     if (beat.moviePrompt && !beat.imagePrompt) {
-        return { ...returnValue, imagePath, imageFromMovie: true, movieAgentInfo }; // no image prompt, only movie prompt
+        return { ...returnValue, imagePath, imageFromMovie: true }; // no image prompt, only movie prompt
     }
     // referenceImages for "edit_image", openai agent.
     const referenceImages = MulmoBeatMethods.getImageReferenceForImageGenerator(beat, imageRefs);
     const prompt = imagePrompt(beat, imageAgentInfo.imageParams.style);
-    return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages, movieAgentInfo };
+    return { ...returnValue, imagePath, referenceImageForMovie: imagePath, imageAgentInfo, prompt, referenceImages };
 };
 export const imagePluginAgent = async (namedInputs) => {
     const { context, beat, index } = namedInputs;

package/lib/actions/image_references.js

@@ -4,7 +4,7 @@ import { getReferenceImagePath, resolveAssetPath } from "../utils/file.js";
 import { getExtention } from "../utils/utils.js";
 import { graphOption } from "./images.js";
 import { MulmoPresentationStyleMethods } from "../methods/index.js";
-import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent } from "../agents/index.js";
+import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent } from "../agents/index.js";
 // public api
 // Application may call this function directly to generate reference image.
 export const generateReferenceImage = async (inputs) => {
@@ -39,7 +39,7 @@ export const generateReferenceImage = async (inputs) => {
         },
     };
     const options = await graphOption(context);
-    const graph = new GraphAI(image_graph_data, { imageGenAIAgent, imageOpenaiAgent, mediaMockAgent }, options);
+    const graph = new GraphAI(image_graph_data, { imageGenAIAgent, imageOpenaiAgent, mediaMockAgent, imageReplicateAgent }, options);
     await graph.run();
     return imagePath;
 };

package/lib/actions/images.d.ts

@@ -54,6 +54,10 @@ export declare const beat_graph_data: {
                 duration?: number;
                 audioFile?: string;
                 beatDuration?: number;
+                movieAgentInfo?: {
+                    agent: string;
+                    movieParams: import("../types/type.js").MulmoMovieParams;
+                };
                 htmlPrompt?: undefined;
                 htmlImageFile?: undefined;
                 htmlPath?: undefined;
@@ -61,21 +65,6 @@ export declare const beat_graph_data: {
             } | {
                 imagePath: string;
                 imageFromMovie: boolean;
-                movieAgentInfo: {
-                    agent: string;
-                    movieParams: {
-                        speed?: number | undefined;
-                        provider?: string | undefined;
-                        model?: string | undefined;
-                        fillOption?: {
-                            style: "aspectFit" | "aspectFill";
-                        } | undefined;
-                        transition?: {
-                            type: "fade" | "slideout_left";
-                            duration: number;
-                        } | undefined;
-                    };
-                };
                 imageParams: MulmoImageParams;
                 movieFile: string | undefined;
                 soundEffectFile?: string;
@@ -94,6 +83,10 @@ export declare const beat_graph_data: {
                 duration?: number;
                 audioFile?: string;
                 beatDuration?: number;
+                movieAgentInfo?: {
+                    agent: string;
+                    movieParams: import("../types/type.js").MulmoMovieParams;
+                };
                 htmlPrompt?: undefined;
                 htmlImageFile?: undefined;
                 htmlPath?: undefined;
@@ -104,21 +97,6 @@ export declare const beat_graph_data: {
                 imageAgentInfo: import("../types/type.js").Text2ImageAgentInfo;
                 prompt: string;
                 referenceImages: string[];
-                movieAgentInfo: {
-                    agent: string;
-                    movieParams: {
-                        speed?: number | undefined;
-                        provider?: string | undefined;
-                        model?: string | undefined;
-                        fillOption?: {
-                            style: "aspectFit" | "aspectFill";
-                        } | undefined;
-                        transition?: {
-                            type: "fade" | "slideout_left";
-                            duration: number;
-                        } | undefined;
-                    };
-                };
                 imageParams: MulmoImageParams;
                 movieFile: string | undefined;
                 soundEffectFile?: string;
@@ -137,6 +115,10 @@ export declare const beat_graph_data: {
                 duration?: number;
                 audioFile?: string;
                 beatDuration?: number;
+                movieAgentInfo?: {
+                    agent: string;
+                    movieParams: import("../types/type.js").MulmoMovieParams;
+                };
                 htmlPrompt?: undefined;
                 htmlImageFile?: undefined;
                 htmlPath?: undefined;

package/lib/actions/images.js

@@ -5,7 +5,7 @@ import * as vanilla from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { imageGenAIAgent, imageOpenaiAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, } from "../agents/index.js";
+import { imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, } from "../agents/index.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
 import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
@@ -17,6 +17,7 @@ const vanillaAgents = vanilla.default ?? vanilla;
 const imageAgents = {
     imageGenAIAgent,
     imageOpenaiAgent,
+    imageReplicateAgent,
 };
 const movieAgents = {
     movieGenAIAgent,

package/lib/agents/image_replicate_agent.d.ts

@@ -0,0 +1,7 @@
+import { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { ReplicateImageAgentParams } from "../types/agent.js";
+import type { AgentBufferResult, ImageAgentInputs, AgentConfig } from "../types/agent.js";
+export type ReplicateImageAgentConfig = AgentConfig;
+export declare const imageReplicateAgent: AgentFunction<ReplicateImageAgentParams, AgentBufferResult, ImageAgentInputs, ReplicateImageAgentConfig>;
+declare const imageReplicateAgentInfo: AgentFunctionInfo;
+export default imageReplicateAgentInfo;

package/lib/agents/image_replicate_agent.js

@@ -0,0 +1,84 @@
+import { GraphAILogger } from "graphai";
+import Replicate from "replicate";
+import { getAspectRatio } from "./movie_replicate_agent.js";
+import { provider2ImageAgent } from "../utils/provider2agent.js";
+export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
+    const { prompt } = namedInputs;
+    const { canvasSize } = params;
+    const model = params.model ?? provider2ImageAgent.replicate.defaultModel;
+    const apiKey = config?.apiKey;
+    if (!apiKey) {
+        throw new Error("Replicate API key is required (REPLICATE_API_TOKEN)");
+    }
+    const replicate = new Replicate({
+        auth: apiKey,
+    });
+    const input = {
+        prompt,
+        width: canvasSize.width,
+        height: canvasSize.height,
+    };
+    if (model === "bytedance/seedream-4") {
+        input.size = "custom";
+        if (input.width < 1024) {
+            const ratio = 1024 / input.width;
+            input.width = 1024;
+            input.height = Math.round(input.height * ratio);
+        }
+        if (input.height < 1024) {
+            const ratio = 1024 / input.height;
+            input.width = Math.round(input.width * ratio);
+            input.height = 1024;
+        }
+    }
+    else if (model === "qwen/qwen-image") {
+        input.aspect_ratio = getAspectRatio(canvasSize);
+    }
+    // Add image if provided (for image-to-image generation)
+    /*
+    if (imagePath) {
+      const buffer = readFileSync(imagePath);
+      const base64Image = `data:image/png;base64,${buffer.toString("base64")}`;
+      const start_image = provider2MovieAgent.replicate.modelParams[model]?.start_image;
+      if (start_image === "first_frame_image" || start_image === "image" || start_image === "start_image") {
+        input[start_image] = base64Image;
+      } else if (start_image === undefined) {
+        throw new Error(`Model ${model} does not support image-to-video generation`);
+      } else {
+        input.image = base64Image;
+      }
+    }
+    */
+    try {
+        const output = await replicate.run(model, { input });
+        // Download the generated video
+        if (output && Array.isArray(output) && output.length > 0 && typeof output[0] === "object" && "url" in output[0]) {
+            const imageUrl = output[0].url();
+            const imageResponse = await fetch(imageUrl);
+            if (!imageResponse.ok) {
+                throw new Error(`Error downloading video: ${imageResponse.status} - ${imageResponse.statusText}`);
+            }
+            const arrayBuffer = await imageResponse.arrayBuffer();
+            const buffer = Buffer.from(arrayBuffer);
+            return { buffer };
+        }
+        throw new Error("ERROR: generateImage returned undefined");
+    }
+    catch (error) {
+        GraphAILogger.info("Replicate generation error:", error);
+        throw error;
+    }
+};
+const imageReplicateAgentInfo = {
+    name: "imageReplicateAgent",
+    agent: imageReplicateAgent,
+    mock: imageReplicateAgent,
+    samples: [],
+    description: "Replicate Image agent using FLUX and other models",
+    category: ["image"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["REPLICATE_API_TOKEN"],
+};
+export default imageReplicateAgentInfo;

package/lib/agents/index.d.ts

@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
 import combineAudioFilesAgent from "./combine_audio_files_agent.js";
 import imageGenAIAgent from "./image_genai_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
+import imageReplicateAgent from "./image_replicate_agent.js";
 import tavilySearchAgent from "./tavily_agent.js";
 import movieGenAIAgent from "./movie_genai_agent.js";
 import movieReplicateAgent from "./movie_replicate_agent.js";
@@ -18,4 +19,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/agents/index.js

@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
 import combineAudioFilesAgent from "./combine_audio_files_agent.js";
 import imageGenAIAgent from "./image_genai_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
+import imageReplicateAgent from "./image_replicate_agent.js";
 import tavilySearchAgent from "./tavily_agent.js";
 import movieGenAIAgent from "./movie_genai_agent.js";
 import movieReplicateAgent from "./movie_replicate_agent.js";
@@ -19,4 +20,4 @@ import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/agents/puppeteer_crawler_agent.d.ts

@@ -1,13 +1,4 @@
 import { AgentFunction, AgentFunctionInfo } from "graphai";
-type Article = {
-    url: string;
-    title: string | null;
-    byline: string | null;
-    excerpt: string | null;
-    length: number | null;
-    textContent: string | null;
-};
-export declare const fetchArticle: (url: string) => Promise<Article>;
 export declare const puppeteerCrawlerAgent: AgentFunction;
 declare const puppeteerCrawlerAgentInfo: AgentFunctionInfo;
 export default puppeteerCrawlerAgentInfo;

package/lib/agents/puppeteer_crawler_agent.js

@@ -17,7 +17,7 @@ const waitStable = async (page, ms = 1200, step = 200) => {
         await new Promise((r) => setTimeout(r, step));
     }
 };
-export const fetchArticle = async (url) => {
+const fetchArticle = async (url) => {
     const browser = await puppeteer.launch({
         headless: true,
         args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"],

package/lib/data/promptTemplates.js

@@ -256,6 +256,48 @@ export const promptTemplates = [
         systemPrompt: "Another AI will generate comic strips for each beat based on the text description of that beat. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
         title: "American Comic Strips",
     },
+    {
+        description: "Template for documentary, biography, and other long form content.",
+        filename: "documentary",
+        presentationStyle: {
+            $mulmocast: {
+                credit: "closing",
+                version: "1.1",
+            },
+            audioParams: {
+                audioVolume: 1,
+                bgmVolume: 0.2,
+                closingPadding: 0.8,
+                introPadding: 1,
+                outroPadding: 1,
+                padding: 0.3,
+                suppressSpeech: false,
+            },
+            canvasSize: {
+                height: 1024,
+                width: 1536,
+            },
+            imageParams: {
+                style: "<style>Photo realistic and cinematic. Let the art convey the story and emotions without text. Use the image for the aspect ratio</style>",
+            },
+            soundEffectParams: {
+                provider: "replicate",
+            },
+            speechParams: {
+                speakers: {
+                    Presenter: {
+                        displayName: {
+                            en: "Presenter",
+                        },
+                        voiceId: "shimmer",
+                    },
+                },
+            },
+        },
+        scriptName: "text_only_template.json",
+        systemPrompt: "Generate 30 to 40 beats. Another AI will generate image for each beat based on the text description of that beat. Mention the reference in one of beats, if it exists. Use the JSON below as a template.",
+        title: "Documentary",
+    },
     {
         description: "Template for Dr. Slump style comic presentation.",
         filename: "drslump_comic",