mulmocast 0.0.10 → 0.0.11

package/README.md

@@ -90,11 +90,28 @@ Create a `.env` file in your project directory with the following API keys:
 ```bash
 OPENAI_API_KEY=your_openai_api_key
 ```
-### Optional
+
+#### (Optional) For the advanced image generation model
 ```bash
 DEFAULT_OPENAI_IMAGE_MODEL=gpt-image-1 # for the advanced image generation model
-GOOGLE_PROJECT_ID=your_google_project_id # for Google's image generation model
-NIJIVOICE_API_KEY=your_nijivoice_api_key # for Nijivoice's TTS model
+```
+
+#### (Optional) For Google's image generation model
+```bash
+GOOGLE_PROJECT_ID=your_google_project_id
+```
+
+You may also need to take the following steps before running any commands:
+1. Install [gcloud CLI](https://cloud.google.com/sdk/docs/install)
+2. Login by `gcloud auth application-default login`
+
+#### (Optional) For Nijivoice's TTS model
+```bash
+NIJIVOICE_API_KEY=your_nijivoice_api_key
+```
+
+#### (Optional) to access web in mulmo tool
+```bash
 BROWSERLESS_API_TOKEN=your_browserless_api_token # to access web in mulmo tool
 ```
 

package/assets/templates/shorts.json

@@ -0,0 +1,18 @@
+{
+  "title": "Short movie template",
+  "description": "Template for Youtube shorts.",
+  "systemPrompt": "Generate a script for a Youtube shorts of the given topic. The first beat should be a hook, which describes the topic. Another AI will generate images for each beat based on the image prompt of that beat. Movie prompts must be written in English.",
+  "presentationStyle": {
+    "$mulmocast": {
+      "version": "1.0"
+    },
+    "canvasSize": {
+      "width": 720,
+      "height": 1280
+    },
+    "imageParams": {
+      "style": "<style>Photo realistic, cinematic.</style>"
+    }
+  },
+  "scriptName": "movie_prompts_template.json"
+}

package/lib/actions/audio.d.ts

@@ -1,3 +1,4 @@
 import "dotenv/config";
+import type { CallbackFunction } from "graphai";
 import { MulmoStudioContext } from "../types/index.js";
-export declare const audio: (context: MulmoStudioContext) => Promise<void>;
+export declare const audio: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;

package/lib/actions/audio.js

@@ -160,7 +160,7 @@ const agentFilters = [
         nodeIds: ["tts"],
     },
 ];
-export const audio = async (context) => {
+export const audio = async (context, callbacks) => {
     try {
         MulmoStudioMethods.setSessionState(context.studio, "audio", true);
         const { studio, fileDirs, lang } = context;
@@ -187,6 +187,11 @@ export const audio = async (context) => {
         graph.injectValue("outputStudioFilePath", outputStudioFilePath);
         graph.injectValue("audioSegmentDirPath", audioSegmentDirPath);
         graph.injectValue("audioDirPath", audioDirPath);
+        if (callbacks) {
+            callbacks.forEach((callback) => {
+                graph.registerCallback(callback);
+            });
+        }
         await graph.run();
         writingMessage(audioCombinedFilePath);
     }

package/lib/actions/images.d.ts

@@ -1,2 +1,3 @@
+import type { CallbackFunction } from "graphai";
 import { MulmoStudioContext } from "../types/index.js";
-export declare const images: (context: MulmoStudioContext) => Promise<void>;
+export declare const images: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;

package/lib/actions/images.js

@@ -25,14 +25,10 @@ const htmlStyle = (script, beat) => {
 const imagePreprocessAgent = async (namedInputs) => {
     const { context, beat, index, suffix, imageDirPath, imageAgentInfo, imageRefs } = namedInputs;
     const imageParams = { ...imageAgentInfo.imageParams, ...beat.imageParams };
-    if (!imageParams.size) {
-        const canvasSize = MulmoScriptMethods.getCanvasSize(context.studio.script);
-        imageParams.size = `${canvasSize.width}x${canvasSize.height}`;
-    }
     const imagePath = `${imageDirPath}/${context.studio.filename}/${index}${suffix}.png`;
     const returnValue = {
-        aspectRatio: MulmoScriptMethods.getAspectRatio(context.studio.script),
         imageParams,
+        movieFile: beat.moviePrompt ? `${imageDirPath}/${context.studio.filename}/${index}.mov` : undefined,
     };
     if (beat.image) {
         const plugin = imagePlugins.find((plugin) => plugin.imageType === beat?.image?.type);
@@ -42,20 +38,24 @@ const imagePreprocessAgent = async (namedInputs) => {
                 const processorParams = { beat, context, imagePath, ...htmlStyle(context.studio.script, beat) };
                 const path = await plugin.process(processorParams);
                 // undefined prompt indicates that image generation is not needed
-                return { path, ...returnValue };
+                return { imagePath: path, ...returnValue };
             }
             finally {
                 MulmoStudioMethods.setBeatSessionState(context.studio, "image", index, false);
             }
         }
     }
-    const prompt = imagePrompt(beat, imageParams.style);
+    // images for "edit_image"
     const images = (() => {
         const imageNames = beat.imageNames ?? Object.keys(imageRefs); // use all images if imageNames is not specified
         const sources = imageNames.map((name) => imageRefs[name]);
         return sources.filter((source) => source !== undefined);
     })();
-    return { path: imagePath, prompt, ...returnValue, images };
+    if (beat.moviePrompt && !beat.imagePrompt) {
+        return { ...returnValue, images }; // no image prompt, only movie prompt
+    }
+    const prompt = imagePrompt(beat, imageParams.style);
+    return { imagePath, prompt, ...returnValue, images };
 };
 const graph_data = {
     version: 0.5,
@@ -100,54 +100,36 @@ const graph_data = {
                         retry: 3,
                         inputs: {
                             prompt: ":preprocessor.prompt",
-                            file: ":preprocessor.path", // only for fileCacheAgentFilter
+                            images: ":preprocessor.images",
+                            file: ":preprocessor.imagePath", // only for fileCacheAgentFilter
                             text: ":preprocessor.prompt", // only for fileCacheAgentFilter
-                            force: ":context.force",
-                            studio: ":context.studio", // for cache
-                            index: ":__mapIndex", // for cache
-                            sessionType: "image", // for cache
+                            force: ":context.force", // only for fileCacheAgentFilter
+                            studio: ":context.studio", // for fileCacheAgentFilter
+                            index: ":__mapIndex", // for fileCacheAgentFilter
+                            sessionType: "image", // for fileCacheAgentFilter
                             params: {
                                 model: ":preprocessor.imageParams.model",
-                                size: ":preprocessor.imageParams.size",
                                 moderation: ":preprocessor.imageParams.moderation",
-                                aspectRatio: ":preprocessor.aspectRatio",
-                                images: ":preprocessor.images",
+                                canvasSize: ":context.studio.script.canvasSize",
                             },
                         },
                         defaultValue: {},
                     },
-                    prepareMovie: {
-                        agent: (namedInputs) => {
-                            const { beat, imageDirPath, index, context } = namedInputs;
-                            if (beat.moviePrompt) {
-                                const movieFile = `${imageDirPath}/${context.studio.filename}/${index}.mov`;
-                                return { movieFile };
-                            }
-                            return {};
-                        },
-                        inputs: {
-                            result: ":imageGenerator", // to wait for imageGenerator to finish
-                            imagePath: ":preprocessor.path",
-                            beat: ":beat",
-                            imageDirPath: ":imageDirPath",
-                            index: ":__mapIndex",
-                            context: ":context",
-                        },
-                    },
                     movieGenerator: {
-                        if: ":prepareMovie.movieFile",
+                        if: ":preprocessor.movieFile",
                         agent: "movieGoogleAgent",
                         inputs: {
+                            onComplete: ":imageGenerator", // to wait for imageGenerator to finish
                             prompt: ":beat.moviePrompt",
-                            imagePath: ":preprocessor.path",
-                            file: ":prepareMovie.movieFile",
+                            imagePath: ":preprocessor.imagePath",
+                            file: ":preprocessor.movieFile",
                             studio: ":context.studio", // for cache
                             index: ":__mapIndex", // for cache
                             sessionType: "movie", // for cache
                             params: {
                                 model: ":context.studio.script.movieParams.model",
-                                aspectRatio: ":preprocessor.aspectRatio",
                                 duration: ":beat.duration",
+                                canvasSize: ":context.studio.script.canvasSize",
                             },
                         },
                         defaultValue: {},
@@ -156,8 +138,8 @@ const graph_data = {
                         agent: "copyAgent",
                         inputs: {
                             onComplete: ":movieGenerator",
-                            imageFile: ":preprocessor.path",
-                            movieFile: ":prepareMovie.movieFile",
+                            imageFile: ":preprocessor.imagePath",
+                            movieFile: ":preprocessor.movieFile",
                         },
                         isResult: true,
                     },
@@ -180,7 +162,7 @@ const graph_data = {
                 context: ":context",
             },
         },
-        writeOutout: {
+        writeOutput: {
             // console: { before: true },
             agent: "fileWriteAgent",
             inputs: {
@@ -191,14 +173,20 @@ const graph_data = {
     },
 };
 const googleAuth = async () => {
-    const auth = new GoogleAuth({
-        scopes: ["https://www.googleapis.com/auth/cloud-platform"],
-    });
-    const client = await auth.getClient();
-    const accessToken = await client.getAccessToken();
-    return accessToken.token;
+    try {
+        const auth = new GoogleAuth({
+            scopes: ["https://www.googleapis.com/auth/cloud-platform"],
+        });
+        const client = await auth.getClient();
+        const accessToken = await client.getAccessToken();
+        return accessToken.token;
+    }
+    catch (__error) {
+        GraphAILogger.info("install gcloud and run 'gcloud auth application-default login'");
+        process.exit(1);
+    }
 };
-const generateImages = async (context) => {
+const generateImages = async (context, callbacks) => {
     const { studio, fileDirs } = context;
     const { outDirPath, imageDirPath } = fileDirs;
     mkdir(`${imageDirPath}/${studio.filename}`);
@@ -266,12 +254,17 @@ const generateImages = async (context) => {
     Object.keys(injections).forEach((key) => {
         graph.injectValue(key, injections[key]);
     });
+    if (callbacks) {
+        callbacks.forEach((callback) => {
+            graph.registerCallback(callback);
+        });
+    }
     await graph.run();
 };
-export const images = async (context) => {
+export const images = async (context, callbacks) => {
     try {
         MulmoStudioMethods.setSessionState(context.studio, "image", true);
-        await generateImages(context);
+        await generateImages(context, callbacks);
     }
     finally {
         MulmoStudioMethods.setSessionState(context.studio, "image", false);

package/lib/actions/movie.js

@@ -61,8 +61,8 @@ const getOutputOption = (audioId) => {
 const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, caption) => {
     const start = performance.now();
     const ffmpegContext = FfmpegContextInit();
-    if (studio.beats.some((beat) => !beat.imageFile)) {
-        GraphAILogger.info("beat.imageFile is not set. Please run `yarn run images ${file}` ");
+    if (studio.beats.some((beat) => !beat.imageFile && !beat.movieFile)) {
+        GraphAILogger.info("beat.imageFile or beat.movieFile is not set. Please run `yarn run images ${file}` ");
         return;
     }
     const canvasInfo = MulmoScriptMethods.getCanvasSize(studio.script);
@@ -71,10 +71,14 @@ const createVideo = async (audioArtifactFilePath, outputVideoPath, studio, capti
     const filterComplexAudioIds = [];
     studio.beats.reduce((timestamp, studioBeat, index) => {
         const beat = studio.script.beats[index];
-        if (!studioBeat.imageFile || !studioBeat.duration) {
-            throw new Error(`studioBeat.imageFile or studioBeat.duration is not set: index=${index}`);
+        const sourceFile = studioBeat.movieFile ?? studioBeat.imageFile;
+        if (!sourceFile) {
+            throw new Error(`studioBeat.imageFile or studioBeat.movieFile is not set: index=${index}`);
         }
-        const inputIndex = FfmpegContextAddInput(ffmpegContext, studioBeat.movieFile ?? studioBeat.imageFile);
+        if (!studioBeat.duration) {
+            throw new Error(`studioBeat.duration is not set: index=${index}`);
+        }
+        const inputIndex = FfmpegContextAddInput(ffmpegContext, sourceFile);
         const mediaType = studioBeat.movieFile ? "movie" : MulmoScriptMethods.getImageType(studio.script, beat);
         const extraPadding = (() => {
             // We need to consider only intro and outro padding because the other paddings were already added to the beat.duration

package/lib/actions/translate.d.ts

@@ -1,3 +1,4 @@
 import "dotenv/config";
+import type { CallbackFunction } from "graphai";
 import { MulmoStudioContext } from "../types/index.js";
-export declare const translate: (context: MulmoStudioContext) => Promise<void>;
+export declare const translate: (context: MulmoStudioContext, callbacks?: CallbackFunction[]) => Promise<void>;

package/lib/actions/translate.js

@@ -208,7 +208,7 @@ const agentFilters = [
 ];
 const defaultLang = "en";
 const targetLangs = ["ja", "en"];
-export const translate = async (context) => {
+export const translate = async (context, callbacks) => {
     try {
         MulmoStudioMethods.setSessionState(context.studio, "multiLingual", true);
         const { studio, fileDirs } = context;
@@ -222,6 +222,11 @@ export const translate = async (context) => {
         graph.injectValue("targetLangs", targetLangs);
         graph.injectValue("outDirPath", outDirPath);
         graph.injectValue("outputStudioFilePath", outputStudioFilePath);
+        if (callbacks) {
+            callbacks.forEach((callback) => {
+                graph.registerCallback(callback);
+            });
+        }
         const results = await graph.run();
         writingMessage(outputStudioFilePath);
         if (results.mergeStudioResult) {

package/lib/agents/combine_audio_files_agent.js

@@ -26,11 +26,15 @@ const combineAudioFilesAgent = async ({ namedInputs, }) => {
             const totalPadding = await (async () => {
                 if (beat.image?.type === "movie" && (beat.image.source.kind === "url" || beat.image.source.kind === "path")) {
                     const pathOrUrl = beat.image.source.kind === "url" ? beat.image.source.url : beat.image.source.path;
+                    // NOTE: We respect the duration of the movie, only if the movie is specified as a madia source, NOT generated.
                     const movieDuration = await ffmpegGetMediaDuration(pathOrUrl);
                     if (movieDuration > audioDuration) {
                         return padding + (movieDuration - audioDuration);
                     }
                 }
+                else if (beat.duration && beat.duration > audioDuration) {
+                    return padding + (beat.duration - audioDuration);
+                }
                 return padding;
             })();
             studioBeat.duration = audioDuration + totalPadding;

package/lib/agents/image_google_agent.d.ts

@@ -5,7 +5,10 @@ export type ImageGoogleConfig = {
 };
 export declare const imageGoogleAgent: AgentFunction<{
     model: string;
-    aspectRatio: string;
+    canvasSize: {
+        width: number;
+        height: number;
+    };
 }, {
     buffer: Buffer;
 }, {

package/lib/agents/image_google_agent.js

@@ -1,4 +1,5 @@
 import { GraphAILogger } from "graphai";
+import { getAspectRatio } from "./movie_google_agent.js";
 async function generateImage(projectId, model, token, prompt, aspectRatio) {
     const GOOGLE_IMAGEN_ENDPOINT = `https://us-central1-aiplatform.googleapis.com/v1/projects/${projectId}/locations/us-central1/publishers/google/models/${model}:predict`;
     try {
@@ -50,9 +51,9 @@ async function generateImage(projectId, model, token, prompt, aspectRatio) {
         throw error;
     }
 }
-export const imageGoogleAgent = async ({ namedInputs, params, config, }) => {
+export const imageGoogleAgent = async ({ namedInputs, params, config }) => {
     const { prompt } = namedInputs;
-    const aspectRatio = params.aspectRatio ?? "16:9";
+    const aspectRatio = getAspectRatio(params.canvasSize);
     const model = params.model ?? "imagen-3.0-fast-generate-001";
     //const projectId = process.env.GOOGLE_PROJECT_ID; // Your Google Cloud Project ID
     const projectId = config?.projectId;

package/lib/agents/image_openai_agent.d.ts

@@ -1,16 +1,18 @@
 import { AgentFunction, AgentFunctionInfo } from "graphai";
-type OpenAIImageSize = "1792x1024" | "auto" | "1024x1024" | "1536x1024" | "1024x1536" | "256x256";
 type OpenAIModeration = "low" | "auto";
 export declare const imageOpenaiAgent: AgentFunction<{
     apiKey: string;
     model: string;
-    size: OpenAIImageSize | null | undefined;
     moderation: OpenAIModeration | null | undefined;
-    images: string[] | null | undefined;
+    canvasSize: {
+        width: number;
+        height: number;
+    };
 }, {
     buffer: Buffer;
 }, {
     prompt: string;
+    images: string[] | null | undefined;
 }>;
 declare const imageOpenaiAgentInfo: AgentFunctionInfo;
 export default imageOpenaiAgentInfo;

package/lib/agents/image_openai_agent.js

@@ -2,14 +2,39 @@ import fs from "fs";
 import OpenAI, { toFile } from "openai";
 // https://platform.openai.com/docs/guides/image-generation
 export const imageOpenaiAgent = async ({ namedInputs, params }) => {
-    const { prompt } = namedInputs;
-    const { apiKey, model, size, moderation, images } = params;
+    const { prompt, images } = namedInputs;
+    const { apiKey, moderation, canvasSize } = params;
+    const model = params.model ?? "dall-e-3";
     const openai = new OpenAI({ apiKey });
+    const size = (() => {
+        if (model === "gpt-image-1") {
+            if (canvasSize.width > canvasSize.height) {
+                return "1536x1024";
+            }
+            else if (canvasSize.width < canvasSize.height) {
+                return "1024x1536";
+            }
+            else {
+                return "1024x1024";
+            }
+        }
+        else {
+            if (canvasSize.width > canvasSize.height) {
+                return "1792x1024";
+            }
+            else if (canvasSize.width < canvasSize.height) {
+                return "1024x1792";
+            }
+            else {
+                return "1024x1024";
+            }
+        }
+    })();
     const imageOptions = {
-        model: model ?? "dall-e-3",
+        model,
         prompt,
         n: 1,
-        size: size ?? (model === "gpt-image-1" ? "1536x1024" : "1792x1024"),
+        size,
     };
     if (model === "gpt-image-1") {
         imageOptions.moderation = moderation || "auto";

package/lib/agents/movie_google_agent.d.ts

@@ -3,15 +3,22 @@ export type MovieGoogleConfig = {
     projectId?: string;
     token?: string;
 };
+export declare const getAspectRatio: (canvasSize: {
+    width: number;
+    height: number;
+}) => string;
 export declare const movieGoogleAgent: AgentFunction<{
     model: string;
-    aspectRatio: string;
+    canvasSize: {
+        width: number;
+        height: number;
+    };
     duration?: number;
 }, {
     buffer: Buffer;
 }, {
     prompt: string;
-    imagePath: string;
+    imagePath?: string;
 }, MovieGoogleConfig>;
 declare const movieGoogleAgentInfo: AgentFunctionInfo;
 export default movieGoogleAgentInfo;

package/lib/agents/movie_google_agent.js

@@ -2,26 +2,29 @@ import { readFileSync } from "fs";
 import { GraphAILogger, sleep } from "graphai";
 async function generateMovie(projectId, model, token, prompt, imagePath, aspectRatio, duration) {
     const GOOGLE_IMAGEN_ENDPOINT = `https://us-central1-aiplatform.googleapis.com/v1/projects/${projectId}/locations/us-central1/publishers/google/models/${model}`;
-    // Prepare the payload for the API request
-    const buffer = readFileSync(imagePath);
-    const bytesBase64Encoded = buffer.toString("base64");
     const payload = {
         instances: [
             {
                 prompt: prompt,
-                image: {
-                    bytesBase64Encoded,
-                    mimeType: "image/png",
-                },
+                image: undefined,
             },
         ],
         parameters: {
             sampleCount: 1,
             aspectRatio: aspectRatio,
-            //safetySetting: "block_only_high",
+            safetySetting: "block_only_high",
+            personGeneration: "allow_all",
             durationSeconds: duration,
         },
     };
+    if (imagePath) {
+        const buffer = readFileSync(imagePath);
+        const bytesBase64Encoded = buffer.toString("base64");
+        payload.instances[0].image = {
+            bytesBase64Encoded,
+            mimeType: "image/png",
+        };
+    }
     // Make the API call using fetch
     const response = await fetch(`${GOOGLE_IMAGEN_ENDPOINT}:predictLongRunning`, {
         method: "POST",
@@ -32,6 +35,7 @@ async function generateMovie(projectId, model, token, prompt, imagePath, aspectR
         body: JSON.stringify(payload),
     });
     if (!response.ok) {
+        GraphAILogger.info("create project on google cloud console and setup the project. More details see readme.");
         throw new Error(`Error: ${response.status} - ${response.statusText}`);
     }
     const initialResponse = await response.json();
@@ -72,18 +76,22 @@ async function generateMovie(projectId, model, token, prompt, imagePath, aspectR
     }
     return undefined;
 }
+export const getAspectRatio = (canvasSize) => {
+    if (canvasSize.width > canvasSize.height) {
+        return "16:9";
+    }
+    else if (canvasSize.width < canvasSize.height) {
+        return "9:16";
+    }
+    else {
+        return "1:1";
+    }
+};
 export const movieGoogleAgent = async ({ namedInputs, params, config }) => {
     const { prompt, imagePath } = namedInputs;
-    /*
-    if (prompt) {
-      const buffer = Buffer.from(prompt);
-      return { buffer };
-    }
-    */
-    const aspectRatio = params.aspectRatio ?? "16:9";
+    const aspectRatio = getAspectRatio(params.canvasSize);
     const model = params.model ?? "veo-2.0-generate-001"; // "veo-3.0-generate-preview";
     const duration = params.duration ?? 8;
-    //const projectId = process.env.GOOGLE_PROJECT_ID; // Your Google Cloud Project ID
     const projectId = config?.projectId;
     const token = config?.token;
     try {

package/lib/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./actions/index.js";
+export * from "./cli/helpers.js";
+export * from "./utils/file.js";
+export * from "./methods/index.js";
+export * from "./types/index.js";

package/lib/index.js

@@ -0,0 +1,5 @@
+export * from "./actions/index.js";
+export * from "./cli/helpers.js";
+export * from "./utils/file.js";
+export * from "./methods/index.js";
+export * from "./types/index.js";

package/lib/methods/mulmo_script.d.ts

@@ -2,7 +2,6 @@ import "dotenv/config";
 import { MulmoCanvasDimension, MulmoScript, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, BeatMediaType } from "../types/index.js";
 export declare const MulmoScriptMethods: {
     getCanvasSize(script: MulmoScript): MulmoCanvasDimension;
-    getAspectRatio(script: MulmoScript): string;
     getSpeechProvider(script: MulmoScript): Text2SpeechProvider;
     getTextSlideStyle(script: MulmoScript, beat: MulmoBeat): string;
     getSpeechOptions(script: MulmoScript, beat: MulmoBeat): SpeechOptions | undefined;

package/lib/methods/mulmo_script.js

@@ -18,11 +18,6 @@ export const MulmoScriptMethods = {
     getCanvasSize(script) {
         return mulmoCanvasDimensionSchema.parse(script.canvasSize);
     },
-    getAspectRatio(script) {
-        // Google's text2image specific parameter
-        const size = this.getCanvasSize(script);
-        return size.width > size.height ? "16:9" : "9:16";
-    },
     getSpeechProvider(script) {
         return text2SpeechProviderSchema.parse(script.speechParams?.provider);
     },

package/lib/types/index.d.ts

@@ -1,2 +1,3 @@
 export * from "./type.js";
 export * from "./schema.js";
+export * from "./cli_types.js";

package/lib/types/index.js

@@ -1,2 +1,3 @@
 export * from "./type.js";
 export * from "./schema.js";
+export * from "./cli_types.js";