mulmocast 1.1.11 → 1.2.0

package/assets/templates/html.json

@@ -2,5 +2,16 @@
   "title": "Business presentation in HTML",
   "description": "Template for business presentation in HTML.",
   "systemPrompt": "Generate a script for a business presentation of the given topic. Another LLM will generate actual slides from the prompt and data for each beat. Adding optional data would help it to generate more compelling slide. Mention the reference in one of beats, if it exists. The valid type of reference is 'article', 'paper', 'image', 'video', 'audio'. Use the JSON below as a template.",
+  "presentationStyle": {
+    "$mulmocast": {
+      "version": "1.1",
+      "credit": "closing"
+    },
+    "lang": "en",
+    "canvasSize": {
+      "width": 1536,
+      "height": 1024
+    }
+  },
   "scriptName": "html.json"
 }

package/lib/actions/image_agents.d.ts

@@ -8,7 +8,7 @@ export declare const imagePreprocessAgent: (namedInputs: {
     imagePath: string;
     htmlPrompt: string | undefined;
     htmlPath: string;
-    htmlImageSystemPrompt: string[];
+    htmlImageSystemPrompt: string;
 } | {
     imagePath: string | undefined;
     referenceImageForMovie: string | undefined;

package/lib/actions/image_references.js

@@ -4,7 +4,7 @@ import { getReferenceImagePath } from "../utils/file.js";
 import { getExtention } from "../utils/utils.js";
 import { graphOption } from "./images.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
-import { imageGoogleAgent, imageOpenaiAgent, mediaMockAgent } from "../agents/index.js";
+import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent } from "../agents/index.js";
 // public api
 // Application may call this function directly to generate reference image.
 export const generateReferenceImage = async (inputs) => {
@@ -39,7 +39,7 @@ export const generateReferenceImage = async (inputs) => {
         },
     };
     const options = await graphOption(context);
-    const graph = new GraphAI(image_graph_data, { imageGoogleAgent, imageOpenaiAgent, mediaMockAgent }, options);
+    const graph = new GraphAI(image_graph_data, { imageGenAIAgent, imageOpenaiAgent, mediaMockAgent }, options);
     await graph.run();
     return imagePath;
 };

package/lib/actions/images.js

@@ -1,26 +1,25 @@
 import dotenv from "dotenv";
 import fs from "fs";
 import { GraphAI, GraphAILogger, TaskManager } from "graphai";
-import { GoogleAuth } from "google-auth-library";
 import * as vanilla from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { imageGoogleAgent, imageOpenaiAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, } from "../agents/index.js";
+import { imageGenAIAgent, imageOpenaiAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, } from "../agents/index.js";
 import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
 import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
 import { fileCacheAgentFilter } from "../utils/filters.js";
-import { userAssert, settings2GraphAIConfig } from "../utils/utils.js";
+import { settings2GraphAIConfig } from "../utils/utils.js";
 import { extractImageFromMovie, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
 import { getImageRefs } from "./image_references.js";
 import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent } from "./image_agents.js";
 const vanillaAgents = vanilla.default ?? vanilla;
 const imageAgents = {
-    imageGoogleAgent,
+    imageGenAIAgent,
     imageOpenaiAgent,
 };
 const movieAgents = {
-    movieGoogleAgent,
+    movieGenAIAgent,
     movieReplicateAgent,
     mediaMockAgent,
 };
@@ -151,6 +150,7 @@ const beat_graph_data = {
                 onComplete: [":imageGenerator", ":imagePlugin"], // to wait for imageGenerator to finish
                 prompt: ":beat.moviePrompt",
                 imagePath: ":preprocessor.referenceImageForMovie",
+                movieFile: ":preprocessor.movieFile", // for google genai agent
                 cache: {
                     force: [":context.force", ":forceMovie"],
                     file: ":preprocessor.movieFile",
@@ -331,20 +331,6 @@ const graph_data = {
         },
     },
 };
-const googleAuth = async () => {
-    try {
-        const auth = new GoogleAuth({
-            scopes: ["https://www.googleapis.com/auth/cloud-platform"],
-        });
-        const client = await auth.getClient();
-        const accessToken = await client.getAccessToken();
-        return accessToken.token;
-    }
-    catch (error) {
-        GraphAILogger.info("install gcloud and run 'gcloud auth application-default login'");
-        throw error;
-    }
-};
 export const graphOption = async (context, settings) => {
     const options = {
         agentFilters: [
@@ -356,17 +342,7 @@ export const graphOption = async (context, settings) => {
         ],
         taskManager: new TaskManager(MulmoPresentationStyleMethods.getConcurrency(context.presentationStyle)),
     };
-    const provider = MulmoPresentationStyleMethods.getText2ImageProvider(context.presentationStyle.imageParams?.provider);
-    const config = settings2GraphAIConfig(settings, process.env);
-    // We need to get google's auth token only if the google is the text2image provider.
-    if (provider === "google" || context.presentationStyle.movieParams?.provider === "google") {
-        userAssert(!!config.movieGoogleAgent || !!config.imageGoogleAgent, "GOOGLE_PROJECT_ID is not set");
-        GraphAILogger.log("google was specified as text2image engine");
-        const token = await googleAuth();
-        config["imageGoogleAgent"].token = token;
-        config["movieGoogleAgent"].token = token;
-    }
-    options.config = config;
+    options.config = settings2GraphAIConfig(settings, process.env);
     return options;
 };
 const prepareGenerateImages = async (context) => {

package/lib/agents/image_genai_agent.d.ts

@@ -0,0 +1,5 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { AgentBufferResult, ImageAgentInputs, ImageAgentParams, GenAIImageAgentConfig } from "../types/agent.js";
+export declare const imageGenAIAgent: AgentFunction<ImageAgentParams, AgentBufferResult, ImageAgentInputs, GenAIImageAgentConfig>;
+declare const imageGenAIAgentInfo: AgentFunctionInfo;
+export default imageGenAIAgentInfo;

package/lib/agents/image_genai_agent.js

@@ -0,0 +1,52 @@
+import { GraphAILogger } from "graphai";
+import { getAspectRatio } from "./movie_google_agent.js";
+import { provider2ImageAgent } from "../utils/provider2agent.js";
+import { GoogleGenAI, PersonGeneration } from "@google/genai";
+export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
+    const { prompt } = namedInputs;
+    const aspectRatio = getAspectRatio(params.canvasSize);
+    const model = params.model ?? provider2ImageAgent["google"].defaultModel;
+    const apiKey = config?.apiKey;
+    if (!apiKey) {
+        throw new Error("API key is required for Google GenAI agent");
+    }
+    try {
+        const ai = new GoogleGenAI({ apiKey });
+        const response = await ai.models.generateImages({
+            model,
+            prompt,
+            config: {
+                numberOfImages: 1, // default is 4!
+                aspectRatio,
+                personGeneration: PersonGeneration.ALLOW_ALL,
+                // safetyFilterLevel: SafetyFilterLevel.BLOCK_ONLY_HIGH,
+            },
+        });
+        if (!response.generatedImages || response.generatedImages.length === 0) {
+            throw new Error("ERROR: generateImage returned no generated images");
+        }
+        const image = response.generatedImages[0].image;
+        if (image && image.imageBytes) {
+            return { buffer: Buffer.from(image.imageBytes, "base64") };
+        }
+        throw new Error("ERROR: generateImage returned no image bytes");
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate image:", error);
+        throw error;
+    }
+};
+const imageGenAIAgentInfo = {
+    name: "imageGenAIAgent",
+    agent: imageGenAIAgent,
+    mock: imageGenAIAgent,
+    samples: [],
+    description: "Google Image agent",
+    category: ["image"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    // source: "https://github.com/receptron/mulmocast-cli/blob/main/src/agents/image_google_agent.ts",
+    license: "MIT",
+    environmentVariables: [],
+};
+export default imageGenAIAgentInfo;

package/lib/agents/index.d.ts

@@ -1,9 +1,9 @@
 import addBGMAgent from "./add_bgm_agent.js";
 import combineAudioFilesAgent from "./combine_audio_files_agent.js";
-import imageGoogleAgent from "./image_google_agent.js";
+import imageGenAIAgent from "./image_genai_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
 import tavilySearchAgent from "./tavily_agent.js";
-import movieGoogleAgent from "./movie_google_agent.js";
+import movieGenAIAgent from "./movie_genai_agent.js";
 import movieReplicateAgent from "./movie_replicate_agent.js";
 import mediaMockAgent from "./media_mock_agent.js";
 import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
@@ -17,4 +17,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };

package/lib/agents/index.js

@@ -1,9 +1,9 @@
 import addBGMAgent from "./add_bgm_agent.js";
 import combineAudioFilesAgent from "./combine_audio_files_agent.js";
-import imageGoogleAgent from "./image_google_agent.js";
+import imageGenAIAgent from "./image_genai_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
 import tavilySearchAgent from "./tavily_agent.js";
-import movieGoogleAgent from "./movie_google_agent.js";
+import movieGenAIAgent from "./movie_genai_agent.js";
 import movieReplicateAgent from "./movie_replicate_agent.js";
 import mediaMockAgent from "./media_mock_agent.js";
 import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
@@ -18,4 +18,4 @@ import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, tavilySearchAgent, movieGoogleAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, };

package/lib/agents/movie_genai_agent.d.ts

@@ -0,0 +1,9 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { AgentBufferResult, GenAIImageAgentConfig, GoogleMovieAgentParams, MovieAgentInputs } from "../types/agent.js";
+export declare const getAspectRatio: (canvasSize: {
+    width: number;
+    height: number;
+}) => string;
+export declare const movieGenAIAgent: AgentFunction<GoogleMovieAgentParams, AgentBufferResult, MovieAgentInputs, GenAIImageAgentConfig>;
+declare const movieGenAIAgentInfo: AgentFunctionInfo;
+export default movieGenAIAgentInfo;

package/lib/agents/movie_genai_agent.js

@@ -0,0 +1,86 @@
+import { readFileSync } from "fs";
+import { GraphAILogger, sleep } from "graphai";
+import { GoogleGenAI, PersonGeneration } from "@google/genai";
+export const getAspectRatio = (canvasSize) => {
+    if (canvasSize.width > canvasSize.height) {
+        return "16:9";
+    }
+    else if (canvasSize.width < canvasSize.height) {
+        return "9:16";
+    }
+    else {
+        return "1:1";
+    }
+};
+export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
+    const { prompt, imagePath, movieFile } = namedInputs;
+    const aspectRatio = getAspectRatio(params.canvasSize);
+    const model = params.model ?? "veo-2.0-generate-001"; // "veo-3.0-generate-preview";
+    const duration = params.duration ?? 8;
+    const apiKey = config?.apiKey;
+    if (!apiKey) {
+        throw new Error("API key is required for Google GenAI agent");
+    }
+    try {
+        const ai = new GoogleGenAI({ apiKey });
+        const payload = {
+            model,
+            prompt,
+            config: {
+                durationSeconds: duration,
+                aspectRatio,
+                personGeneration: undefined,
+            },
+            image: undefined,
+        };
+        if (imagePath) {
+            const buffer = readFileSync(imagePath);
+            const imageBytes = buffer.toString("base64");
+            payload.image = {
+                imageBytes,
+                mimeType: "image/png",
+            };
+        }
+        else {
+            payload.config.personGeneration = PersonGeneration.ALLOW_ALL;
+        }
+        const operation = await ai.models.generateVideos(payload);
+        const response = { operation };
+        // Poll the operation status until the video is ready.
+        while (!response.operation.done) {
+            await sleep(5000);
+            response.operation = await ai.operations.getVideosOperation(response);
+        }
+        if (!response.operation.response?.generatedVideos) {
+            throw new Error(`No video: ${JSON.stringify(response.operation, null, 2)}`);
+        }
+        const video = response.operation.response.generatedVideos[0].video;
+        if (!video) {
+            throw new Error(`No video: ${JSON.stringify(response.operation, null, 2)}`);
+        }
+        await ai.files.download({
+            file: video,
+            downloadPath: movieFile,
+        });
+        await sleep(5000); // HACK: Without this, the file is not ready yet.
+        return { saved: movieFile };
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate movie:", error.message);
+        throw error;
+    }
+};
+const movieGenAIAgentInfo = {
+    name: "movieGenAIAgent",
+    agent: movieGenAIAgent,
+    mock: movieGenAIAgent,
+    samples: [],
+    description: "Google Movie agent",
+    category: ["movie"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    // source: "https://github.com/receptron/mulmocast-cli/blob/main/src/agents/image_google_agent.ts",
+    license: "MIT",
+    environmentVariables: [],
+};
+export default movieGenAIAgentInfo;

package/lib/cli/helpers.js

@@ -37,9 +37,11 @@ export const getFileObject = (args) => {
             // We generate a new unique script file from clipboard text in the output directory
             const generatedFileName = generateTimestampedFileName("script");
             const clipboardText = clipboardy.readSync();
+            const json = JSON.parse(clipboardText);
+            const formattedText = JSON.stringify(json, null, 2);
             const resolvedFilePath = resolveDirPath(outDirPath, `${generatedFileName}.json`);
             mkdir(outDirPath);
-            fs.writeFileSync(resolvedFilePath, clipboardText, "utf8");
+            fs.writeFileSync(resolvedFilePath, formattedText, "utf8");
             return { fileOrUrl: resolvedFilePath, fileName: generatedFileName };
         }
         const resolvedFileOrUrl = file ?? "";

package/lib/methods/mulmo_beat.js

@@ -2,7 +2,7 @@ import { findImagePlugin } from "../utils/image_plugins/index.js";
 export const MulmoBeatMethods = {
     getHtmlPrompt(beat) {
         if (beat?.htmlPrompt?.data) {
-            return beat.htmlPrompt.prompt + "\n\n data\n" + JSON.stringify(beat.htmlPrompt.data, null, 2);
+            return beat.htmlPrompt.prompt + "\n\n[data]\n" + JSON.stringify(beat.htmlPrompt.data, null, 2);
         }
         return beat?.htmlPrompt?.prompt;
     },

package/lib/types/agent.d.ts

@@ -10,7 +10,9 @@ export type OpenAIImageOptions = {
     quality?: OpenAIImageQuality;
 };
 export type AgentBufferResult = {
-    buffer: Buffer;
+    buffer?: Buffer;
+    saved?: string;
+    text?: string;
 };
 export type AgentPromptInputs = {
     prompt: string;
@@ -47,8 +49,12 @@ export type GoogleImageAgentConfig = {
     projectId?: string;
     token?: string;
 };
+export type GenAIImageAgentConfig = {
+    apiKey?: string;
+};
 export type MovieAgentInputs = AgentPromptInputs & {
     imagePath?: string;
+    movieFile: string;
 };
 export type GoogleMovieAgentParams = ImageAgentParams & {
     duration?: number;