mulmocast 0.0.22 → 0.0.24

package/lib/agents/movie_replicate_agent.js

@@ -0,0 +1,93 @@
+import { readFileSync } from "fs";
+import { GraphAILogger } from "graphai";
+import Replicate from "replicate";
+async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, duration) {
+    const replicate = new Replicate({
+        auth: apiKey,
+    });
+    const input = {
+        prompt: prompt,
+        duration: duration,
+        image: undefined,
+        start_image: undefined,
+        aspect_ratio: aspectRatio, // only for bytedance/seedance-1-lite
+        // resolution: "720p", // only for bytedance/seedance-1-lite
+        // fps: 24, // only for bytedance/seedance-1-lite
+        // camera_fixed: false, // only for bytedance/seedance-1-lite
+        // mode: "standard" // only for kwaivgi/kling-v2.1
+        // negative_prompt: "" // only for kwaivgi/kling-v2.1
+    };
+    // Add image if provided (for image-to-video generation)
+    if (imagePath) {
+        const buffer = readFileSync(imagePath);
+        const base64Image = `data:image/png;base64,${buffer.toString("base64")}`;
+        if (model === "kwaivgi/kling-v2.1") {
+            input.start_image = base64Image;
+        }
+        else {
+            input.image = base64Image;
+        }
+    }
+    try {
+        const output = await replicate.run(model ?? "bytedance/seedance-1-lite", { input });
+        // Download the generated video
+        if (output && typeof output === "object" && "url" in output) {
+            const videoUrl = output.url();
+            const videoResponse = await fetch(videoUrl);
+            if (!videoResponse.ok) {
+                throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`);
+            }
+            const arrayBuffer = await videoResponse.arrayBuffer();
+            return Buffer.from(arrayBuffer);
+        }
+        return undefined;
+    }
+    catch (error) {
+        GraphAILogger.info("Replicate generation error:", error);
+        throw error;
+    }
+}
+export const getAspectRatio = (canvasSize) => {
+    if (canvasSize.width > canvasSize.height) {
+        return "16:9";
+    }
+    else if (canvasSize.width < canvasSize.height) {
+        return "9:16";
+    }
+    else {
+        return "1:1";
+    }
+};
+export const movieReplicateAgent = async ({ namedInputs, params, config }) => {
+    const { prompt, imagePath } = namedInputs;
+    const aspectRatio = getAspectRatio(params.canvasSize);
+    const duration = params.duration ?? 5;
+    const apiKey = config?.apiKey ?? process.env.REPLICATE_API_TOKEN;
+    if (!apiKey) {
+        throw new Error("REPLICATE_API_TOKEN environment variable is required");
+    }
+    try {
+        const buffer = await generateMovie(params.model, apiKey, prompt, imagePath, aspectRatio, duration);
+        if (buffer) {
+            return { buffer };
+        }
+        throw new Error("ERROR: generateMovie returned undefined");
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate movie:", error.message);
+        throw error;
+    }
+};
+const movieReplicateAgentInfo = {
+    name: "movieReplicateAgent",
+    agent: movieReplicateAgent,
+    mock: movieReplicateAgent,
+    samples: [],
+    description: "Replicate Movie agent using seedance-1-lite",
+    category: ["movie"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["REPLICATE_API_TOKEN"],
+};
+export default movieReplicateAgentInfo;

package/lib/agents/tts_elevenlabs_agent.js

@@ -1,8 +1,8 @@
 import { GraphAILogger } from "graphai";
-export const ttsElevenlabsAgent = async ({ namedInputs, params }) => {
+export const ttsElevenlabsAgent = async ({ namedInputs, params, config }) => {
     const { text } = namedInputs;
     const { voice, model, stability, similarityBoost, suppressError } = params;
-    const apiKey = process.env.ELEVENLABS_API_KEY;
+    const apiKey = config?.apiKey ?? process.env.ELEVENLABS_API_KEY;
     if (!apiKey) {
         throw new Error("ELEVENLABS_API_KEY environment variable is required");
     }

package/lib/agents/tts_nijivoice_agent.js

@@ -6,8 +6,9 @@ const errorMessage = [
     "1. Obtain an API key from Niji Voice (https://platform.nijivoice.com/) and set it as the NIJIVOICE_API_KEY environment variable.",
     '2. Use OpenAI\'s TTS instead of Niji Voice by changing speechParams.provider from "nijivoice" to "openai".',
 ].join("\n");
-export const ttsNijivoiceAgent = async ({ params, namedInputs }) => {
-    const { apiKey, suppressError, voice, speed, speed_global } = params;
+export const ttsNijivoiceAgent = async ({ params, namedInputs, config }) => {
+    const { suppressError, voice, speed, speed_global } = params;
+    const { apiKey } = config ?? {};
     const { text } = namedInputs;
     assert(apiKey ?? nijovoiceApiKey, errorMessage);
     const url = `https://api.nijivoice.com/api/platform/v1/voice-actors/${voice}/generate-voice`;

package/lib/agents/tts_openai_agent.js

@@ -1,8 +1,9 @@
 import { GraphAILogger } from "graphai";
 import OpenAI from "openai";
-export const ttsOpenaiAgent = async ({ namedInputs, params }) => {
+export const ttsOpenaiAgent = async ({ namedInputs, params, config }) => {
     const { text } = namedInputs;
-    const { apiKey, model, voice, suppressError, instructions } = params;
+    const { model, voice, suppressError, instructions } = params;
+    const { apiKey } = config ?? {};
     const openai = new OpenAI({ apiKey });
     try {
         const tts_options = {

package/lib/cli/commands/tool/scripting/builder.d.ts

@@ -5,6 +5,8 @@ export declare const builder: (yargs: Argv) => Argv<{
     b: string | undefined;
 } & {
     u: string[] | never[];
+} & {
+    "input-file": string | undefined;
 } & {
     i: boolean | undefined;
 } & {
@@ -14,7 +16,7 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     s: string;
 } & {
-    llm: "openAI" | "anthropic" | "gemini" | "groq" | undefined;
+    llm: "anthropic" | "openAI" | "gemini" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
 }>;

package/lib/cli/commands/tool/scripting/builder.js

@@ -22,6 +22,11 @@ export const builder = (yargs) => {
         default: [],
         type: "array",
         string: true,
+    })
+        .option("input-file", {
+        description: "input file name",
+        demandOption: false,
+        type: "string",
     })
         .option("i", {
         alias: "interactive",

package/lib/cli/commands/tool/scripting/handler.d.ts

@@ -6,6 +6,7 @@ export declare const handler: (argv: ToolCliArgs<{
     u?: string[];
     i?: boolean;
     t?: string;
+    "input-file"?: string;
     c?: string;
     s?: string;
     llm?: LLM;

package/lib/cli/commands/tool/scripting/handler.js

@@ -1,18 +1,23 @@
 import { getBaseDirPath, getFullPath } from "../../../../utils/file.js";
 import { outDirName, cacheDirName } from "../../../../utils/const.js";
 import { getUrlsIfNeeded, selectTemplate } from "../../../../utils/inquirer.js";
-import { createMulmoScriptFromUrl } from "../../../../tools/create_mulmo_script_from_url.js";
+import { createMulmoScriptFromUrl, createMulmoScriptFromFile } from "../../../../tools/create_mulmo_script_from_url.js";
 import { createMulmoScriptInteractively } from "../../../../tools/create_mulmo_script_interactively.js";
 import { setGraphAILogger } from "../../../../cli/helpers.js";
 export const handler = async (argv) => {
-    const { o: outdir, b: basedir, v: verbose, i: interactive, c: cache, s: filename, llm, llm_model } = argv;
+    const { o: outdir, b: basedir, "input-file": inputFile, v: verbose, i: interactive, c: cache, s: filename, llm, llm_model } = argv;
     let { t: template } = argv;
     const urls = argv.u || [];
     const baseDirPath = getBaseDirPath(basedir);
     const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
     const cacheDirPath = getFullPath(outDirPath, cache ?? cacheDirName);
     if (!template) {
-        template = await selectTemplate();
+        if (interactive) {
+            template = await selectTemplate();
+        }
+        else {
+            template = "business";
+        }
     }
     setGraphAILogger(verbose, {
         baseDirPath,
@@ -22,13 +27,17 @@ export const handler = async (argv) => {
         urls,
         interactive,
         filename,
+        inputFile,
         llm,
         llm_model,
     });
-    const context = { outDirPath, templateName: template, urls, filename: filename, cacheDirPath, llm_model, llm };
+    const context = { outDirPath, templateName: template, urls, filename: filename, cacheDirPath, llm_model, llm, verbose };
     if (interactive) {
         await createMulmoScriptInteractively(context);
     }
+    if (inputFile) {
+        await createMulmoScriptFromFile(inputFile, context);
+    }
     else {
         context.urls = await getUrlsIfNeeded(urls);
         await createMulmoScriptFromUrl(context);

package/lib/cli/commands/tool/story_to_script/builder.d.ts

@@ -10,7 +10,7 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     beats_per_scene: number;
 } & {
-    llm: "openAI" | "anthropic" | "gemini" | "groq" | undefined;
+    llm: "anthropic" | "openAI" | "gemini" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
 } & {

package/lib/cli/helpers.js

@@ -7,7 +7,7 @@ import { isHttp } from "../utils/utils.js";
 import { createOrUpdateStudioData } from "../utils/preprocess.js";
 import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
 import { translate } from "../actions/translate.js";
-import { mulmoPresentationStyleSchema, mulmoStudioMultiLingualSchema } from "../types/schema.js";
+import { mulmoCaptionParamsSchema, mulmoPresentationStyleSchema, mulmoStudioMultiLingualSchema } from "../types/schema.js";
 export const setGraphAILogger = (verbose, logValues) => {
     if (verbose) {
         if (logValues) {
@@ -126,13 +126,18 @@ export const initializeContext = async (argv) => {
         // validate mulmoStudioSchema. skip if __test_invalid__ is true
         const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName);
         const multiLingual = getMultiLingual(outputMultilingualFilePath, studio.beats.length);
+        if (argv.c) {
+            studio.script.captionParams = mulmoCaptionParamsSchema.parse({
+                ...(studio.script.captionParams ?? {}),
+                lang: argv.c,
+            });
+        }
         return {
             studio,
             fileDirs: files,
             force: Boolean(argv.f),
             dryRun: Boolean(argv.dryRun),
             lang: argv.l,
-            caption: argv.c,
             sessionState: {
                 inSession: {
                     audio: false,
@@ -160,7 +165,7 @@ export const initializeContext = async (argv) => {
     }
 };
 export const runTranslateIfNeeded = async (context, argv) => {
-    if (argv.l || argv.c) {
+    if (argv.l || context.studio.script.captionParams?.lang) {
         GraphAILogger.log("run translate");
         await translate(context);
     }

package/lib/methods/mulmo_presentation_style.d.ts

@@ -1,5 +1,5 @@
 import "dotenv/config";
-import { MulmoCanvasDimension, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
     getSpeechProvider(presentationStyle: MulmoPresentationStyle): Text2SpeechProvider;
@@ -10,5 +10,6 @@ export declare const MulmoPresentationStyleMethods: {
     getProvider(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): Text2SpeechProvider;
     getVoiceId(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, dryRun?: boolean): Text2ImageAgentInfo;
+    getHtmlImageAgentInfo(presentationStyle: MulmoPresentationStyle): Text2HtmlAgentInfo;
     getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;
 };

package/lib/methods/mulmo_presentation_style.js

@@ -1,5 +1,6 @@
 import "dotenv/config";
-import { text2ImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
+import { userAssert } from "../utils/utils.js";
+import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
 import { defaultOpenAIImageModel } from "../utils/const.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
@@ -42,7 +43,11 @@ export const MulmoPresentationStyleMethods = {
         return { ...presentationStyle.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };
     },
     getSpeaker(presentationStyle, beat) {
-        return presentationStyle.speechParams.speakers[beat.speaker];
+        userAssert(!!presentationStyle?.speechParams?.speakers, "presentationStyle.speechParams.speakers is not set!!");
+        userAssert(!!beat?.speaker, "beat.speaker is not set");
+        const speaker = presentationStyle.speechParams.speakers[beat.speaker];
+        userAssert(!!speaker, `speaker is not set: speaker "${beat.speaker}"`);
+        return speaker;
     },
     getProvider(presentationStyle, beat) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
@@ -65,6 +70,20 @@ export const MulmoPresentationStyleMethods = {
             imageParams: { ...defaultImageParams, ...presentationStyle.imageParams },
         };
     },
+    getHtmlImageAgentInfo(presentationStyle) {
+        const provider = text2HtmlImageProviderSchema.parse(presentationStyle.htmlImageParams?.provider);
+        const agent = provider === "anthropic" ? "anthropicAgent" : "openAIAgent";
+        const model = presentationStyle.htmlImageParams?.model
+            ? presentationStyle.htmlImageParams?.model
+            : provider === "anthropic"
+                ? "claude-3-7-sonnet-20250219"
+                : "gpt-4o-mini";
+        return {
+            provider,
+            agent,
+            model,
+        };
+    },
     getImageType(_, beat) {
         return beat.image?.type == "movie" ? "movie" : "image";
     },

package/lib/methods/mulmo_studio_context.js

@@ -44,7 +44,7 @@ export const MulmoStudioContextMethods = {
         return context.studio.filename;
     },
     getCaption(context) {
-        return context.caption;
+        return context.studio.script.captionParams?.lang;
     },
     setSessionState(context, sessionType, value) {
         context.sessionState.inSession[sessionType] = value;

package/lib/tools/create_mulmo_script_from_url.d.ts

@@ -1,3 +1,4 @@
 import "dotenv/config";
 import { ScriptingParams } from "../types/index.js";
 export declare const createMulmoScriptFromUrl: ({ urls, templateName, outDirPath, filename, cacheDirPath, llm, llm_model }: ScriptingParams) => Promise<void>;
+export declare const createMulmoScriptFromFile: (fileName: string, { templateName, outDirPath, filename, cacheDirPath, llm, llm_model, verbose }: ScriptingParams) => Promise<void>;

package/lib/tools/create_mulmo_script_from_url.js

@@ -1,4 +1,5 @@
 import "dotenv/config";
+import path from "path";
 import { GraphAI } from "graphai";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
@@ -14,7 +15,56 @@ import { mulmoScriptSchema, urlsSchema } from "../types/schema.js";
 import { cliLoadingPlugin } from "../utils/plugins.js";
 import { graphDataScriptFromUrlPrompt } from "../utils/prompt.js";
 import { llmPair } from "../utils/utils.js";
+import { readFileSync } from "fs";
 const vanillaAgents = agents.default ?? agents;
+const graphMulmoScript = {
+    version: 0.5,
+    loop: {
+        // If the script is not valid and the counter is less than 3, continue the loop
+        while: ":continue",
+    },
+    nodes: {
+        sourceText: {},
+        systemPrompt: {},
+        llmAgent: {},
+        llmModel: {},
+        maxTokens: {},
+        counter: {
+            value: 0,
+            update: ":counter.add(1)",
+        },
+        llm: {
+            agent: ":llmAgent",
+            // console: { before: true },
+            inputs: {
+                system: ":systemPrompt",
+                prompt: graphDataScriptFromUrlPrompt("${:sourceText.text}"),
+                params: {
+                    model: ":llmModel",
+                    system: ":systemPrompt",
+                    max_tokens: ":maxTokens",
+                },
+            },
+        },
+        validateSchemaAgent: {
+            agent: "validateSchemaAgent",
+            inputs: {
+                text: ":llm.text.codeBlock()",
+                schema: mulmoScriptSchema,
+            },
+            isResult: true,
+        },
+        continue: {
+            agent: ({ isValid, counter }) => {
+                return !isValid && counter < 3;
+            },
+            inputs: {
+                isValid: ":validateSchemaAgent.isValid",
+                counter: ":counter",
+            },
+        },
+    },
+};
 const graphData = {
     version: 0.5,
     // Execute sequentially because the free version of browserless API doesn't support concurrent execution.
@@ -23,7 +73,7 @@ const graphData = {
         urls: {
             value: [],
         },
-        prompt: {
+        systemPrompt: {
             value: "",
         },
         outdir: {
@@ -87,52 +137,60 @@ const graphData = {
             agent: "nestedAgent",
             inputs: {
                 sourceText: ":sourceText",
-                prompt: ":prompt",
+                systemPrompt: ":systemPrompt",
                 llmAgent: ":llmAgent",
                 llmModel: ":llmModel",
                 maxTokens: ":maxTokens",
             },
-            graph: {
-                loop: {
-                    // If the script is not valid and the counter is less than 3, continue the loop
-                    while: ":continue",
-                },
-                nodes: {
-                    counter: {
-                        value: 0,
-                        update: ":counter.add(1)",
-                    },
-                    llm: {
-                        agent: ":llmAgent",
-                        inputs: {
-                            system: ":prompt",
-                            prompt: graphDataScriptFromUrlPrompt("${:sourceText.text}"),
-                            params: {
-                                model: ":llmModel",
-                                system: ":prompt",
-                                max_tokens: ":maxTokens",
-                            },
-                        },
-                    },
-                    validateSchemaAgent: {
-                        agent: "validateSchemaAgent",
-                        inputs: {
-                            text: ":llm.text.codeBlock()",
-                            schema: mulmoScriptSchema,
-                        },
-                        isResult: true,
-                    },
-                    continue: {
-                        agent: ({ isValid, counter }) => {
-                            return !isValid && counter < 3;
-                        },
-                        inputs: {
-                            isValid: ":validateSchemaAgent.isValid",
-                            counter: ":counter",
-                        },
-                    },
-                },
+            graph: graphMulmoScript,
+        },
+        writeJSON: {
+            if: ":mulmoScript.validateSchemaAgent.isValid",
+            agent: "fileWriteAgent",
+            inputs: {
+                file: "${:outdir}/${:fileName}-${@now}.json",
+                text: ":mulmoScript.validateSchemaAgent.data.toJSON()",
+            },
+            isResult: true,
+        },
+    },
+};
+const graphDataText = {
+    version: 0.5,
+    // Execute sequentially because the free version of browserless API doesn't support concurrent execution.
+    concurrency: 1,
+    nodes: {
+        systemPrompt: {
+            value: "",
+        },
+        outdir: {
+            value: "",
+        },
+        fileName: {
+            value: "",
+        },
+        llmAgent: {
+            value: "",
+        },
+        llmModel: {
+            value: "",
+        },
+        maxTokens: {
+            value: 0,
+        },
+        sourceText: {},
+        // generate the mulmo script
+        mulmoScript: {
+            agent: "nestedAgent",
+            // console: { before: true },
+            inputs: {
+                sourceText: ":sourceText",
+                systemPrompt: ":systemPrompt",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
             },
+            graph: graphMulmoScript,
         },
         writeJSON: {
             if: ":mulmoScript.validateSchemaAgent.isValid",
@@ -169,7 +227,7 @@ export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath,
         fileWriteAgent,
     }, { agentFilters });
     graph.injectValue("urls", parsedUrls);
-    graph.injectValue("prompt", readTemplatePrompt(templateName));
+    graph.injectValue("systemPrompt", readTemplatePrompt(templateName));
     graph.injectValue("outdir", outDirPath);
     graph.injectValue("fileName", filename);
     graph.injectValue("llmAgent", agent);
@@ -179,3 +237,31 @@ export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath,
     const result = await graph.run();
     writingMessage(result?.writeJSON?.path ?? "");
 };
+export const createMulmoScriptFromFile = async (fileName, { templateName, outDirPath, filename, cacheDirPath, llm, llm_model, verbose }) => {
+    mkdir(outDirPath);
+    mkdir(cacheDirPath);
+    const filePath = path.resolve(process.cwd(), fileName);
+    const text = readFileSync(filePath, "utf-8");
+    const { agent, model, max_tokens } = llmPair(llm, llm_model);
+    const graph = new GraphAI(graphDataText, {
+        ...vanillaAgents,
+        openAIAgent,
+        anthropicAgent,
+        geminiAgent,
+        groqAgent,
+        validateSchemaAgent,
+        fileWriteAgent,
+    });
+    graph.injectValue("sourceText", { text });
+    graph.injectValue("systemPrompt", readTemplatePrompt(templateName));
+    graph.injectValue("outdir", outDirPath);
+    graph.injectValue("fileName", filename);
+    graph.injectValue("llmAgent", agent);
+    graph.injectValue("llmModel", model);
+    graph.injectValue("maxTokens", max_tokens);
+    if (!verbose) {
+        graph.registerCallback(cliLoadingPlugin({ nodeId: "mulmoScript", message: "Generating script..." }));
+    }
+    const result = await graph.run();
+    writingMessage(result?.writeJSON?.path ?? "");
+};