mulmocast 0.0.2 → 0.0.4

package/lib/cli/cli.js

@@ -3,34 +3,45 @@ import "dotenv/config";
 import fs from "fs";
 import path from "path";
 import { GraphAILogger } from "graphai";
-import { args } from "./args.js";
+import { getArgs } from "./args.js";
 import { createOrUpdateStudioData } from "../utils/preprocess.js";
 import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
-import { MulmoScriptMethods } from "../methods/index.js";
-import { translate } from "../actions/translate.js";
-import { images } from "../actions/images.js";
-import { audio } from "../actions/audio.js";
-import { movie } from "../actions/movie.js";
-import { getBaseDirPath, getFullPath, readMulmoScriptFile, fetchMulmoScriptFile } from "../utils/file.js";
-import { mulmoScriptSchema } from "../types/schema.js";
-const isHttp = (fileOrUrl) => {
-    return /^https?:\/\//.test(fileOrUrl);
-};
-const getFileObject = () => {
-    const { basedir, file, outdir, imagedir, audiodir } = args;
+import { translate, audio, images, movie, pdf } from "../actions/index.js";
+import { getBaseDirPath, getFullPath, readMulmoScriptFile, fetchMulmoScriptFile, getOutputStudioFilePath } from "../utils/file.js";
+import { isHttp } from "../utils/utils.js";
+export const getFileObject = (_args) => {
+    const { basedir, file, outdir, imagedir, audiodir } = _args;
     const baseDirPath = getBaseDirPath(basedir);
     const fileOrUrl = file ?? "";
+    const fileName = path.parse(fileOrUrl).name;
     const isHttpPath = isHttp(fileOrUrl);
     const mulmoFilePath = isHttpPath ? "" : getFullPath(baseDirPath, fileOrUrl);
     const mulmoFileDirPath = path.dirname(isHttpPath ? baseDirPath : mulmoFilePath);
     const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
     const imageDirPath = getFullPath(outDirPath, imagedir ?? imageDirName);
     const audioDirPath = getFullPath(outDirPath, audiodir ?? audioDirName);
-    return { baseDirPath, mulmoFilePath, mulmoFileDirPath, outDirPath, imageDirPath, audioDirPath, isHttpPath, fileOrUrl };
+    const outputStudioFilePath = getOutputStudioFilePath(outDirPath, fileName);
+    return { baseDirPath, mulmoFilePath, mulmoFileDirPath, outDirPath, imageDirPath, audioDirPath, isHttpPath, fileOrUrl, outputStudioFilePath, fileName };
+};
+const fetchScript = async (isHttpPath, mulmoFilePath, fileOrUrl) => {
+    if (isHttpPath) {
+        const res = await fetchMulmoScriptFile(fileOrUrl);
+        if (!res.result || !res.script) {
+            GraphAILogger.info(`ERROR: HTTP error! ${res.status} ${fileOrUrl}`);
+            process.exit(1);
+        }
+        return res.script;
+    }
+    if (!fs.existsSync(mulmoFilePath)) {
+        GraphAILogger.info(`ERROR: File not exists ${mulmoFilePath}`);
+        process.exit(1);
+    }
+    return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath).mulmoData;
 };
-const main = async () => {
-    const files = getFileObject();
-    const { mulmoFilePath, isHttpPath, fileOrUrl } = files;
+export const main = async () => {
+    const args = getArgs();
+    const files = getFileObject(args);
+    const { mulmoFilePath, isHttpPath, fileOrUrl, fileName, outputStudioFilePath } = files;
     if (args.v) {
         GraphAILogger.info(files);
     }
@@ -39,37 +50,20 @@ const main = async () => {
         GraphAILogger.setLevelEnabled("log", false);
         GraphAILogger.setLevelEnabled("warn", false);
     }
-    const { action, force } = args;
-    const readData = await (async () => {
-        if (isHttpPath) {
-            const res = await fetchMulmoScriptFile(fileOrUrl);
-            if (!res.result || !res.script) {
-                GraphAILogger.info(`ERROR: HTTP error! ${res.status} ${fileOrUrl}`);
-                process.exit(1);
-            }
-            return {
-                mulmoData: res.script,
-                fileName: path.parse(fileOrUrl).name,
-            };
+    const { action, force, pdf_mode, pdf_size } = args;
+    const mulmoScript = await fetchScript(isHttpPath, mulmoFilePath, fileOrUrl);
+    // Create or update MulmoStudio file with MulmoScript
+    const currentStudio = readMulmoScriptFile(outputStudioFilePath);
+    const studio = (() => {
+        try {
+            // validate mulmoStudioSchema. skip if __test_invalid__ is true
+            return createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName);
         }
-        if (!fs.existsSync(mulmoFilePath)) {
-            GraphAILogger.info("ERROR: File not exists " + mulmoFilePath);
+        catch (error) {
+            GraphAILogger.info(`Error: invalid MulmoScript Schema: ${isHttpPath ? fileOrUrl : mulmoFilePath} \n ${error}`);
             process.exit(1);
         }
-        return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath);
     })();
-    const { mulmoData: mulmoScript, fileName } = readData;
-    // validate mulmoStudioSchema. skip if __test_invalid__ is true
-    try {
-        if (!mulmoScript?.__test_invalid__) {
-            mulmoScriptSchema.parse(mulmoScript);
-        }
-    }
-    catch (error) {
-        GraphAILogger.info(`Error: invalid MulmoScript Schema: ${isHttpPath ? fileOrUrl : mulmoFilePath} \n ${error}`);
-        process.exit(1);
-    }
-    const studio = createOrUpdateStudioData(mulmoScript, fileName, files);
     const context = {
         studio,
         fileDirs: files,
@@ -79,15 +73,18 @@ const main = async () => {
         await translate(context);
     }
     if (action === "audio") {
-        await audio(context, MulmoScriptMethods.getSpeechProvider(studio.script) === "nijivoice" ? 1 : 8);
+        await audio(context);
     }
     if (action === "images") {
         await images(context);
     }
     if (action === "movie") {
-        await audio(context, MulmoScriptMethods.getSpeechProvider(studio.script) === "nijivoice" ? 1 : 8);
+        await audio(context);
         await images(context);
         await movie(context);
     }
+    if (action === "pdf") {
+        await images(context);
+        await pdf(context, pdf_mode, pdf_size);
+    }
 };
-main();

package/lib/cli/run.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/cli/run.js

@@ -0,0 +1,2 @@
+import { main } from "./cli.js";
+main();

package/lib/cli/tool-args.d.ts

@@ -8,6 +8,8 @@ export declare const args: {
     t: string | undefined;
     c: string | undefined;
     s: string;
+    llm_agent: string | undefined;
+    llm_model: string | undefined;
     _: (string | number)[];
     $0: string;
 };

package/lib/cli/tool-args.js

@@ -1,9 +1,8 @@
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import { commonOptions } from "./common.js";
-import { GraphAILogger } from "graphai";
 import { getAvailableTemplates } from "../utils/file.js";
-GraphAILogger.setLevelEnabled("error", false);
+import { llmAgents } from "../utils/utils.js";
 const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
 export const args = commonOptions(yargs(hideBin(process.argv)))
     .scriptName("mulmo-tool")
@@ -40,6 +39,17 @@ export const args = commonOptions(yargs(hideBin(process.argv)))
     demandOption: false,
     default: "script",
     type: "string",
+})
+    .option("llm_agent", {
+    description: "llm agent",
+    demandOption: false,
+    choices: llmAgents,
+    type: "string",
+})
+    .option("llm_model", {
+    description: "llm model",
+    demandOption: false,
+    type: "string",
 })
     .command("$0 <action>", "Run mulmocast tool", (yargs) => {
     return yargs.positional("action", {

package/lib/cli/tool-cli.js

@@ -11,7 +11,8 @@ import { getUrlsIfNeeded, selectTemplate } from "../utils/inquirer.js";
 const main = async () => {
     const { o: outdir, b: basedir, action, v: verbose, i: interactive, s: filename, cache } = args;
     let { t: template } = args;
-    let { u: urls } = args;
+    const { u: urls } = args;
+    const { llm_model, llm_agent } = args;
     const baseDirPath = getBaseDirPath(basedir);
     const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
     const cacheDirPath = getFullPath(outDirPath, cache ?? cacheDirName);
@@ -38,12 +39,13 @@ const main = async () => {
         }
     }
     if (action === "scripting") {
+        const context = { outDirPath, templateName: template, urls, filename, cacheDirPath, llm_model, llm_agent };
         if (interactive) {
-            await createMulmoScriptInteractively({ outDirPath, templateName: template, urls, filename, cacheDirPath });
+            await createMulmoScriptInteractively(context);
         }
         else {
-            urls = await getUrlsIfNeeded(urls);
-            await createMulmoScriptFromUrl({ urls, templateName: template, outDirPath, filename, cacheDirPath });
+            context.urls = await getUrlsIfNeeded(urls);
+            await createMulmoScriptFromUrl(context);
         }
     }
     else if (action === "prompt") {

package/lib/methods/index.d.ts

@@ -1,3 +1,4 @@
 export * from "./mulmo_script.js";
 export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
+export * from "./mulmo_media_source.js";

package/lib/methods/index.js

@@ -1,3 +1,4 @@
 export * from "./mulmo_script.js";
 export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
+export * from "./mulmo_media_source.js";

package/lib/methods/mulmo_media_source.d.ts

@@ -0,0 +1,4 @@
+import { MulmoMediaSource, MulmoStudioContext } from "../types/index.js";
+export declare const MulmoMediaSourceMethods: {
+    getText(mediaSource: MulmoMediaSource, context: MulmoStudioContext): Promise<string | null>;
+};

package/lib/methods/mulmo_media_source.js

@@ -0,0 +1,21 @@
+import fs from "fs";
+import { getFullPath } from "../utils/file.js";
+export const MulmoMediaSourceMethods = {
+    async getText(mediaSource, context) {
+        if (mediaSource.kind === "text") {
+            return mediaSource.text;
+        }
+        if (mediaSource.kind === "url") {
+            const res = await fetch(mediaSource.url);
+            if (!res.ok) {
+                throw new Error(`Failed to fetch media source: ${mediaSource.url}`);
+            }
+            return await res.text();
+        }
+        if (mediaSource.kind === "path") {
+            const path = getFullPath(context.fileDirs.mulmoFileDirPath, mediaSource.path);
+            return fs.readFileSync(path, "utf-8");
+        }
+        return null;
+    },
+};

package/lib/methods/mulmo_script.d.ts

@@ -1,10 +1,5 @@
 import "dotenv/config";
-import { MulmoCanvasDimension, MulmoScript, MulmoBeat, SpeechOptions, Text2ImageProvider, MulmoImageParams, Text2SpeechProvider } from "../types/index.js";
-export type Text2ImageAgentInfo = {
-    provider: Text2ImageProvider;
-    agent: string;
-    imageParams: MulmoImageParams;
-};
+import { MulmoCanvasDimension, MulmoScript, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, BeatMediaType } from "../types/index.js";
 export declare const MulmoScriptMethods: {
     getPadding(script: MulmoScript): number;
     getCanvasSize(script: MulmoScript): MulmoCanvasDimension;
@@ -13,4 +8,5 @@ export declare const MulmoScriptMethods: {
     getTextSlideStyle(script: MulmoScript, beat: MulmoBeat): string;
     getSpeechOptions(script: MulmoScript, beat: MulmoBeat): SpeechOptions | undefined;
     getImageAgentInfo(script: MulmoScript): Text2ImageAgentInfo;
+    getImageType(_: MulmoScript, beat: MulmoBeat): BeatMediaType;
 };

package/lib/methods/mulmo_script.js

@@ -1,12 +1,15 @@
 import "dotenv/config";
 import { text2ImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
 const defaultTextSlideStyles = [
-    "body { margin: 40px; margin-top: 60px; color:#333; font-size: 48px }",
-    "h1 { font-size: 60px; text-align: center }",
+    '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
+    "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
+    "h1 { font-size: 56px; margin-bottom: 20px; text-align: center; }",
+    "h2 { font-size: 48px }",
+    "h3 { font-size: 36px }",
     "ul { margin-left: 40px } ",
-    "pre { margin-left: 40px; font-size: 32px }",
+    "pre { background: #eeeecc; font-size: 16px; padding:4px }",
     "p { margin-left: 40px }",
-    "table { font-size: 40px; margin: auto; border: 1px solid gray; border-collapse: collapse }",
+    "table { font-size: 36px; margin: auto; border: 1px solid gray; border-collapse: collapse }",
     "th { border-bottom: 1px solid gray }",
     "td, th { padding: 8px }",
     "tr:nth-child(even) { background-color: #eee }",
@@ -30,7 +33,8 @@ export const MulmoScriptMethods = {
         const styles = script.textSlideParams?.cssStyles ?? defaultTextSlideStyles;
         // NOTES: Taking advantage of CSS override rule (you can redefine it to override)
         const extraStyles = beat.textSlideParams?.cssStyles ?? [];
-        return [...styles, ...extraStyles].join("\n");
+        // This code allows us to support both string and array of strings for cssStyles
+        return [...[styles], ...[extraStyles]].flat().join("\n");
     },
     getSpeechOptions(script, beat) {
         return { ...script.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };
@@ -48,4 +52,7 @@ export const MulmoScriptMethods = {
             imageParams: { ...defaultImageParams, ...script.imageParams },
         };
     },
+    getImageType(_, beat) {
+        return beat.image?.type == "movie" ? "movie" : "image";
+    },
 };

package/lib/tools/create_mulmo_script_interactively.d.ts

@@ -1,3 +1,3 @@
 import "dotenv/config";
 import { ScriptingParams } from "../types/index.js";
-export declare const createMulmoScriptInteractively: ({ outDirPath, cacheDirPath, filename, templateName, urls }: ScriptingParams) => Promise<void>;
+export declare const createMulmoScriptInteractively: ({ outDirPath, cacheDirPath, filename, templateName, urls, llm_agent, llm_model }: ScriptingParams) => Promise<void>;

package/lib/tools/create_mulmo_script_interactively.js

@@ -1,14 +1,19 @@
 import "dotenv/config";
 import { GraphAILogger, GraphAI } from "graphai";
 import { textInputAgent } from "@graphai/input_agents";
+import { streamAgentFilterGenerator } from "@graphai/agent_filters";
 import { openAIAgent } from "@graphai/openai_agent";
+import { anthropicAgent } from "@graphai/anthropic_agent";
+import { geminiAgent } from "@graphai/gemini_agent";
+import { groqAgent } from "@graphai/groq_agent";
 import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { readTemplatePrompt, mkdir } from "../utils/file.js";
 import { browserlessCacheGenerator } from "../utils/filters.js";
 import { browserlessAgent } from "@graphai/browserless_agent";
 import validateMulmoScriptAgent from "../agents/validate_mulmo_script_agent.js";
-import { cliLoadingPlugin } from "../utils/plugins.js";
+import { llmPair } from "../utils/utils.js";
+// import { cliLoadingPlugin } from "../utils/plugins.js";
 const { default: __, ...vanillaAgents } = agents;
 const agentHeader = "\x1b[34m● \x1b[0m\x1b[1mAgent\x1b[0m:\x1b[0m";
 const graphDataForScraping = {
@@ -65,6 +70,15 @@ const graphData = {
         while: ":continue",
     },
     nodes: {
+        llmAgent: {
+            update: ":llmAgent",
+        },
+        llmModel: {
+            update: ":llmModel",
+        },
+        maxTokens: {
+            update: ":maxTokens",
+        },
         fileName: {
             update: ":fileName",
         },
@@ -73,7 +87,7 @@ const graphData = {
         },
         messages: {
             value: [],
-            update: ":reply.llmAgent.messages",
+            update: ":reply.chatAgent.messages",
         },
         userInput: {
             agent: "textInputAgent",
@@ -87,6 +101,9 @@ const graphData = {
             inputs: {
                 messages: ":messages",
                 prompt: ":userInput.text",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
             },
             graph: {
                 loop: {
@@ -97,10 +114,12 @@ const graphData = {
                         value: 0,
                         update: ":counter.add(1)",
                     },
-                    llmAgent: {
-                        agent: "openAIAgent",
+                    chatAgent: {
+                        agent: ":llmAgent",
                         params: {
-                            model: "gpt-4o",
+                            model: ":llmModel",
+                            stream: true,
+                            max_tokens: ":maxTokens",
                         },
                         inputs: {
                             messages: ":messages",
@@ -111,7 +130,7 @@ const graphData = {
                     validateMulmoScriptAgent: {
                         agent: "validateMulmoScriptAgent",
                         inputs: {
-                            text: ":llmAgent.text.codeBlock()",
+                            text: ":chatAgent.text.codeBlock()",
                         },
                     },
                     continue: {
@@ -120,11 +139,15 @@ const graphData = {
                                 GraphAILogger.info("\n" + agentHeader + " \x1b[31mFailed to generate a valid script. Please try again.\n");
                                 return false;
                             }
-                            return !!codeBlock && !isValid;
+                            const result = !!codeBlock && !isValid;
+                            if (result) {
+                                GraphAILogger.info("\n" + agentHeader + " Generated script was broken. Retry generate a script.");
+                            }
+                            return result;
                         },
                         inputs: {
                             counter: ":counter",
-                            codeBlock: ":llmAgent.text.codeBlock()",
+                            codeBlock: ":chatAgent.text.codeBlock()",
                             isValid: ":validateMulmoScriptAgent.isValid",
                         },
                     },
@@ -134,8 +157,8 @@ const graphData = {
         json: {
             agent: "copyAgent",
             inputs: {
-                json: ":reply.llmAgent.text.codeBlock().jsonParse()",
-                text: ":reply.llmAgent.text.codeBlock()",
+                json: ":reply.chatAgent.text.codeBlock().jsonParse()",
+                text: ":reply.chatAgent.text.codeBlock()",
             },
         },
         writeJSON: {
@@ -159,13 +182,6 @@ const graphData = {
                 array: [[":json.json", "==", undefined], "&&", [":userInput.text", "!=", "/bye"]],
             },
         },
-        agentResponse: {
-            if: ":shouldResponse.result",
-            agent: "consoleAgent",
-            inputs: {
-                text: "\n" + agentHeader + " ${:reply.llmAgent.text}\n",
-            },
-        },
         checkInput: {
             agent: "compareAgent",
             inputs: { array: [":userInput.text", "!=", "/bye"] },
@@ -197,11 +213,23 @@ const scrapeWebContent = async (urls, cacheDirPath) => {
     const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
     return `\n\n${prefixPrompt}\n${result?.sourceText.text}`;
 };
-export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath, filename, templateName, urls }) => {
+export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath, filename, templateName, urls, llm_agent, llm_model }) => {
     mkdir(outDirPath);
     // if urls is not empty, scrape web content and reference it in the prompt
     const webContentPrompt = urls.length > 0 ? await scrapeWebContent(urls, cacheDirPath) : "";
-    const graph = new GraphAI(graphData, { ...vanillaAgents, openAIAgent, textInputAgent, fileWriteAgent, validateMulmoScriptAgent });
+    const { agent, model, max_tokens } = llmPair(llm_agent, llm_model);
+    GraphAILogger.log({ agent, model, max_tokens });
+    const streamAgentFilter = streamAgentFilterGenerator((context, data) => {
+        process.stdout.write(String(data));
+    });
+    const agentFilters = [
+        {
+            name: "streamAgentFilter",
+            agent: streamAgentFilter,
+            nodeIds: ["chatAgent"],
+        },
+    ];
+    const graph = new GraphAI(graphData, { ...vanillaAgents, anthropicAgent, geminiAgent, groqAgent, openAIAgent, textInputAgent, fileWriteAgent, validateMulmoScriptAgent }, { agentFilters });
     const prompt = readTemplatePrompt(templateName);
     graph.injectValue("messages", [
         {
@@ -211,7 +239,20 @@ export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath,
     ]);
     graph.injectValue("outdir", outDirPath);
     graph.injectValue("fileName", filename);
-    graph.registerCallback(cliLoadingPlugin({ nodeId: "reply", message: "Loading..." }));
+    graph.injectValue("llmAgent", agent);
+    graph.injectValue("llmModel", model);
+    graph.injectValue("maxTokens", max_tokens);
+    graph.registerCallback(({ nodeId, state }) => {
+        if (nodeId === "chatAgent") {
+            if (state === "executing") {
+                process.stdout.write(String("\n" + agentHeader + " "));
+            }
+            if (state === "completed") {
+                process.stdout.write("\n\n");
+            }
+        }
+    });
+    // graph.registerCallback(cliLoadingPlugin({ nodeId: "reply", message: "Loading..." }));
     GraphAILogger.info(`${agentHeader} Hi! What topic would you like me to generate about?\n`);
     await graph.run();
 };

package/lib/types/index.d.ts

@@ -1 +1,2 @@
 export * from "./type.js";
+export * from "./schema.js";

package/lib/types/index.js

@@ -1 +1,2 @@
 export * from "./type.js";
+export * from "./schema.js";