mulmocast 0.0.5 → 0.0.6

package/lib/cli/helpers.js

@@ -0,0 +1,115 @@
+import { GraphAILogger } from "graphai";
+import fs from "fs";
+import path from "path";
+import clipboardy from "clipboardy";
+import { getBaseDirPath, getFullPath, readMulmoScriptFile, fetchMulmoScriptFile, getOutputStudioFilePath, resolveDirPath } from "../utils/file.js";
+import { isHttp } from "../utils/utils.js";
+import { createOrUpdateStudioData } from "../utils/preprocess.js";
+import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
+import { translate } from "../actions/translate.js";
+export const setGraphAILogger = (verbose, logValues) => {
+    if (verbose) {
+        if (logValues) {
+            Object.entries(logValues).forEach(([key, value]) => {
+                GraphAILogger.info(`${key}:`, value);
+            });
+        }
+    }
+    else {
+        GraphAILogger.setLevelEnabled("error", false);
+        GraphAILogger.setLevelEnabled("log", false);
+        GraphAILogger.setLevelEnabled("warn", false);
+    }
+};
+export const getFileObject = (args) => {
+    const { basedir, outdir, imagedir, audiodir, file } = args;
+    const baseDirPath = getBaseDirPath(basedir);
+    const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
+    const { fileOrUrl, fileName } = (() => {
+        if (file === "__clipboard") {
+            // We generate a new unique script file from clipboard text in the output directory
+            const now = new Date();
+            const pad = (n) => n.toString().padStart(2, "0");
+            const fileName = `script_${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
+            const clipboardText = clipboardy.readSync();
+            const fileOrUrl = resolveDirPath(outDirPath, `${fileName}.json`);
+            fs.writeFileSync(fileOrUrl, clipboardText, "utf8");
+            return { fileOrUrl, fileName };
+        }
+        const fileOrUrl = file ?? "";
+        const fileName = path.parse(fileOrUrl).name;
+        return { fileOrUrl, fileName };
+    })();
+    const isHttpPath = isHttp(fileOrUrl);
+    const mulmoFilePath = isHttpPath ? "" : getFullPath(baseDirPath, fileOrUrl);
+    const mulmoFileDirPath = path.dirname(isHttpPath ? baseDirPath : mulmoFilePath);
+    const imageDirPath = getFullPath(outDirPath, imagedir ?? imageDirName);
+    const audioDirPath = getFullPath(outDirPath, audiodir ?? audioDirName);
+    const outputStudioFilePath = getOutputStudioFilePath(outDirPath, fileName);
+    return {
+        baseDirPath,
+        mulmoFilePath,
+        mulmoFileDirPath,
+        outDirPath,
+        imageDirPath,
+        audioDirPath,
+        isHttpPath,
+        fileOrUrl,
+        outputStudioFilePath,
+        fileName,
+    };
+};
+export const fetchScript = async (isHttpPath, mulmoFilePath, fileOrUrl) => {
+    if (isHttpPath) {
+        const res = await fetchMulmoScriptFile(fileOrUrl);
+        if (!res.result || !res.script) {
+            GraphAILogger.info(`ERROR: HTTP error! ${res.status} ${fileOrUrl}`);
+            process.exit(1);
+        }
+        return res.script;
+    }
+    if (!fs.existsSync(mulmoFilePath)) {
+        GraphAILogger.info(`ERROR: File not exists ${mulmoFilePath}`);
+        process.exit(1);
+    }
+    return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath).mulmoData;
+};
+export const initializeContext = async (argv) => {
+    const files = getFileObject({
+        basedir: argv.b,
+        outdir: argv.o,
+        imagedir: argv.i,
+        audiodir: argv.a,
+        file: argv.file ?? "",
+    });
+    const { fileName, isHttpPath, fileOrUrl, mulmoFilePath, outputStudioFilePath } = files;
+    setGraphAILogger(argv.v, {
+        files,
+    });
+    const mulmoScript = await fetchScript(isHttpPath, mulmoFilePath, fileOrUrl);
+    // Create or update MulmoStudio file with MulmoScript
+    const currentStudio = readMulmoScriptFile(outputStudioFilePath);
+    const studio = (() => {
+        try {
+            // validate mulmoStudioSchema. skip if __test_invalid__ is true
+            return createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName);
+        }
+        catch (error) {
+            GraphAILogger.info(`Error: invalid MulmoScript Schema: ${isHttpPath ? fileOrUrl : mulmoFilePath} \n ${error}`);
+            process.exit(1);
+        }
+    })();
+    return {
+        studio,
+        fileDirs: files,
+        force: Boolean(argv.f),
+        lang: argv.l,
+        caption: argv.c,
+    };
+};
+export const runTranslateIfNeeded = async (context, argv) => {
+    if (argv.l || argv.c) {
+        GraphAILogger.log("run translate");
+        await translate(context);
+    }
+};

package/lib/cli/tool-args.d.ts

@@ -3,6 +3,7 @@ export declare const args: {
     v: boolean;
     o: string | undefined;
     b: string | undefined;
+    l: string | undefined;
     u: string[] | never[];
     i: boolean | undefined;
     t: string | undefined;

package/lib/cli/tool-args.js

@@ -54,7 +54,7 @@ export const args = commonOptions(yargs(hideBin(process.argv)))
     .command("$0 <action>", "Run mulmocast tool", (yargs) => {
     return yargs.positional("action", {
         describe: "action to perform",
-        choices: ["scripting", "prompt"],
+        choices: ["scripting", "prompt", "schema"],
         type: "string",
     });
 })

package/lib/cli/tool-cli.js

@@ -8,6 +8,8 @@ import { getBaseDirPath, getFullPath } from "../utils/file.js";
 import { createMulmoScriptInteractively } from "../tools/create_mulmo_script_interactively.js";
 import { dumpPromptFromTemplate } from "../tools/dump_prompt.js";
 import { getUrlsIfNeeded, selectTemplate } from "../utils/inquirer.js";
+import { mulmoScriptSchema } from "../types/schema.js";
+import { zodToJsonSchema } from "zod-to-json-schema";
 const main = async () => {
     const { o: outdir, b: basedir, action, v: verbose, i: interactive, s: filename, cache } = args;
     let { t: template } = args;
@@ -51,6 +53,12 @@ const main = async () => {
     else if (action === "prompt") {
         await dumpPromptFromTemplate({ templateName: template });
     }
+    else if (action === "schema") {
+        const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
+            strictUnions: true,
+        });
+        GraphAILogger.info(JSON.stringify(defaultSchema, null, 2));
+    }
     else {
         throw new Error(`Unknown or unsupported action: ${action}`);
     }

package/lib/methods/mulmo_script.d.ts

@@ -1,7 +1,6 @@
 import "dotenv/config";
 import { MulmoCanvasDimension, MulmoScript, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, BeatMediaType } from "../types/index.js";
 export declare const MulmoScriptMethods: {
-    getPadding(script: MulmoScript): number;
     getCanvasSize(script: MulmoScript): MulmoCanvasDimension;
     getAspectRatio(script: MulmoScript): string;
     getSpeechProvider(script: MulmoScript): Text2SpeechProvider;

package/lib/methods/mulmo_script.js

@@ -3,8 +3,8 @@ import { text2ImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensi
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
-    "h1 { font-size: 56px; margin-bottom: 20px; text-align: center; }",
-    "h2 { font-size: 48px }",
+    "h1 { font-size: 56px; margin-bottom: 20px; text-align: center }",
+    "h2 { font-size: 48px; text-align: center }",
     "h3 { font-size: 36px }",
     "ul { margin-left: 40px } ",
     "pre { background: #eeeecc; font-size: 16px; padding:4px }",
@@ -15,9 +15,6 @@ const defaultTextSlideStyles = [
     "tr:nth-child(even) { background-color: #eee }",
 ];
 export const MulmoScriptMethods = {
-    getPadding(script) {
-        return script.videoParams?.padding ?? 1000; // msec
-    },
     getCanvasSize(script) {
         return mulmoCanvasDimensionSchema.parse(script.canvasSize);
     },
@@ -30,11 +27,11 @@ export const MulmoScriptMethods = {
         return text2SpeechProviderSchema.parse(script.speechParams?.provider);
     },
     getTextSlideStyle(script, beat) {
-        const styles = script.textSlideParams?.cssStyles ?? defaultTextSlideStyles;
+        const styles = script.textSlideParams?.cssStyles ?? [];
         // NOTES: Taking advantage of CSS override rule (you can redefine it to override)
         const extraStyles = beat.textSlideParams?.cssStyles ?? [];
         // This code allows us to support both string and array of strings for cssStyles
-        return [...[styles], ...[extraStyles]].flat().join("\n");
+        return [...defaultTextSlideStyles, ...[styles], ...[extraStyles]].flat().join("\n");
     },
     getSpeechOptions(script, beat) {
         return { ...script.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };

package/lib/methods/mulmo_script_template.js

@@ -1,16 +1,6 @@
-import { zodToJsonSchema } from "zod-to-json-schema";
-import { mulmoScriptSchema } from "../types/schema.js";
+import { getMulmoScriptTemplateSystemPrompt } from "../utils/prompt.js";
 export const MulmoScriptTemplateMethods = {
     getSystemPrompt(template) {
-        // script is provided, use it as a script template
-        if (template.script) {
-            return `${template.systemPrompt}\n\`\`\`JSON\n${JSON.stringify(template.script)}\n\`\`\``;
-        }
-        // script is not provided, use the default schema
-        const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
-            strictUnions: true,
-        });
-        const specificOutputPrompt = `The output should follow the JSON schema specified below. Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`;
-        return `${template.systemPrompt}\n\n${specificOutputPrompt}\n\n\`\`\`JSON\n${JSON.stringify(defaultSchema)}\n\`\`\``;
+        return getMulmoScriptTemplateSystemPrompt(template);
     },
 };

package/lib/tools/create_mulmo_script_from_url.d.ts

@@ -1,3 +1,3 @@
 import "dotenv/config";
 import { ScriptingParams } from "../types/index.js";
-export declare const createMulmoScriptFromUrl: ({ urls, templateName, outDirPath, filename, cacheDirPath }: ScriptingParams) => Promise<void>;
+export declare const createMulmoScriptFromUrl: ({ urls, templateName, outDirPath, filename, cacheDirPath, llm_agent, llm_model }: ScriptingParams) => Promise<void>;

package/lib/tools/create_mulmo_script_from_url.js

@@ -1,14 +1,19 @@
 import "dotenv/config";
 import { GraphAI } from "graphai";
 import { openAIAgent } from "@graphai/openai_agent";
+import { anthropicAgent } from "@graphai/anthropic_agent";
+import { geminiAgent } from "@graphai/gemini_agent";
+import { groqAgent } from "@graphai/groq_agent";
 import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { browserlessAgent } from "@graphai/browserless_agent";
-import validateMulmoScriptAgent from "../agents/validate_mulmo_script_agent.js";
+import validateSchemaAgent from "../agents/validate_schema_agent.js";
 import { readTemplatePrompt, mkdir, writingMessage } from "../utils/file.js";
 import { browserlessCacheGenerator } from "../utils/filters.js";
-import { urlsSchema } from "../types/schema.js";
+import { mulmoScriptSchema, urlsSchema } from "../types/schema.js";
 import { cliLoadingPlugin } from "../utils/plugins.js";
+import { graphDataScriptFromUrlPrompt } from "../utils/prompt.js";
+import { llmPair } from "../utils/utils.js";
 const { default: __, ...vanillaAgents } = agents;
 const graphData = {
     version: 0.5,
@@ -27,6 +32,15 @@ const graphData = {
         fileName: {
             value: "",
         },
+        llmAgent: {
+            value: "",
+        },
+        llmModel: {
+            value: "",
+        },
+        maxTokens: {
+            value: 0,
+        },
         // get the text content of the urls
         fetchResults: {
             agent: "mapAgent",
@@ -74,6 +88,9 @@ const graphData = {
             inputs: {
                 sourceText: ":sourceText",
                 prompt: ":prompt",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
             },
             graph: {
                 loop: {
@@ -85,18 +102,23 @@ const graphData = {
                         value: 0,
                         update: ":counter.add(1)",
                     },
-                    openAIAgent: {
-                        agent: "openAIAgent",
+                    llm: {
+                        agent: ":llmAgent",
+                        params: {
+                            model: ":llmModel",
+                            system: ":prompt",
+                            max_tokens: ":maxTokens",
+                        },
                         inputs: {
-                            model: "gpt-4o",
                             system: ":prompt",
-                            prompt: "Please create a script using the information from the following URLs as reference: ${:sourceText.text}",
+                            prompt: graphDataScriptFromUrlPrompt("${:sourceText.text}"),
                         },
                     },
-                    validateMulmoScriptAgent: {
-                        agent: "validateMulmoScriptAgent",
+                    validateSchemaAgent: {
+                        agent: "validateSchemaAgent",
                         inputs: {
-                            text: ":openAIAgent.text.codeBlock()",
+                            text: ":llm.text.codeBlock()",
+                            schema: mulmoScriptSchema,
                         },
                         isResult: true,
                     },
@@ -105,7 +127,7 @@ const graphData = {
                             return !isValid && counter < 3;
                         },
                         inputs: {
-                            isValid: ":validateMulmoScriptAgent.isValid",
+                            isValid: ":validateSchemaAgent.isValid",
                             counter: ":counter",
                         },
                     },
@@ -113,17 +135,17 @@ const graphData = {
             },
         },
         writeJSON: {
-            if: ":mulmoScript.validateMulmoScriptAgent.isValid",
+            if: ":mulmoScript.validateSchemaAgent.isValid",
             agent: "fileWriteAgent",
             inputs: {
                 file: "${:outdir}/${:fileName}-${@now}.json",
-                text: ":mulmoScript.validateMulmoScriptAgent.data.toJSON()",
+                text: ":mulmoScript.validateSchemaAgent.data.toJSON()",
             },
             isResult: true,
         },
     },
 };
-export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath, filename, cacheDirPath }) => {
+export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath, filename, cacheDirPath, llm_agent, llm_model }) => {
     mkdir(outDirPath);
     mkdir(cacheDirPath);
     const parsedUrls = urlsSchema.parse(urls);
@@ -135,17 +157,24 @@ export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath,
             nodeIds: ["fetcher"],
         },
     ];
+    const { agent, model, max_tokens } = llmPair(llm_agent, llm_model);
     const graph = new GraphAI(graphData, {
         ...vanillaAgents,
         openAIAgent,
+        anthropicAgent,
+        geminiAgent,
+        groqAgent,
         browserlessAgent,
-        validateMulmoScriptAgent,
+        validateSchemaAgent,
         fileWriteAgent,
     }, { agentFilters });
     graph.injectValue("urls", parsedUrls);
     graph.injectValue("prompt", readTemplatePrompt(templateName));
     graph.injectValue("outdir", outDirPath);
     graph.injectValue("fileName", filename);
+    graph.injectValue("llmAgent", agent);
+    graph.injectValue("llmModel", model);
+    graph.injectValue("maxTokens", max_tokens);
     graph.registerCallback(cliLoadingPlugin({ nodeId: "mulmoScript", message: "Generating script..." }));
     const result = await graph.run();
     writingMessage(result?.writeJSON?.path ?? "");

package/lib/tools/create_mulmo_script_interactively.js

@@ -1,7 +1,7 @@
 import "dotenv/config";
 import { GraphAILogger, GraphAI } from "graphai";
 import { textInputAgent } from "@graphai/input_agents";
-import { streamAgentFilterGenerator } from "@graphai/agent_filters";
+import { consoleStreamDataAgentFilter } from "@graphai/stream_agent_filter/node";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
 import { geminiAgent } from "@graphai/gemini_agent";
@@ -10,9 +10,11 @@ import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { readTemplatePrompt, mkdir } from "../utils/file.js";
 import { browserlessCacheGenerator } from "../utils/filters.js";
+import { mulmoScriptSchema } from "../types/index.js";
 import { browserlessAgent } from "@graphai/browserless_agent";
-import validateMulmoScriptAgent from "../agents/validate_mulmo_script_agent.js";
+import validateSchemaAgent from "../agents/validate_schema_agent.js";
 import { llmPair } from "../utils/utils.js";
+import { interactiveClarificationPrompt, prefixPrompt } from "../utils/prompt.js";
 // import { cliLoadingPlugin } from "../utils/plugins.js";
 const { default: __, ...vanillaAgents } = agents;
 const agentHeader = "\x1b[34m● \x1b[0m\x1b[1mAgent\x1b[0m:\x1b[0m";
@@ -119,6 +121,7 @@ const graphData = {
                         params: {
                             model: ":llmModel",
                             stream: true,
+                            dataStream: true,
                             max_tokens: ":maxTokens",
                         },
                         inputs: {
@@ -127,10 +130,13 @@ const graphData = {
                         },
                         isResult: true,
                     },
-                    validateMulmoScriptAgent: {
-                        agent: "validateMulmoScriptAgent",
+                    validateSchemaAgent: {
+                        if: ":chatAgent.text.codeBlock()",
+                        defaultValue: false,
+                        agent: "validateSchemaAgent",
                         inputs: {
                             text: ":chatAgent.text.codeBlock()",
+                            schema: mulmoScriptSchema,
                         },
                     },
                     continue: {
@@ -148,7 +154,7 @@ const graphData = {
                         inputs: {
                             counter: ":counter",
                             codeBlock: ":chatAgent.text.codeBlock()",
-                            isValid: ":validateMulmoScriptAgent.isValid",
+                            isValid: ":validateSchemaAgent.isValid",
                         },
                     },
                 },
@@ -192,7 +198,6 @@ const graphData = {
         },
     },
 };
-const interactiveClarificationPrompt = `If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.`;
 const scrapeWebContent = async (urls, cacheDirPath) => {
     mkdir(cacheDirPath);
     GraphAILogger.info(`${agentHeader} Scraping ${urls.length} URLs...\n`);
@@ -210,7 +215,6 @@ const scrapeWebContent = async (urls, cacheDirPath) => {
     if (!result?.sourceText?.text) {
         return "";
     }
-    const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
     return `\n\n${prefixPrompt}\n${result?.sourceText.text}`;
 };
 export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath, filename, templateName, urls, llm_agent, llm_model }) => {
@@ -219,17 +223,14 @@ export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath,
     const webContentPrompt = urls.length > 0 ? await scrapeWebContent(urls, cacheDirPath) : "";
     const { agent, model, max_tokens } = llmPair(llm_agent, llm_model);
     GraphAILogger.log({ agent, model, max_tokens });
-    const streamAgentFilter = streamAgentFilterGenerator((context, data) => {
-        process.stdout.write(String(data));
-    });
     const agentFilters = [
         {
-            name: "streamAgentFilter",
-            agent: streamAgentFilter,
+            name: "consoleStreamDataAgentFilter",
+            agent: consoleStreamDataAgentFilter,
             nodeIds: ["chatAgent"],
         },
     ];
-    const graph = new GraphAI(graphData, { ...vanillaAgents, anthropicAgent, geminiAgent, groqAgent, openAIAgent, textInputAgent, fileWriteAgent, validateMulmoScriptAgent }, { agentFilters });
+    const graph = new GraphAI(graphData, { ...vanillaAgents, anthropicAgent, geminiAgent, groqAgent, openAIAgent, textInputAgent, fileWriteAgent, validateSchemaAgent }, { agentFilters });
     const prompt = readTemplatePrompt(templateName);
     graph.injectValue("messages", [
         {

package/lib/tools/dump_prompt.js

@@ -1,6 +1,8 @@
 import { GraphAILogger } from "graphai";
 import { readTemplatePrompt } from "../utils/file.js";
+import clipboardy from "clipboardy";
 export const dumpPromptFromTemplate = async ({ templateName }) => {
     const prompt = readTemplatePrompt(templateName);
     GraphAILogger.info(prompt);
+    clipboardy.write(prompt);
 };

package/lib/tools/story_to_script.d.ts

@@ -0,0 +1,10 @@
+import { MulmoStoryboard } from "../types/index.js";
+export declare const storyToScript: ({ story, beatsPerScene, templateName, outdir, fileName, llmAgent, llmModel, }: {
+    story: MulmoStoryboard;
+    beatsPerScene: number;
+    templateName: string;
+    outdir: string;
+    fileName: string;
+    llmAgent?: string;
+    llmModel?: string;
+}) => Promise<void>;