mulmocast 0.0.21 → 0.0.23

package/lib/cli/commands/tool/story_to_script/builder.d.ts

@@ -10,7 +10,7 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     beats_per_scene: number;
 } & {
-    llm: "openAI" | "anthropic" | "gemini" | "groq" | undefined;
+    llm: "anthropic" | "openAI" | "gemini" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
 } & {

package/lib/cli/helpers.js

@@ -7,7 +7,7 @@ import { isHttp } from "../utils/utils.js";
 import { createOrUpdateStudioData } from "../utils/preprocess.js";
 import { outDirName, imageDirName, audioDirName } from "../utils/const.js";
 import { translate } from "../actions/translate.js";
-import { mulmoPresentationStyleSchema, mulmoStudioMultiLingualSchema } from "../types/schema.js";
+import { mulmoCaptionParamsSchema, mulmoPresentationStyleSchema, mulmoStudioMultiLingualSchema } from "../types/schema.js";
 export const setGraphAILogger = (verbose, logValues) => {
     if (verbose) {
         if (logValues) {
@@ -126,13 +126,18 @@ export const initializeContext = async (argv) => {
         // validate mulmoStudioSchema. skip if __test_invalid__ is true
         const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName);
         const multiLingual = getMultiLingual(outputMultilingualFilePath, studio.beats.length);
+        if (argv.c) {
+            studio.script.captionParams = mulmoCaptionParamsSchema.parse({
+                ...(studio.script.captionParams ?? {}),
+                lang: argv.c,
+            });
+        }
         return {
             studio,
             fileDirs: files,
             force: Boolean(argv.f),
             dryRun: Boolean(argv.dryRun),
             lang: argv.l,
-            caption: argv.c,
             sessionState: {
                 inSession: {
                     audio: false,
@@ -160,7 +165,7 @@ export const initializeContext = async (argv) => {
     }
 };
 export const runTranslateIfNeeded = async (context, argv) => {
-    if (argv.l || argv.c) {
+    if (argv.l || context.studio.script.captionParams?.lang) {
         GraphAILogger.log("run translate");
         await translate(context);
     }

package/lib/methods/mulmo_presentation_style.d.ts

@@ -1,5 +1,5 @@
 import "dotenv/config";
-import { MulmoCanvasDimension, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
     getSpeechProvider(presentationStyle: MulmoPresentationStyle): Text2SpeechProvider;
@@ -10,5 +10,6 @@ export declare const MulmoPresentationStyleMethods: {
     getProvider(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): Text2SpeechProvider;
     getVoiceId(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, dryRun?: boolean): Text2ImageAgentInfo;
+    getHtmlImageAgentInfo(presentationStyle: MulmoPresentationStyle): Text2HtmlAgentInfo;
     getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;
 };

package/lib/methods/mulmo_presentation_style.js

@@ -1,5 +1,6 @@
 import "dotenv/config";
-import { text2ImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
+import { userAssert } from "../utils/utils.js";
+import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
 import { defaultOpenAIImageModel } from "../utils/const.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
@@ -42,7 +43,11 @@ export const MulmoPresentationStyleMethods = {
         return { ...presentationStyle.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };
     },
     getSpeaker(presentationStyle, beat) {
-        return presentationStyle.speechParams.speakers[beat.speaker];
+        userAssert(!!presentationStyle?.speechParams?.speakers, "presentationStyle.speechParams.speakers is not set!!");
+        userAssert(!!beat?.speaker, "beat.speaker is not set");
+        const speaker = presentationStyle.speechParams.speakers[beat.speaker];
+        userAssert(!!speaker, `speaker is not set: speaker "${beat.speaker}"`);
+        return speaker;
     },
     getProvider(presentationStyle, beat) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
@@ -65,6 +70,20 @@ export const MulmoPresentationStyleMethods = {
             imageParams: { ...defaultImageParams, ...presentationStyle.imageParams },
         };
     },
+    getHtmlImageAgentInfo(presentationStyle) {
+        const provider = text2HtmlImageProviderSchema.parse(presentationStyle.htmlImageParams?.provider);
+        const agent = provider === "anthropic" ? "anthropicAgent" : "openAIAgent";
+        const model = presentationStyle.htmlImageParams?.model
+            ? presentationStyle.htmlImageParams?.model
+            : provider === "anthropic"
+                ? "claude-3-7-sonnet-20250219"
+                : "gpt-4o-mini";
+        return {
+            provider,
+            agent,
+            model,
+        };
+    },
     getImageType(_, beat) {
         return beat.image?.type == "movie" ? "movie" : "image";
     },

package/lib/methods/mulmo_studio_context.js

@@ -44,7 +44,7 @@ export const MulmoStudioContextMethods = {
         return context.studio.filename;
     },
     getCaption(context) {
-        return context.caption;
+        return context.studio.script.captionParams?.lang;
     },
     setSessionState(context, sessionType, value) {
         context.sessionState.inSession[sessionType] = value;

package/lib/tools/create_mulmo_script_from_url.d.ts

@@ -1,3 +1,4 @@
 import "dotenv/config";
 import { ScriptingParams } from "../types/index.js";
 export declare const createMulmoScriptFromUrl: ({ urls, templateName, outDirPath, filename, cacheDirPath, llm, llm_model }: ScriptingParams) => Promise<void>;
+export declare const createMulmoScriptFromFile: (fileName: string, { templateName, outDirPath, filename, cacheDirPath, llm, llm_model, verbose }: ScriptingParams) => Promise<void>;

package/lib/tools/create_mulmo_script_from_url.js

@@ -1,4 +1,5 @@
 import "dotenv/config";
+import path from "path";
 import { GraphAI } from "graphai";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
@@ -14,7 +15,56 @@ import { mulmoScriptSchema, urlsSchema } from "../types/schema.js";
 import { cliLoadingPlugin } from "../utils/plugins.js";
 import { graphDataScriptFromUrlPrompt } from "../utils/prompt.js";
 import { llmPair } from "../utils/utils.js";
+import { readFileSync } from "fs";
 const vanillaAgents = agents.default ?? agents;
+const graphMulmoScript = {
+    version: 0.5,
+    loop: {
+        // If the script is not valid and the counter is less than 3, continue the loop
+        while: ":continue",
+    },
+    nodes: {
+        sourceText: {},
+        systemPrompt: {},
+        llmAgent: {},
+        llmModel: {},
+        maxTokens: {},
+        counter: {
+            value: 0,
+            update: ":counter.add(1)",
+        },
+        llm: {
+            agent: ":llmAgent",
+            // console: { before: true },
+            inputs: {
+                system: ":systemPrompt",
+                prompt: graphDataScriptFromUrlPrompt("${:sourceText.text}"),
+                params: {
+                    model: ":llmModel",
+                    system: ":systemPrompt",
+                    max_tokens: ":maxTokens",
+                },
+            },
+        },
+        validateSchemaAgent: {
+            agent: "validateSchemaAgent",
+            inputs: {
+                text: ":llm.text.codeBlock()",
+                schema: mulmoScriptSchema,
+            },
+            isResult: true,
+        },
+        continue: {
+            agent: ({ isValid, counter }) => {
+                return !isValid && counter < 3;
+            },
+            inputs: {
+                isValid: ":validateSchemaAgent.isValid",
+                counter: ":counter",
+            },
+        },
+    },
+};
 const graphData = {
     version: 0.5,
     // Execute sequentially because the free version of browserless API doesn't support concurrent execution.
@@ -23,7 +73,7 @@ const graphData = {
         urls: {
             value: [],
         },
-        prompt: {
+        systemPrompt: {
             value: "",
         },
         outdir: {
@@ -87,52 +137,60 @@ const graphData = {
             agent: "nestedAgent",
             inputs: {
                 sourceText: ":sourceText",
-                prompt: ":prompt",
+                systemPrompt: ":systemPrompt",
                 llmAgent: ":llmAgent",
                 llmModel: ":llmModel",
                 maxTokens: ":maxTokens",
             },
-            graph: {
-                loop: {
-                    // If the script is not valid and the counter is less than 3, continue the loop
-                    while: ":continue",
-                },
-                nodes: {
-                    counter: {
-                        value: 0,
-                        update: ":counter.add(1)",
-                    },
-                    llm: {
-                        agent: ":llmAgent",
-                        inputs: {
-                            system: ":prompt",
-                            prompt: graphDataScriptFromUrlPrompt("${:sourceText.text}"),
-                            params: {
-                                model: ":llmModel",
-                                system: ":prompt",
-                                max_tokens: ":maxTokens",
-                            },
-                        },
-                    },
-                    validateSchemaAgent: {
-                        agent: "validateSchemaAgent",
-                        inputs: {
-                            text: ":llm.text.codeBlock()",
-                            schema: mulmoScriptSchema,
-                        },
-                        isResult: true,
-                    },
-                    continue: {
-                        agent: ({ isValid, counter }) => {
-                            return !isValid && counter < 3;
-                        },
-                        inputs: {
-                            isValid: ":validateSchemaAgent.isValid",
-                            counter: ":counter",
-                        },
-                    },
-                },
+            graph: graphMulmoScript,
+        },
+        writeJSON: {
+            if: ":mulmoScript.validateSchemaAgent.isValid",
+            agent: "fileWriteAgent",
+            inputs: {
+                file: "${:outdir}/${:fileName}-${@now}.json",
+                text: ":mulmoScript.validateSchemaAgent.data.toJSON()",
+            },
+            isResult: true,
+        },
+    },
+};
+const graphDataText = {
+    version: 0.5,
+    // Execute sequentially because the free version of browserless API doesn't support concurrent execution.
+    concurrency: 1,
+    nodes: {
+        systemPrompt: {
+            value: "",
+        },
+        outdir: {
+            value: "",
+        },
+        fileName: {
+            value: "",
+        },
+        llmAgent: {
+            value: "",
+        },
+        llmModel: {
+            value: "",
+        },
+        maxTokens: {
+            value: 0,
+        },
+        sourceText: {},
+        // generate the mulmo script
+        mulmoScript: {
+            agent: "nestedAgent",
+            // console: { before: true },
+            inputs: {
+                sourceText: ":sourceText",
+                systemPrompt: ":systemPrompt",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
             },
+            graph: graphMulmoScript,
         },
         writeJSON: {
             if: ":mulmoScript.validateSchemaAgent.isValid",
@@ -169,7 +227,7 @@ export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath,
         fileWriteAgent,
     }, { agentFilters });
     graph.injectValue("urls", parsedUrls);
-    graph.injectValue("prompt", readTemplatePrompt(templateName));
+    graph.injectValue("systemPrompt", readTemplatePrompt(templateName));
     graph.injectValue("outdir", outDirPath);
     graph.injectValue("fileName", filename);
     graph.injectValue("llmAgent", agent);
@@ -179,3 +237,31 @@ export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath,
     const result = await graph.run();
     writingMessage(result?.writeJSON?.path ?? "");
 };
+export const createMulmoScriptFromFile = async (fileName, { templateName, outDirPath, filename, cacheDirPath, llm, llm_model, verbose }) => {
+    mkdir(outDirPath);
+    mkdir(cacheDirPath);
+    const filePath = path.resolve(process.cwd(), fileName);
+    const text = readFileSync(filePath, "utf-8");
+    const { agent, model, max_tokens } = llmPair(llm, llm_model);
+    const graph = new GraphAI(graphDataText, {
+        ...vanillaAgents,
+        openAIAgent,
+        anthropicAgent,
+        geminiAgent,
+        groqAgent,
+        validateSchemaAgent,
+        fileWriteAgent,
+    });
+    graph.injectValue("sourceText", { text });
+    graph.injectValue("systemPrompt", readTemplatePrompt(templateName));
+    graph.injectValue("outdir", outDirPath);
+    graph.injectValue("fileName", filename);
+    graph.injectValue("llmAgent", agent);
+    graph.injectValue("llmModel", model);
+    graph.injectValue("maxTokens", max_tokens);
+    if (!verbose) {
+        graph.registerCallback(cliLoadingPlugin({ nodeId: "mulmoScript", message: "Generating script..." }));
+    }
+    const result = await graph.run();
+    writingMessage(result?.writeJSON?.path ?? "");
+};