mulmocast 0.0.6 → 0.0.7

package/lib/cli/commands/tool/scripting/builder.js

@@ -1,4 +1,4 @@
-import { llmAgents } from "../../../../utils/utils.js";
+import { llm } from "../../../../utils/utils.js";
 import { getAvailableTemplates } from "../../../../utils/file.js";
 const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
 export const builder = (yargs) => {
@@ -49,10 +49,10 @@ export const builder = (yargs) => {
         default: "script",
         type: "string",
     })
-        .option("llm_agent", {
-        description: "llm agent",
+        .option("llm", {
+        description: "llm",
         demandOption: false,
-        choices: llmAgents,
+        choices: llm,
         type: "string",
     })
         .option("llm_model", {

package/lib/cli/commands/tool/scripting/handler.d.ts

@@ -1,4 +1,5 @@
 import { ToolCliArgs } from "../../../../types/cli_types.js";
+import { LLM } from "../../../../utils/utils.js";
 export declare const handler: (argv: ToolCliArgs<{
     o?: string;
     b?: string;
@@ -7,6 +8,6 @@ export declare const handler: (argv: ToolCliArgs<{
     t?: string;
     c?: string;
     s?: string;
-    llm_agent?: string;
+    llm?: LLM;
     llm_model?: string;
 }>) => Promise<void>;

package/lib/cli/commands/tool/scripting/handler.js

@@ -5,7 +5,7 @@ import { createMulmoScriptFromUrl } from "../../../../tools/create_mulmo_script_
 import { createMulmoScriptInteractively } from "../../../../tools/create_mulmo_script_interactively.js";
 import { setGraphAILogger } from "../../../../cli/helpers.js";
 export const handler = async (argv) => {
-    const { o: outdir, b: basedir, v: verbose, i: interactive, c: cache, s: filename, llm_agent, llm_model } = argv;
+    const { o: outdir, b: basedir, v: verbose, i: interactive, c: cache, s: filename, llm, llm_model } = argv;
     let { t: template } = argv;
     const urls = argv.u || [];
     const baseDirPath = getBaseDirPath(basedir);
@@ -22,10 +22,10 @@ export const handler = async (argv) => {
         urls,
         interactive,
         filename,
-        llm_agent,
+        llm,
         llm_model,
     });
-    const context = { outDirPath, templateName: template, urls, filename: filename, cacheDirPath, llm_model, llm_agent };
+    const context = { outDirPath, templateName: template, urls, filename: filename, cacheDirPath, llm_model, llm };
     if (interactive) {
         await createMulmoScriptInteractively(context);
     }

package/lib/cli/commands/tool/story_to_script/builder.d.ts

@@ -10,9 +10,11 @@ export declare const builder: (yargs: Argv) => Argv<{
 } & {
     beats_per_scene: number;
 } & {
-    llm_agent: string | undefined;
+    llm: "openAI" | "anthropic" | "gemini" | "groq" | undefined;
 } & {
     llm_model: string | undefined;
+} & {
+    mode: string;
 } & {
     file: string | undefined;
 }>;

package/lib/cli/commands/tool/story_to_script/builder.js

@@ -1,5 +1,6 @@
 import { getAvailableTemplates } from "../../../../utils/file.js";
-import { llmAgents } from "../../../../utils/utils.js";
+import { llm } from "../../../../utils/utils.js";
+import { storyToScriptGenerateMode } from "../../../../utils/const.js";
 const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
 export const builder = (yargs) => {
     return yargs
@@ -35,16 +36,23 @@ export const builder = (yargs) => {
         default: 3,
         type: "number",
     })
-        .option("llm_agent", {
-        description: "llm agent",
+        .option("llm", {
+        description: "llm",
         demandOption: false,
-        choices: llmAgents,
+        choices: llm,
         type: "string",
     })
         .option("llm_model", {
         description: "llm model",
         demandOption: false,
         type: "string",
+    })
+        .option("mode", {
+        description: "story to script generation mode",
+        demandOption: false,
+        choices: Object.values(storyToScriptGenerateMode),
+        default: storyToScriptGenerateMode.stepWise,
+        type: "string",
     })
         .positional("file", {
         description: "story file path",

package/lib/cli/commands/tool/story_to_script/handler.d.ts

@@ -1,4 +1,5 @@
 import { ToolCliArgs } from "../../../../types/cli_types.js";
+import { LLM } from "../../../../utils/utils.js";
 export declare const handler: (argv: ToolCliArgs<{
     o?: string;
     b?: string;
@@ -6,6 +7,7 @@ export declare const handler: (argv: ToolCliArgs<{
     s?: string;
     beats_per_scene?: number;
     file?: string;
-    llm_agent?: string;
+    llm?: LLM;
     llm_model?: string;
+    mode?: string;
 }>) => Promise<void>;

package/lib/cli/commands/tool/story_to_script/handler.js

@@ -5,7 +5,7 @@ import { mulmoStoryboardSchema } from "../../../../types/schema.js";
 import { getBaseDirPath, getFullPath, readAndParseJson } from "../../../../utils/file.js";
 import { outDirName } from "../../../../utils/const.js";
 export const handler = async (argv) => {
-    const { v: verbose, s: filename, file, o: outdir, b: basedir, beats_per_scene, llm_agent, llm_model } = argv;
+    const { v: verbose, s: filename, file, o: outdir, b: basedir, beats_per_scene, llm, llm_model, mode } = argv;
     let { t: template } = argv;
     const baseDirPath = getBaseDirPath(basedir);
     const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
@@ -19,7 +19,7 @@ export const handler = async (argv) => {
         fileName: filename,
         beatsPerScene: beats_per_scene,
         storyFilePath: file,
-        llmAgent: llm_agent,
+        llmAgent: llm,
         llmModel: llm_model,
     });
     const parsedStory = readAndParseJson(file, mulmoStoryboardSchema);
@@ -29,7 +29,8 @@ export const handler = async (argv) => {
         templateName: template,
         outdir: outDirPath,
         fileName: filename,
-        llmAgent: llm_agent,
+        llm,
         llmModel: llm_model,
+        generateMode: mode,
     });
 };

package/lib/methods/mulmo_script_template.d.ts

@@ -1,4 +1,4 @@
-import { MulmoScriptTemplate } from "../types/index.js";
+import { MulmoScript, MulmoScriptTemplate } from "../types/index.js";
 export declare const MulmoScriptTemplateMethods: {
-    getSystemPrompt(template: MulmoScriptTemplate): string;
+    getSystemPrompt(template: MulmoScriptTemplate, script?: MulmoScript): string;
 };

package/lib/methods/mulmo_script_template.js

@@ -1,6 +1,6 @@
 import { getMulmoScriptTemplateSystemPrompt } from "../utils/prompt.js";
 export const MulmoScriptTemplateMethods = {
-    getSystemPrompt(template) {
-        return getMulmoScriptTemplateSystemPrompt(template);
+    getSystemPrompt(template, script) {
+        return getMulmoScriptTemplateSystemPrompt(template, script);
     },
 };

package/lib/methods/mulmo_studio.d.ts

@@ -0,0 +1,8 @@
+import { MulmoStudio } from "../types/index.js";
+type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
+type BeatSessionType = "audio" | "image" | "multiLingual" | "caption";
+export declare const MulmoStudioMethods: {
+    setSessionState(studio: MulmoStudio, sessionType: SessionType, value: boolean): void;
+    setBeatSessionState(studio: MulmoStudio, sessionType: BeatSessionType, index: number, value: boolean): void;
+};
+export {};

package/lib/methods/mulmo_studio.js

@@ -0,0 +1,24 @@
+import { GraphAILogger } from "graphai";
+const notifyStateChange = (studio, sessionType) => {
+    const prefix = studio.state.inSession[sessionType] ? "<" : " >";
+    GraphAILogger.info(`${prefix} ${sessionType}`);
+};
+const notifyBeatStateChange = (studio, sessionType, index) => {
+    const prefix = studio.state.inBeatSession[sessionType].has(index) ? "{" : " }";
+    GraphAILogger.info(`${prefix} ${sessionType} ${index}`);
+};
+export const MulmoStudioMethods = {
+    setSessionState(studio, sessionType, value) {
+        studio.state.inSession[sessionType] = value;
+        notifyStateChange(studio, sessionType);
+    },
+    setBeatSessionState(studio, sessionType, index, value) {
+        if (value) {
+            studio.state.inBeatSession[sessionType].add(index);
+        }
+        else {
+            studio.state.inBeatSession[sessionType].delete(index);
+        }
+        notifyBeatStateChange(studio, sessionType, index);
+    },
+};

package/lib/tools/create_mulmo_script_from_url.d.ts

@@ -1,3 +1,3 @@
 import "dotenv/config";
 import { ScriptingParams } from "../types/index.js";
-export declare const createMulmoScriptFromUrl: ({ urls, templateName, outDirPath, filename, cacheDirPath, llm_agent, llm_model }: ScriptingParams) => Promise<void>;
+export declare const createMulmoScriptFromUrl: ({ urls, templateName, outDirPath, filename, cacheDirPath, llm, llm_model }: ScriptingParams) => Promise<void>;

package/lib/tools/create_mulmo_script_from_url.js

@@ -104,14 +104,14 @@ const graphData = {
                     },
                     llm: {
                         agent: ":llmAgent",
-                        params: {
-                            model: ":llmModel",
-                            system: ":prompt",
-                            max_tokens: ":maxTokens",
-                        },
                         inputs: {
                             system: ":prompt",
                             prompt: graphDataScriptFromUrlPrompt("${:sourceText.text}"),
+                            params: {
+                                model: ":llmModel",
+                                system: ":prompt",
+                                max_tokens: ":maxTokens",
+                            },
                         },
                     },
                     validateSchemaAgent: {
@@ -145,7 +145,7 @@ const graphData = {
         },
     },
 };
-export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath, filename, cacheDirPath, llm_agent, llm_model }) => {
+export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath, filename, cacheDirPath, llm, llm_model }) => {
     mkdir(outDirPath);
     mkdir(cacheDirPath);
     const parsedUrls = urlsSchema.parse(urls);
@@ -157,7 +157,7 @@ export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath,
             nodeIds: ["fetcher"],
         },
     ];
-    const { agent, model, max_tokens } = llmPair(llm_agent, llm_model);
+    const { agent, model, max_tokens } = llmPair(llm, llm_model);
     const graph = new GraphAI(graphData, {
         ...vanillaAgents,
         openAIAgent,

package/lib/tools/create_mulmo_script_interactively.d.ts

@@ -1,3 +1,3 @@
 import "dotenv/config";
 import { ScriptingParams } from "../types/index.js";
-export declare const createMulmoScriptInteractively: ({ outDirPath, cacheDirPath, filename, templateName, urls, llm_agent, llm_model }: ScriptingParams) => Promise<void>;
+export declare const createMulmoScriptInteractively: ({ outDirPath, cacheDirPath, filename, templateName, urls, llm, llm_model }: ScriptingParams) => Promise<void>;

package/lib/tools/create_mulmo_script_interactively.js

@@ -118,15 +118,15 @@ const graphData = {
                     },
                     chatAgent: {
                         agent: ":llmAgent",
-                        params: {
-                            model: ":llmModel",
-                            stream: true,
-                            dataStream: true,
-                            max_tokens: ":maxTokens",
-                        },
                         inputs: {
                             messages: ":messages",
                             prompt: ":prompt",
+                            params: {
+                                model: ":llmModel",
+                                stream: true,
+                                dataStream: true,
+                                max_tokens: ":maxTokens",
+                            },
                         },
                         isResult: true,
                     },
@@ -217,11 +217,11 @@ const scrapeWebContent = async (urls, cacheDirPath) => {
     }
     return `\n\n${prefixPrompt}\n${result?.sourceText.text}`;
 };
-export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath, filename, templateName, urls, llm_agent, llm_model }) => {
+export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath, filename, templateName, urls, llm, llm_model }) => {
     mkdir(outDirPath);
     // if urls is not empty, scrape web content and reference it in the prompt
     const webContentPrompt = urls.length > 0 ? await scrapeWebContent(urls, cacheDirPath) : "";
-    const { agent, model, max_tokens } = llmPair(llm_agent, llm_model);
+    const { agent, model, max_tokens } = llmPair(llm, llm_model);
     GraphAILogger.log({ agent, model, max_tokens });
     const agentFilters = [
         {

package/lib/tools/story_to_script.d.ts

@@ -1,10 +1,12 @@
-import { MulmoStoryboard } from "../types/index.js";
-export declare const storyToScript: ({ story, beatsPerScene, templateName, outdir, fileName, llmAgent, llmModel, }: {
+import { MulmoStoryboard, StoryToScriptGenerateMode } from "../types/index.js";
+import { LLM } from "../utils/utils.js";
+export declare const storyToScript: ({ story, beatsPerScene, templateName, outdir, fileName, llm, llmModel, generateMode, }: {
     story: MulmoStoryboard;
     beatsPerScene: number;
     templateName: string;
     outdir: string;
     fileName: string;
-    llmAgent?: string;
+    llm?: LLM;
     llmModel?: string;
+    generateMode: StoryToScriptGenerateMode;
 }) => Promise<void>;

package/lib/tools/story_to_script.js

@@ -1,5 +1,4 @@
-import path from "path";
-import { getTemplateFilePath, writingMessage } from "../utils/file.js";
+import { getTemplateFilePath, readAndParseJson, readScriptTemplateFile, writingMessage } from "../utils/file.js";
 import { mulmoScriptSchema, mulmoScriptTemplateSchema } from "../types/schema.js";
 import { GraphAI, GraphAILogger } from "graphai";
 import { openAIAgent } from "@graphai/openai_agent";
@@ -7,10 +6,12 @@ import { anthropicAgent } from "@graphai/anthropic_agent";
 import { geminiAgent } from "@graphai/gemini_agent";
 import { groqAgent } from "@graphai/groq_agent";
 import * as agents from "@graphai/vanilla";
-import { graphDataScriptGeneratePrompt, sceneToBeatsPrompt, storyToScriptInfoPrompt } from "../utils/prompt.js";
+import { graphDataScriptGeneratePrompt, sceneToBeatsPrompt, storyToScriptInfoPrompt, storyToScriptPrompt } from "../utils/prompt.js";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import validateSchemaAgent from "../agents/validate_schema_agent.js";
 import { llmPair } from "../utils/utils.js";
+import { storyToScriptGenerateMode } from "../utils/const.js";
+import { cliLoadingPlugin } from "../utils/plugins.js";
 const { default: __, ...vanillaAgents } = agents;
 const createValidatedScriptGraphData = ({ systemPrompt, prompt, schema, llmAgent, llmModel, maxTokens, }) => {
     return {
@@ -55,7 +56,7 @@ const createValidatedScriptGraphData = ({ systemPrompt, prompt, schema, llmAgent
         },
     };
 };
-const graphData = {
+const stepWiseGraphData = {
     version: 0.5,
     nodes: {
         scenes: {
@@ -167,35 +168,108 @@ const graphData = {
         },
     },
 };
+const oneStepGraphData = {
+    version: 0.5,
+    nodes: {
+        prompt: {
+            value: "",
+        },
+        outdir: {
+            value: "",
+        },
+        fileName: {
+            value: "",
+        },
+        llmAgent: {
+            value: "",
+        },
+        llmModel: {
+            value: "",
+        },
+        maxTokens: {
+            value: 0,
+        },
+        script: {
+            agent: "nestedAgent",
+            inputs: {
+                prompt: ":prompt",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
+            },
+            graph: createValidatedScriptGraphData({
+                systemPrompt: "",
+                prompt: graphDataScriptGeneratePrompt("${:prompt}"),
+                schema: mulmoScriptSchema,
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
+            }),
+        },
+        json: {
+            agent: "copyAgent",
+            inputs: {
+                json: ":script.validateSchema.data",
+            },
+            params: {
+                namedKey: "json",
+            },
+            isResult: true,
+        },
+        writeJSON: {
+            agent: "fileWriteAgent",
+            inputs: {
+                file: "${:outdir}/${:fileName}-${@now}.json",
+                text: ":json.toJSON()",
+            },
+            isResult: true,
+        },
+    },
+};
 const generateBeatsPrompt = async (template, beatsPerScene, story) => {
     const allScenes = story.scenes.map((scene) => scene.description).join("\n");
-    const sampleBeats = template.script?.beats ?? [];
+    const sampleBeats = template.scriptName ? readScriptTemplateFile(template.scriptName).beats : [];
     return sceneToBeatsPrompt({ sampleBeats, beatsPerScene, allScenes });
 };
 const generateScriptInfoPrompt = async (template, story) => {
-    if (!template.script) {
+    if (!template.scriptName) {
         // TODO: use default schema
         throw new Error("script is not provided");
     }
-    const { beats: __, ...sampleScriptWithoutBeats } = template.script;
-    return storyToScriptInfoPrompt(sampleScriptWithoutBeats, story);
+    const script = readScriptTemplateFile(template.scriptName);
+    const { beats: __, ...scriptWithoutBeats } = script;
+    return storyToScriptInfoPrompt(scriptWithoutBeats, story);
 };
-export const storyToScript = async ({ story, beatsPerScene, templateName, outdir, fileName, llmAgent, llmModel, }) => {
-    const templatePath = getTemplateFilePath(templateName);
-    const rowTemplate = await import(path.resolve(templatePath), { assert: { type: "json" } }).then((mod) => mod.default);
-    const template = mulmoScriptTemplateSchema.parse(rowTemplate);
-    const { agent, model, max_tokens } = llmPair(llmAgent, llmModel);
+const generateScriptPrompt = async (template, beatsPerScene, story) => {
+    if (!template.scriptName) {
+        // TODO: use default schema
+        throw new Error("script is not provided");
+    }
+    const script = readScriptTemplateFile(template.scriptName);
+    return storyToScriptPrompt(script, beatsPerScene, story);
+};
+export const storyToScript = async ({ story, beatsPerScene, templateName, outdir, fileName, llm, llmModel, generateMode, }) => {
+    const template = readAndParseJson(getTemplateFilePath(templateName), mulmoScriptTemplateSchema);
+    const { agent, model, max_tokens } = llmPair(llm, llmModel);
     const beatsPrompt = await generateBeatsPrompt(template, beatsPerScene, story);
     const scriptInfoPrompt = await generateScriptInfoPrompt(template, story);
+    const scriptPrompt = await generateScriptPrompt(template, beatsPerScene, story);
+    const graphData = generateMode === storyToScriptGenerateMode.stepWise ? stepWiseGraphData : oneStepGraphData;
     const graph = new GraphAI(graphData, { ...vanillaAgents, openAIAgent, anthropicAgent, geminiAgent, groqAgent, fileWriteAgent, validateSchemaAgent });
-    graph.injectValue("beatsPrompt", beatsPrompt);
-    graph.injectValue("scriptInfoPrompt", scriptInfoPrompt);
-    graph.injectValue("scenes", story.scenes);
+    if (generateMode === storyToScriptGenerateMode.stepWise) {
+        graph.injectValue("scenes", story.scenes);
+        graph.injectValue("beatsPrompt", beatsPrompt);
+        graph.injectValue("scriptInfoPrompt", scriptInfoPrompt);
+    }
+    else {
+        graph.injectValue("prompt", scriptPrompt);
+    }
     graph.injectValue("outdir", outdir);
     graph.injectValue("fileName", fileName);
     graph.injectValue("llmAgent", agent);
     graph.injectValue("llmModel", model);
     graph.injectValue("maxTokens", max_tokens);
+    graph.registerCallback(cliLoadingPlugin({ nodeId: "script", message: "Generating script..." }));
     const result = await graph.run();
     writingMessage(result?.writeJSON?.path ?? "");
 };