mulmocast 0.0.5 → 0.0.7

package/lib/tools/create_mulmo_script_from_url.js

@@ -1,14 +1,19 @@
 import "dotenv/config";
 import { GraphAI } from "graphai";
 import { openAIAgent } from "@graphai/openai_agent";
+import { anthropicAgent } from "@graphai/anthropic_agent";
+import { geminiAgent } from "@graphai/gemini_agent";
+import { groqAgent } from "@graphai/groq_agent";
 import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { browserlessAgent } from "@graphai/browserless_agent";
-import validateMulmoScriptAgent from "../agents/validate_mulmo_script_agent.js";
+import validateSchemaAgent from "../agents/validate_schema_agent.js";
 import { readTemplatePrompt, mkdir, writingMessage } from "../utils/file.js";
 import { browserlessCacheGenerator } from "../utils/filters.js";
-import { urlsSchema } from "../types/schema.js";
+import { mulmoScriptSchema, urlsSchema } from "../types/schema.js";
 import { cliLoadingPlugin } from "../utils/plugins.js";
+import { graphDataScriptFromUrlPrompt } from "../utils/prompt.js";
+import { llmPair } from "../utils/utils.js";
 const { default: __, ...vanillaAgents } = agents;
 const graphData = {
     version: 0.5,
@@ -27,6 +32,15 @@ const graphData = {
         fileName: {
             value: "",
         },
+        llmAgent: {
+            value: "",
+        },
+        llmModel: {
+            value: "",
+        },
+        maxTokens: {
+            value: 0,
+        },
         // get the text content of the urls
         fetchResults: {
             agent: "mapAgent",
@@ -74,6 +88,9 @@ const graphData = {
             inputs: {
                 sourceText: ":sourceText",
                 prompt: ":prompt",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
             },
             graph: {
                 loop: {
@@ -85,18 +102,23 @@ const graphData = {
                         value: 0,
                         update: ":counter.add(1)",
                     },
-                    openAIAgent: {
-                        agent: "openAIAgent",
+                    llm: {
+                        agent: ":llmAgent",
                         inputs: {
-                            model: "gpt-4o",
                             system: ":prompt",
-                            prompt: "Please create a script using the information from the following URLs as reference: ${:sourceText.text}",
+                            prompt: graphDataScriptFromUrlPrompt("${:sourceText.text}"),
+                            params: {
+                                model: ":llmModel",
+                                system: ":prompt",
+                                max_tokens: ":maxTokens",
+                            },
                         },
                     },
-                    validateMulmoScriptAgent: {
-                        agent: "validateMulmoScriptAgent",
+                    validateSchemaAgent: {
+                        agent: "validateSchemaAgent",
                         inputs: {
-                            text: ":openAIAgent.text.codeBlock()",
+                            text: ":llm.text.codeBlock()",
+                            schema: mulmoScriptSchema,
                         },
                         isResult: true,
                     },
@@ -105,7 +127,7 @@ const graphData = {
                             return !isValid && counter < 3;
                         },
                         inputs: {
-                            isValid: ":validateMulmoScriptAgent.isValid",
+                            isValid: ":validateSchemaAgent.isValid",
                             counter: ":counter",
                         },
                     },
@@ -113,17 +135,17 @@ const graphData = {
             },
         },
         writeJSON: {
-            if: ":mulmoScript.validateMulmoScriptAgent.isValid",
+            if: ":mulmoScript.validateSchemaAgent.isValid",
             agent: "fileWriteAgent",
             inputs: {
                 file: "${:outdir}/${:fileName}-${@now}.json",
-                text: ":mulmoScript.validateMulmoScriptAgent.data.toJSON()",
+                text: ":mulmoScript.validateSchemaAgent.data.toJSON()",
             },
             isResult: true,
         },
     },
 };
-export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath, filename, cacheDirPath }) => {
+export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath, filename, cacheDirPath, llm, llm_model }) => {
     mkdir(outDirPath);
     mkdir(cacheDirPath);
     const parsedUrls = urlsSchema.parse(urls);
@@ -135,17 +157,24 @@ export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath,
             nodeIds: ["fetcher"],
         },
     ];
+    const { agent, model, max_tokens } = llmPair(llm, llm_model);
     const graph = new GraphAI(graphData, {
         ...vanillaAgents,
         openAIAgent,
+        anthropicAgent,
+        geminiAgent,
+        groqAgent,
         browserlessAgent,
-        validateMulmoScriptAgent,
+        validateSchemaAgent,
         fileWriteAgent,
     }, { agentFilters });
     graph.injectValue("urls", parsedUrls);
     graph.injectValue("prompt", readTemplatePrompt(templateName));
     graph.injectValue("outdir", outDirPath);
     graph.injectValue("fileName", filename);
+    graph.injectValue("llmAgent", agent);
+    graph.injectValue("llmModel", model);
+    graph.injectValue("maxTokens", max_tokens);
     graph.registerCallback(cliLoadingPlugin({ nodeId: "mulmoScript", message: "Generating script..." }));
     const result = await graph.run();
     writingMessage(result?.writeJSON?.path ?? "");

package/lib/tools/create_mulmo_script_interactively.d.ts

@@ -1,3 +1,3 @@
 import "dotenv/config";
 import { ScriptingParams } from "../types/index.js";
-export declare const createMulmoScriptInteractively: ({ outDirPath, cacheDirPath, filename, templateName, urls, llm_agent, llm_model }: ScriptingParams) => Promise<void>;
+export declare const createMulmoScriptInteractively: ({ outDirPath, cacheDirPath, filename, templateName, urls, llm, llm_model }: ScriptingParams) => Promise<void>;

package/lib/tools/create_mulmo_script_interactively.js

@@ -1,7 +1,7 @@
 import "dotenv/config";
 import { GraphAILogger, GraphAI } from "graphai";
 import { textInputAgent } from "@graphai/input_agents";
-import { streamAgentFilterGenerator } from "@graphai/agent_filters";
+import { consoleStreamDataAgentFilter } from "@graphai/stream_agent_filter/node";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
 import { geminiAgent } from "@graphai/gemini_agent";
@@ -10,9 +10,11 @@ import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { readTemplatePrompt, mkdir } from "../utils/file.js";
 import { browserlessCacheGenerator } from "../utils/filters.js";
+import { mulmoScriptSchema } from "../types/index.js";
 import { browserlessAgent } from "@graphai/browserless_agent";
-import validateMulmoScriptAgent from "../agents/validate_mulmo_script_agent.js";
+import validateSchemaAgent from "../agents/validate_schema_agent.js";
 import { llmPair } from "../utils/utils.js";
+import { interactiveClarificationPrompt, prefixPrompt } from "../utils/prompt.js";
 // import { cliLoadingPlugin } from "../utils/plugins.js";
 const { default: __, ...vanillaAgents } = agents;
 const agentHeader = "\x1b[34m● \x1b[0m\x1b[1mAgent\x1b[0m:\x1b[0m";
@@ -116,21 +118,25 @@ const graphData = {
                     },
                     chatAgent: {
                         agent: ":llmAgent",
-                        params: {
-                            model: ":llmModel",
-                            stream: true,
-                            max_tokens: ":maxTokens",
-                        },
                         inputs: {
                             messages: ":messages",
                             prompt: ":prompt",
+                            params: {
+                                model: ":llmModel",
+                                stream: true,
+                                dataStream: true,
+                                max_tokens: ":maxTokens",
+                            },
                         },
                         isResult: true,
                     },
-                    validateMulmoScriptAgent: {
-                        agent: "validateMulmoScriptAgent",
+                    validateSchemaAgent: {
+                        if: ":chatAgent.text.codeBlock()",
+                        defaultValue: false,
+                        agent: "validateSchemaAgent",
                         inputs: {
                             text: ":chatAgent.text.codeBlock()",
+                            schema: mulmoScriptSchema,
                         },
                     },
                     continue: {
@@ -148,7 +154,7 @@ const graphData = {
                         inputs: {
                             counter: ":counter",
                             codeBlock: ":chatAgent.text.codeBlock()",
-                            isValid: ":validateMulmoScriptAgent.isValid",
+                            isValid: ":validateSchemaAgent.isValid",
                         },
                     },
                 },
@@ -192,7 +198,6 @@ const graphData = {
         },
     },
 };
-const interactiveClarificationPrompt = `If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.`;
 const scrapeWebContent = async (urls, cacheDirPath) => {
     mkdir(cacheDirPath);
     GraphAILogger.info(`${agentHeader} Scraping ${urls.length} URLs...\n`);
@@ -210,26 +215,22 @@ const scrapeWebContent = async (urls, cacheDirPath) => {
     if (!result?.sourceText?.text) {
         return "";
     }
-    const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
     return `\n\n${prefixPrompt}\n${result?.sourceText.text}`;
 };
-export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath, filename, templateName, urls, llm_agent, llm_model }) => {
+export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath, filename, templateName, urls, llm, llm_model }) => {
     mkdir(outDirPath);
     // if urls is not empty, scrape web content and reference it in the prompt
     const webContentPrompt = urls.length > 0 ? await scrapeWebContent(urls, cacheDirPath) : "";
-    const { agent, model, max_tokens } = llmPair(llm_agent, llm_model);
+    const { agent, model, max_tokens } = llmPair(llm, llm_model);
     GraphAILogger.log({ agent, model, max_tokens });
-    const streamAgentFilter = streamAgentFilterGenerator((context, data) => {
-        process.stdout.write(String(data));
-    });
     const agentFilters = [
         {
-            name: "streamAgentFilter",
-            agent: streamAgentFilter,
+            name: "consoleStreamDataAgentFilter",
+            agent: consoleStreamDataAgentFilter,
             nodeIds: ["chatAgent"],
         },
     ];
-    const graph = new GraphAI(graphData, { ...vanillaAgents, anthropicAgent, geminiAgent, groqAgent, openAIAgent, textInputAgent, fileWriteAgent, validateMulmoScriptAgent }, { agentFilters });
+    const graph = new GraphAI(graphData, { ...vanillaAgents, anthropicAgent, geminiAgent, groqAgent, openAIAgent, textInputAgent, fileWriteAgent, validateSchemaAgent }, { agentFilters });
     const prompt = readTemplatePrompt(templateName);
     graph.injectValue("messages", [
         {

package/lib/tools/dump_prompt.js

@@ -1,6 +1,8 @@
 import { GraphAILogger } from "graphai";
 import { readTemplatePrompt } from "../utils/file.js";
+import clipboardy from "clipboardy";
 export const dumpPromptFromTemplate = async ({ templateName }) => {
     const prompt = readTemplatePrompt(templateName);
     GraphAILogger.info(prompt);
+    clipboardy.write(prompt);
 };

package/lib/tools/story_to_script.d.ts

@@ -0,0 +1,12 @@
+import { MulmoStoryboard, StoryToScriptGenerateMode } from "../types/index.js";
+import { LLM } from "../utils/utils.js";
+export declare const storyToScript: ({ story, beatsPerScene, templateName, outdir, fileName, llm, llmModel, generateMode, }: {
+    story: MulmoStoryboard;
+    beatsPerScene: number;
+    templateName: string;
+    outdir: string;
+    fileName: string;
+    llm?: LLM;
+    llmModel?: string;
+    generateMode: StoryToScriptGenerateMode;
+}) => Promise<void>;

package/lib/tools/story_to_script.js

@@ -0,0 +1,275 @@
+import { getTemplateFilePath, readAndParseJson, readScriptTemplateFile, writingMessage } from "../utils/file.js";
+import { mulmoScriptSchema, mulmoScriptTemplateSchema } from "../types/schema.js";
+import { GraphAI, GraphAILogger } from "graphai";
+import { openAIAgent } from "@graphai/openai_agent";
+import { anthropicAgent } from "@graphai/anthropic_agent";
+import { geminiAgent } from "@graphai/gemini_agent";
+import { groqAgent } from "@graphai/groq_agent";
+import * as agents from "@graphai/vanilla";
+import { graphDataScriptGeneratePrompt, sceneToBeatsPrompt, storyToScriptInfoPrompt, storyToScriptPrompt } from "../utils/prompt.js";
+import { fileWriteAgent } from "@graphai/vanilla_node_agents";
+import validateSchemaAgent from "../agents/validate_schema_agent.js";
+import { llmPair } from "../utils/utils.js";
+import { storyToScriptGenerateMode } from "../utils/const.js";
+import { cliLoadingPlugin } from "../utils/plugins.js";
+const { default: __, ...vanillaAgents } = agents;
+const createValidatedScriptGraphData = ({ systemPrompt, prompt, schema, llmAgent, llmModel, maxTokens, }) => {
+    return {
+        loop: {
+            while: ":continue",
+        },
+        nodes: {
+            counter: {
+                value: 0,
+                update: ":counter.add(1)",
+            },
+            llm: {
+                agent: llmAgent,
+                inputs: {
+                    model: llmModel,
+                    system: systemPrompt,
+                    prompt: prompt,
+                    max_tokens: maxTokens,
+                },
+            },
+            validateSchema: {
+                agent: "validateSchemaAgent",
+                inputs: {
+                    text: ":llm.text.codeBlock()",
+                    schema: schema,
+                },
+                isResult: true,
+            },
+            continue: {
+                agent: ({ isValid, counter }) => {
+                    if (counter >= 3) {
+                        GraphAILogger.info("Failed to generate a valid script. Please try again.");
+                        process.exit(1);
+                    }
+                    return !isValid;
+                },
+                inputs: {
+                    counter: ":counter",
+                    isValid: ":validateSchema.isValid",
+                },
+            },
+        },
+    };
+};
+const stepWiseGraphData = {
+    version: 0.5,
+    nodes: {
+        scenes: {
+            value: [],
+        },
+        beatsPrompt: {
+            value: "",
+        },
+        scriptInfoPrompt: {
+            value: "",
+        },
+        outdir: {
+            value: "",
+        },
+        fileName: {
+            value: "",
+        },
+        llmAgent: {
+            value: "",
+        },
+        llmModel: {
+            value: "",
+        },
+        maxTokens: {
+            value: 0,
+        },
+        script: {
+            agent: "mapAgent",
+            inputs: {
+                rows: ":scenes",
+                prompt: ":beatsPrompt",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
+            },
+            params: {
+                compositeResult: true,
+            },
+            graph: {
+                nodes: {
+                    script: {
+                        agent: "nestedAgent",
+                        inputs: {
+                            prompt: ":prompt",
+                            row: ":row",
+                            llmAgent: ":llmAgent",
+                            llmModel: ":llmModel",
+                            maxTokens: ":maxTokens",
+                        },
+                        graph: createValidatedScriptGraphData({
+                            systemPrompt: ":prompt",
+                            prompt: graphDataScriptGeneratePrompt("${:row}"),
+                            schema: mulmoScriptSchema.shape.beats,
+                            llmAgent: ":llmAgent",
+                            llmModel: ":llmModel",
+                            maxTokens: ":maxTokens",
+                        }),
+                    },
+                    json: {
+                        agent: "copyAgent",
+                        inputs: {
+                            json: ":script.validateSchema.data",
+                        },
+                        params: {
+                            namedKey: "json",
+                        },
+                        isResult: true,
+                    },
+                },
+            },
+        },
+        beats: {
+            agent: "arrayFlatAgent",
+            inputs: {
+                array: ":script.json",
+            },
+            isResult: true,
+        },
+        scriptInfo: {
+            agent: "nestedAgent",
+            inputs: {
+                prompt: ":scriptInfoPrompt",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
+            },
+            graph: createValidatedScriptGraphData({
+                systemPrompt: "",
+                prompt: ":prompt",
+                schema: mulmoScriptSchema.omit({ beats: true }),
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
+            }),
+        },
+        mergedScript: {
+            agent: "mergeObjectAgent",
+            inputs: {
+                items: [":scriptInfo.validateSchema.data", { beats: ":beats.array" }],
+            },
+        },
+        writeJSON: {
+            agent: "fileWriteAgent",
+            inputs: {
+                file: "${:outdir}/${:fileName}-${@now}.json",
+                text: ":mergedScript.toJSON()",
+            },
+            isResult: true,
+        },
+    },
+};
+const oneStepGraphData = {
+    version: 0.5,
+    nodes: {
+        prompt: {
+            value: "",
+        },
+        outdir: {
+            value: "",
+        },
+        fileName: {
+            value: "",
+        },
+        llmAgent: {
+            value: "",
+        },
+        llmModel: {
+            value: "",
+        },
+        maxTokens: {
+            value: 0,
+        },
+        script: {
+            agent: "nestedAgent",
+            inputs: {
+                prompt: ":prompt",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
+            },
+            graph: createValidatedScriptGraphData({
+                systemPrompt: "",
+                prompt: graphDataScriptGeneratePrompt("${:prompt}"),
+                schema: mulmoScriptSchema,
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
+            }),
+        },
+        json: {
+            agent: "copyAgent",
+            inputs: {
+                json: ":script.validateSchema.data",
+            },
+            params: {
+                namedKey: "json",
+            },
+            isResult: true,
+        },
+        writeJSON: {
+            agent: "fileWriteAgent",
+            inputs: {
+                file: "${:outdir}/${:fileName}-${@now}.json",
+                text: ":json.toJSON()",
+            },
+            isResult: true,
+        },
+    },
+};
+const generateBeatsPrompt = async (template, beatsPerScene, story) => {
+    const allScenes = story.scenes.map((scene) => scene.description).join("\n");
+    const sampleBeats = template.scriptName ? readScriptTemplateFile(template.scriptName).beats : [];
+    return sceneToBeatsPrompt({ sampleBeats, beatsPerScene, allScenes });
+};
+const generateScriptInfoPrompt = async (template, story) => {
+    if (!template.scriptName) {
+        // TODO: use default schema
+        throw new Error("script is not provided");
+    }
+    const script = readScriptTemplateFile(template.scriptName);
+    const { beats: __, ...scriptWithoutBeats } = script;
+    return storyToScriptInfoPrompt(scriptWithoutBeats, story);
+};
+const generateScriptPrompt = async (template, beatsPerScene, story) => {
+    if (!template.scriptName) {
+        // TODO: use default schema
+        throw new Error("script is not provided");
+    }
+    const script = readScriptTemplateFile(template.scriptName);
+    return storyToScriptPrompt(script, beatsPerScene, story);
+};
+export const storyToScript = async ({ story, beatsPerScene, templateName, outdir, fileName, llm, llmModel, generateMode, }) => {
+    const template = readAndParseJson(getTemplateFilePath(templateName), mulmoScriptTemplateSchema);
+    const { agent, model, max_tokens } = llmPair(llm, llmModel);
+    const beatsPrompt = await generateBeatsPrompt(template, beatsPerScene, story);
+    const scriptInfoPrompt = await generateScriptInfoPrompt(template, story);
+    const scriptPrompt = await generateScriptPrompt(template, beatsPerScene, story);
+    const graphData = generateMode === storyToScriptGenerateMode.stepWise ? stepWiseGraphData : oneStepGraphData;
+    const graph = new GraphAI(graphData, { ...vanillaAgents, openAIAgent, anthropicAgent, geminiAgent, groqAgent, fileWriteAgent, validateSchemaAgent });
+    if (generateMode === storyToScriptGenerateMode.stepWise) {
+        graph.injectValue("scenes", story.scenes);
+        graph.injectValue("beatsPrompt", beatsPrompt);
+        graph.injectValue("scriptInfoPrompt", scriptInfoPrompt);
+    }
+    else {
+        graph.injectValue("prompt", scriptPrompt);
+    }
+    graph.injectValue("outdir", outdir);
+    graph.injectValue("fileName", fileName);
+    graph.injectValue("llmAgent", agent);
+    graph.injectValue("llmModel", model);
+    graph.injectValue("maxTokens", max_tokens);
+    graph.registerCallback(cliLoadingPlugin({ nodeId: "script", message: "Generating script..." }));
+    const result = await graph.run();
+    writingMessage(result?.writeJSON?.path ?? "");
+};

package/lib/types/cli_types.d.ts

@@ -0,0 +1,14 @@
+import { Arguments } from "yargs";
+export type ToolGlobalOptions = {
+    v?: boolean;
+};
+export type ToolCliArgs<T = object> = Arguments<T & ToolGlobalOptions>;
+export type GlobalOptions = {
+    v?: boolean;
+    o?: string;
+    b?: string;
+    l?: string;
+    f?: boolean;
+    file?: string;
+};
+export type CliArgs<T = object> = Arguments<T & GlobalOptions>;

package/lib/types/cli_types.js

@@ -0,0 +1 @@
+export {};