mulmocast 0.0.1 → 0.0.3

package/lib/tools/create_mulmo_script_from_url.js

@@ -0,0 +1,152 @@
+import "dotenv/config";
+import { GraphAI } from "graphai";
+import { openAIAgent } from "@graphai/openai_agent";
+import * as agents from "@graphai/vanilla";
+import { fileWriteAgent } from "@graphai/vanilla_node_agents";
+import { browserlessAgent } from "@graphai/browserless_agent";
+import validateMulmoScriptAgent from "../agents/validate_mulmo_script_agent.js";
+import { readTemplatePrompt, mkdir, writingMessage } from "../utils/file.js";
+import { browserlessCacheGenerator } from "../utils/filters.js";
+import { urlsSchema } from "../types/schema.js";
+import { cliLoadingPlugin } from "../utils/plugins.js";
+const { default: __, ...vanillaAgents } = agents;
+const graphData = {
+    version: 0.5,
+    // Execute sequentially because the free version of browserless API doesn't support concurrent execution.
+    concurrency: 1,
+    nodes: {
+        urls: {
+            value: [],
+        },
+        prompt: {
+            value: "",
+        },
+        outdir: {
+            value: "",
+        },
+        fileName: {
+            value: "",
+        },
+        // get the text content of the urls
+        fetchResults: {
+            agent: "mapAgent",
+            inputs: {
+                rows: ":urls",
+            },
+            params: {
+                compositeResult: true,
+            },
+            graph: {
+                nodes: {
+                    fetcher: {
+                        agent: "browserlessAgent",
+                        inputs: {
+                            url: ":row",
+                            text_content: true,
+                        },
+                    },
+                    copyAgent: {
+                        agent: "copyAgent",
+                        inputs: {
+                            text: '{ url: "${:row}", text: "${:fetcher.text}" }',
+                        },
+                        params: {
+                            namedKey: "text",
+                        },
+                        isResult: true,
+                    },
+                },
+            },
+        },
+        // join the text content
+        sourceText: {
+            agent: "arrayJoinAgent",
+            inputs: {
+                array: ":fetchResults.copyAgent",
+            },
+            params: {
+                separator: ",",
+            },
+        },
+        // generate the mulmo script
+        mulmoScript: {
+            agent: "nestedAgent",
+            inputs: {
+                sourceText: ":sourceText",
+                prompt: ":prompt",
+            },
+            graph: {
+                loop: {
+                    // If the script is not valid and the counter is less than 3, continue the loop
+                    while: ":continue",
+                },
+                nodes: {
+                    counter: {
+                        value: 0,
+                        update: ":counter.add(1)",
+                    },
+                    openAIAgent: {
+                        agent: "openAIAgent",
+                        inputs: {
+                            model: "gpt-4o",
+                            system: ":prompt",
+                            prompt: "Please create a script using the information from the following URLs as reference: ${:sourceText.text}",
+                        },
+                    },
+                    validateMulmoScriptAgent: {
+                        agent: "validateMulmoScriptAgent",
+                        inputs: {
+                            text: ":openAIAgent.text.codeBlock()",
+                        },
+                        isResult: true,
+                    },
+                    continue: {
+                        agent: ({ isValid, counter }) => {
+                            return !isValid && counter < 3;
+                        },
+                        inputs: {
+                            isValid: ":validateMulmoScriptAgent.isValid",
+                            counter: ":counter",
+                        },
+                    },
+                },
+            },
+        },
+        writeJSON: {
+            if: ":mulmoScript.validateMulmoScriptAgent.isValid",
+            agent: "fileWriteAgent",
+            inputs: {
+                file: "${:outdir}/${:fileName}-${@now}.json",
+                text: ":mulmoScript.validateMulmoScriptAgent.data.toJSON()",
+            },
+            isResult: true,
+        },
+    },
+};
+export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath, filename, cacheDirPath }) => {
+    mkdir(outDirPath);
+    mkdir(cacheDirPath);
+    const parsedUrls = urlsSchema.parse(urls);
+    const browserlessCache = browserlessCacheGenerator(cacheDirPath);
+    const agentFilters = [
+        {
+            name: "browserlessCache",
+            agent: browserlessCache,
+            nodeIds: ["fetcher"],
+        },
+    ];
+    const graph = new GraphAI(graphData, {
+        ...vanillaAgents,
+        openAIAgent,
+        browserlessAgent,
+        validateMulmoScriptAgent,
+        fileWriteAgent,
+    }, { agentFilters });
+    graph.injectValue("urls", parsedUrls);
+    graph.injectValue("prompt", readTemplatePrompt(templateName));
+    graph.injectValue("outdir", outDirPath);
+    graph.injectValue("fileName", filename);
+    graph.registerCallback(cliLoadingPlugin({ nodeId: "mulmoScript", message: "Generating script..." }));
+    const result = await graph.run();
+    writingMessage(result?.writeJSON?.path ?? "");
+};

package/lib/tools/create_mulmo_script_interactively.d.ts

@@ -0,0 +1,3 @@
+import "dotenv/config";
+import { ScriptingParams } from "../types/index.js";
+export declare const createMulmoScriptInteractively: ({ outDirPath, cacheDirPath, filename, templateName, urls, llm_agent, llm_model }: ScriptingParams) => Promise<void>;

package/lib/tools/create_mulmo_script_interactively.js

@@ -0,0 +1,258 @@
+import "dotenv/config";
+import { GraphAILogger, GraphAI } from "graphai";
+import { textInputAgent } from "@graphai/input_agents";
+import { streamAgentFilterGenerator } from "@graphai/agent_filters";
+import { openAIAgent } from "@graphai/openai_agent";
+import { anthropicAgent } from "@graphai/anthropic_agent";
+import { geminiAgent } from "@graphai/gemini_agent";
+import { groqAgent } from "@graphai/groq_agent";
+import * as agents from "@graphai/vanilla";
+import { fileWriteAgent } from "@graphai/vanilla_node_agents";
+import { readTemplatePrompt, mkdir } from "../utils/file.js";
+import { browserlessCacheGenerator } from "../utils/filters.js";
+import { browserlessAgent } from "@graphai/browserless_agent";
+import validateMulmoScriptAgent from "../agents/validate_mulmo_script_agent.js";
+import { llmPair } from "../utils/utils.js";
+// import { cliLoadingPlugin } from "../utils/plugins.js";
+const { default: __, ...vanillaAgents } = agents;
+const agentHeader = "\x1b[34m● \x1b[0m\x1b[1mAgent\x1b[0m:\x1b[0m";
+const graphDataForScraping = {
+    version: 0.5,
+    nodes: {
+        urls: {
+            value: [],
+        },
+        fetchResults: {
+            agent: "mapAgent",
+            inputs: {
+                rows: ":urls",
+            },
+            params: {
+                compositeResult: true,
+            },
+            graph: {
+                nodes: {
+                    fetcher: {
+                        agent: "browserlessAgent",
+                        inputs: {
+                            url: ":row",
+                            text_content: true,
+                        },
+                    },
+                    copyAgent: {
+                        agent: "copyAgent",
+                        inputs: {
+                            text: '{ url: "${:row}", text: "${:fetcher.text}" }',
+                        },
+                        params: {
+                            namedKey: "text",
+                        },
+                        isResult: true,
+                    },
+                },
+            },
+        },
+        sourceText: {
+            agent: "arrayJoinAgent",
+            inputs: {
+                array: ":fetchResults.copyAgent",
+            },
+            params: {
+                separator: ",",
+            },
+            isResult: true,
+        },
+    },
+};
+const graphData = {
+    version: 0.5,
+    loop: {
+        while: ":continue",
+    },
+    nodes: {
+        llmAgent: {
+            update: ":llmAgent",
+        },
+        llmModel: {
+            update: ":llmModel",
+        },
+        maxTokens: {
+            update: ":maxTokens",
+        },
+        fileName: {
+            update: ":fileName",
+        },
+        outdir: {
+            update: ":outdir",
+        },
+        messages: {
+            value: [],
+            update: ":reply.chatAgent.messages",
+        },
+        userInput: {
+            agent: "textInputAgent",
+            params: {
+                message: "You:",
+                required: true,
+            },
+        },
+        reply: {
+            agent: "nestedAgent",
+            inputs: {
+                messages: ":messages",
+                prompt: ":userInput.text",
+                llmAgent: ":llmAgent",
+                llmModel: ":llmModel",
+                maxTokens: ":maxTokens",
+            },
+            graph: {
+                loop: {
+                    while: ":continue",
+                },
+                nodes: {
+                    counter: {
+                        value: 0,
+                        update: ":counter.add(1)",
+                    },
+                    chatAgent: {
+                        agent: ":llmAgent",
+                        params: {
+                            model: ":llmModel",
+                            stream: true,
+                            max_tokens: ":maxTokens",
+                        },
+                        inputs: {
+                            messages: ":messages",
+                            prompt: ":prompt",
+                        },
+                        isResult: true,
+                    },
+                    validateMulmoScriptAgent: {
+                        agent: "validateMulmoScriptAgent",
+                        inputs: {
+                            text: ":chatAgent.text.codeBlock()",
+                        },
+                    },
+                    continue: {
+                        agent: ({ codeBlock, isValid, counter }) => {
+                            if (counter >= 3) {
+                                GraphAILogger.info("\n" + agentHeader + " \x1b[31mFailed to generate a valid script. Please try again.\n");
+                                return false;
+                            }
+                            const result = !!codeBlock && !isValid;
+                            if (result) {
+                                GraphAILogger.info("\n" + agentHeader + " Generated script was broken. Retry generate a script.");
+                            }
+                            return result;
+                        },
+                        inputs: {
+                            counter: ":counter",
+                            codeBlock: ":chatAgent.text.codeBlock()",
+                            isValid: ":validateMulmoScriptAgent.isValid",
+                        },
+                    },
+                },
+            },
+        },
+        json: {
+            agent: "copyAgent",
+            inputs: {
+                json: ":reply.chatAgent.text.codeBlock().jsonParse()",
+                text: ":reply.chatAgent.text.codeBlock()",
+            },
+        },
+        writeJSON: {
+            if: ":json.json",
+            agent: "fileWriteAgent",
+            inputs: {
+                file: "${:outdir}/${:fileName}-${@now}.json",
+                text: ":json.text",
+            },
+        },
+        writeLog: {
+            agent: "consoleAgent",
+            inputs: {
+                text: "\n\x1b[32m🎉 Script file generated successfully! Type /bye to exit.\x1b[0m\nwriting: ${:writeJSON.path}",
+                waiting: ":writeJSON",
+            },
+        },
+        shouldResponse: {
+            agent: "compareAgent",
+            inputs: {
+                array: [[":json.json", "==", undefined], "&&", [":userInput.text", "!=", "/bye"]],
+            },
+        },
+        checkInput: {
+            agent: "compareAgent",
+            inputs: { array: [":userInput.text", "!=", "/bye"] },
+        },
+        continue: {
+            value: true,
+            update: ":checkInput.result",
+        },
+    },
+};
+const interactiveClarificationPrompt = `If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.`;
+const scrapeWebContent = async (urls, cacheDirPath) => {
+    mkdir(cacheDirPath);
+    GraphAILogger.info(`${agentHeader} Scraping ${urls.length} URLs...\n`);
+    const browserlessCache = browserlessCacheGenerator(cacheDirPath);
+    const agentFilters = [
+        {
+            name: "browserlessCache",
+            agent: browserlessCache,
+            nodeIds: ["fetcher"],
+        },
+    ];
+    const graph = new GraphAI(graphDataForScraping, { ...vanillaAgents, openAIAgent, textInputAgent, fileWriteAgent, browserlessAgent }, { agentFilters });
+    graph.injectValue("urls", urls);
+    const result = (await graph.run());
+    if (!result?.sourceText?.text) {
+        return "";
+    }
+    const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
+    return `\n\n${prefixPrompt}\n${result?.sourceText.text}`;
+};
+export const createMulmoScriptInteractively = async ({ outDirPath, cacheDirPath, filename, templateName, urls, llm_agent, llm_model }) => {
+    mkdir(outDirPath);
+    // if urls is not empty, scrape web content and reference it in the prompt
+    const webContentPrompt = urls.length > 0 ? await scrapeWebContent(urls, cacheDirPath) : "";
+    const { agent, model, max_tokens } = llmPair(llm_agent, llm_model);
+    GraphAILogger.log({ agent, model, max_tokens });
+    const streamAgentFilter = streamAgentFilterGenerator((context, data) => {
+        process.stdout.write(String(data));
+    });
+    const agentFilters = [
+        {
+            name: "streamAgentFilter",
+            agent: streamAgentFilter,
+            nodeIds: ["chatAgent"],
+        },
+    ];
+    const graph = new GraphAI(graphData, { ...vanillaAgents, anthropicAgent, geminiAgent, groqAgent, openAIAgent, textInputAgent, fileWriteAgent, validateMulmoScriptAgent }, { agentFilters });
+    const prompt = readTemplatePrompt(templateName);
+    graph.injectValue("messages", [
+        {
+            role: "system",
+            content: `${prompt}\n\n${interactiveClarificationPrompt}${webContentPrompt}`,
+        },
+    ]);
+    graph.injectValue("outdir", outDirPath);
+    graph.injectValue("fileName", filename);
+    graph.injectValue("llmAgent", agent);
+    graph.injectValue("llmModel", model);
+    graph.injectValue("maxTokens", max_tokens);
+    graph.registerCallback(({ nodeId, state }) => {
+        if (nodeId === "chatAgent") {
+            if (state === "executing") {
+                process.stdout.write(String("\n" + agentHeader + " "));
+            }
+            if (state === "completed") {
+                process.stdout.write("\n\n");
+            }
+        }
+    });
+    // graph.registerCallback(cliLoadingPlugin({ nodeId: "reply", message: "Loading..." }));
+    GraphAILogger.info(`${agentHeader} Hi! What topic would you like me to generate about?\n`);
+    await graph.run();
+};

package/lib/tools/dump_prompt.js

@@ -1,9 +1,6 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.dumpPromptFromTemplate = void 0;
-const file_1 = require("../utils/file");
-const dumpPromptFromTemplate = async ({ templateName }) => {
-    const prompt = (0, file_1.readTemplatePrompt)(templateName);
-    console.log(prompt);
+import { GraphAILogger } from "graphai";
+import { readTemplatePrompt } from "../utils/file.js";
+export const dumpPromptFromTemplate = async ({ templateName }) => {
+    const prompt = readTemplatePrompt(templateName);
+    GraphAILogger.info(prompt);
 };
-exports.dumpPromptFromTemplate = dumpPromptFromTemplate;

package/lib/tools/prompt.js

@@ -1,20 +1,18 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const fs_1 = __importDefault(require("fs"));
-const path_1 = __importDefault(require("path"));
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 const main = async () => {
-    const basePath = path_1.default.resolve(__dirname + "/../../prompts/");
+    const basePath = path.resolve(__dirname, "../../prompts/");
     const ret = {};
-    fs_1.default.readdirSync(basePath).map((file) => {
+    fs.readdirSync(basePath).map((file) => {
         const key = file.split(".")[0];
-        const content = fs_1.default.readFileSync(path_1.default.resolve(basePath) + "/" + file, "utf-8");
+        const content = fs.readFileSync(path.resolve(basePath, file), "utf-8");
         ret[key] = content;
     });
     const code = ["// This file is auto-generated by src/tools/prompt.ts and Do not be edited manually.", `export const prompts = ${JSON.stringify(ret)};`].join("\n");
-    fs_1.default.writeFileSync(path_1.default.resolve(__dirname + "/../../src/agents/prompts_data.ts"), code, "utf8");
+    fs.writeFileSync(path.resolve(__dirname, "../../src/agents/prompts_data.ts"), code, "utf8");
     // console.log(hoge);
 };
 main();

package/lib/tools/seed_from_url2.d.ts

@@ -0,0 +1,3 @@
+import "dotenv/config";
+import { ScriptingParams } from "../types";
+export declare const createMulmoScriptFromUrl: ({ urls, templateName, outDirPath, filename, cacheDirPath }: ScriptingParams) => Promise<void>;

package/lib/tools/seed_from_url2.js

@@ -0,0 +1,154 @@
+import "dotenv/config";
+import { GraphAI } from "graphai";
+import { openAIAgent } from "@graphai/openai_agent";
+import * as vanilla from "@graphai/vanilla";
+import { fileWriteAgent } from "@graphai/vanilla_node_agents";
+import { browserlessAgent } from "@graphai/browserless_agent";
+import validateMulmoScriptAgent from "../agents/validate_mulmo_script_agent";
+import { readTemplatePrompt, mkdir, writingMessage } from "../utils/file";
+import { browserlessCacheGenerator } from "../utils/filters";
+import { urlsSchema } from "../types/schema";
+import { cliLoadingPlugin } from "../utils/plugins";
+const graphData = {
+    version: 0.5,
+    // Execute sequentially because the free version of browserless API doesn't support concurrent execution.
+    concurrency: 1,
+    nodes: {
+        urls: {
+            value: [],
+        },
+        prompt: {
+            value: "",
+        },
+        outdir: {
+            value: "",
+        },
+        fileName: {
+            value: "",
+        },
+        fetchGraph: {
+            value: {
+                nodes: {
+                    fetcher: {
+                        agent: "browserlessAgent",
+                        inputs: {
+                            url: ":row",
+                            text_content: true,
+                        },
+                    },
+                    copyAgent: {
+                        agent: "copyAgent",
+                        inputs: {
+                            text: '{ url: "${:row}", text: "${:fetcher.text}" }',
+                        },
+                        params: {
+                            namedKey: "text",
+                        },
+                        isResult: true,
+                    },
+                },
+            },
+        },
+        // get the text content of the urls
+        fetchResults: {
+            agent: "mapAgent",
+            inputs: {
+                rows: ":urls",
+            },
+            params: {
+                compositeResult: true,
+            },
+            graph: ":fetchGraph",
+        },
+        // join the text content
+        sourceText: {
+            agent: "arrayJoinAgent",
+            inputs: {
+                array: ":fetchResults.copyAgent",
+            },
+            params: {
+                separator: ",",
+            },
+        },
+        // generate the mulmo script
+        mulmoScript: {
+            agent: "nestedAgent",
+            inputs: {
+                sourceText: ":sourceText",
+                prompt: ":prompt",
+            },
+            graph: {
+                loop: {
+                    // If the script is not valid and the counter is less than 3, continue the loop
+                    while: ":continue",
+                },
+                nodes: {
+                    counter: {
+                        value: 0,
+                        update: ":counter.add(1)",
+                    },
+                    openAIAgent: {
+                        agent: "openAIAgent",
+                        inputs: {
+                            model: "gpt-4o",
+                            system: ":prompt",
+                            prompt: "Please create a script using the information from the following URLs as reference: ${:sourceText.text}",
+                        },
+                    },
+                    validateMulmoScriptAgent: {
+                        agent: "validateMulmoScriptAgent",
+                        inputs: {
+                            text: ":openAIAgent.text.codeBlock()",
+                        },
+                        isResult: true,
+                    },
+                    continue: {
+                        agent: ({ isValid, counter }) => {
+                            return !isValid && counter < 3;
+                        },
+                        inputs: {
+                            isValid: ":validateMulmoScriptAgent.isValid",
+                            counter: ":counter",
+                        },
+                    },
+                },
+            },
+        },
+        writeJSON: {
+            if: ":mulmoScript.validateMulmoScriptAgent.isValid",
+            agent: "fileWriteAgent",
+            inputs: {
+                file: "${:outdir}/${:fileName}-${@now}.json",
+                text: ":mulmoScript.validateMulmoScriptAgent.data.toJSON()",
+            },
+            isResult: true,
+        },
+    },
+};
+export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath, filename, cacheDirPath }) => {
+    mkdir(outDirPath);
+    mkdir(cacheDirPath);
+    const parsedUrls = urlsSchema.parse(urls);
+    const browserlessCache = browserlessCacheGenerator(cacheDirPath);
+    const agentFilters = [
+        {
+            name: "browserlessCache",
+            agent: browserlessCache,
+            nodeIds: ["fetcher"],
+        },
+    ];
+    const graph = new GraphAI(graphData, {
+        ...vanilla,
+        openAIAgent,
+        browserlessAgent,
+        validateMulmoScriptAgent,
+        fileWriteAgent,
+    }, { agentFilters });
+    graph.injectValue("urls", parsedUrls);
+    graph.injectValue("prompt", readTemplatePrompt(templateName));
+    graph.injectValue("outdir", outDirPath);
+    graph.injectValue("fileName", filename);
+    graph.registerCallback(cliLoadingPlugin({ nodeId: "mulmoScript", message: "Generating script..." }));
+    const result = await graph.run();
+    writingMessage(result?.writeJSON?.path ?? "");
+};

package/lib/types/index.d.ts

@@ -1 +1,2 @@
-export * from "./type";
+export * from "./type.js";
+export * from "./schema.js";

package/lib/types/index.js

@@ -1,17 +1,2 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./type"), exports);
+export * from "./type.js";
+export * from "./schema.js";