mulmocast 1.1.0 → 1.1.2

package/lib/index.browser.d.ts

@@ -1,2 +1,3 @@
 export * from "./index.common.js";
-export * from "./agents/validate_schema_agent.js";
+import validateSchemaAgent from "./agents/validate_schema_agent.js";
+export { validateSchemaAgent };

package/lib/index.browser.js

@@ -1,3 +1,4 @@
 // Entry point for browser code
 export * from "./index.common.js";
-export * from "./agents/validate_schema_agent.js";
+import validateSchemaAgent from "./agents/validate_schema_agent.js";
+export { validateSchemaAgent };

package/lib/index.common.d.ts

@@ -1,2 +1,3 @@
 export * from "./types/index.js";
 export * from "./utils/provider2agent.js";
+export * from "./utils/templates.js";

package/lib/index.common.js

@@ -1,3 +1,4 @@
 // Entry point for universal code
 export * from "./types/index.js";
 export * from "./utils/provider2agent.js";
+export * from "./utils/templates.js";

package/lib/methods/index.d.ts

@@ -1,5 +1,4 @@
 export * from "./mulmo_presentation_style.js";
-export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
 export * from "./mulmo_beat.js";

package/lib/methods/index.js

@@ -1,5 +1,4 @@
 export * from "./mulmo_presentation_style.js";
-export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
 export * from "./mulmo_beat.js";

package/lib/methods/mulmo_presentation_style.d.ts

@@ -32,6 +32,16 @@ export declare const MulmoPresentationStyleMethods: {
             identifier?: `${string}/${string}:${string}`;
         }>;
     };
+    getLipSyncAgentInfo(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): {
+        agentName: string;
+        defaultModel: import("../utils/provider2agent.js").ReplicateModel;
+        models: import("../utils/provider2agent.js").ReplicateModel[];
+        modelParams: Record<import("../utils/provider2agent.js").ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+            video: string;
+            audio: string;
+        }>;
+    };
     getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;
     getHtmlImageAgentInfo(presentationStyle: MulmoPresentationStyle): Text2HtmlAgentInfo;
     getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;

package/lib/methods/mulmo_presentation_style.js

@@ -2,7 +2,7 @@ import "dotenv/config";
 import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, } from "../types/schema.js";
-import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2SoundEffectAgent, defaultProviders, } from "../utils/provider2agent.js";
+import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, } from "../utils/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -93,6 +93,13 @@ export const MulmoPresentationStyleMethods = {
         const agentInfo = provider2SoundEffectAgent[soundEffectProvider];
         return agentInfo;
     },
+    getLipSyncAgentInfo(presentationStyle, beat) {
+        const lipSyncProvider = (beat.lipSyncParams?.provider ??
+            presentationStyle.lipSyncParams?.provider ??
+            defaultProviders.lipSync);
+        const agentInfo = provider2LipSyncAgent[lipSyncProvider];
+        return agentInfo;
+    },
     getConcurrency(presentationStyle) {
         const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(presentationStyle);
         if (imageAgentInfo.imageParams.provider === "openai") {

package/lib/methods/mulmo_script_template.d.ts

@@ -1,4 +1,4 @@
-import { MulmoScript, MulmoScriptTemplate } from "../types/index.js";
+import { MulmoScript, MulmoPromptTemplate } from "../types/index.js";
 export declare const MulmoScriptTemplateMethods: {
-    getSystemPrompt(template: MulmoScriptTemplate, script?: MulmoScript): string;
+    getSystemPrompt(template: MulmoPromptTemplate, script?: MulmoScript): string;
 };

package/lib/tools/create_mulmo_script_from_url.js

@@ -1,6 +1,6 @@
 import "dotenv/config";
 import path from "path";
-import { GraphAI } from "graphai";
+import { GraphAI, GraphAILogger } from "graphai";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
 import { geminiAgent } from "@graphai/gemini_agent";
@@ -9,7 +9,8 @@ import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { browserlessAgent } from "@graphai/browserless_agent";
 import validateSchemaAgent from "../agents/validate_schema_agent.js";
-import { readTemplatePrompt, mkdir, writingMessage } from "../utils/file.js";
+import { mkdir, writingMessage } from "../utils/file.js";
+import { readTemplatePrompt } from "../utils/templates.js";
 import { browserlessCacheGenerator } from "../utils/filters.js";
 import { mulmoScriptSchema, urlsSchema } from "../types/schema.js";
 import { cliLoadingPlugin } from "../utils/plugins.js";
@@ -17,6 +18,9 @@ import { graphDataScriptFromUrlPrompt } from "../utils/prompt.js";
 import { llmPair } from "../utils/utils.js";
 import { readFileSync } from "fs";
 const vanillaAgents = agents.default ?? agents;
+const showErrorMessage = (text) => {
+    GraphAILogger.info("\x1b[31m" + text + "\x1b[0m");
+};
 const graphMulmoScript = {
     version: 0.5,
     loop: {
@@ -235,6 +239,10 @@ export const createMulmoScriptFromUrl = async ({ urls, templateName, outDirPath,
     graph.injectValue("maxTokens", max_tokens);
     graph.registerCallback(cliLoadingPlugin({ nodeId: "mulmoScript", message: "Generating script..." }));
     const result = await graph.run();
+    if (!result?.writeJSON?.path) {
+        showErrorMessage("Script generation failed. Please try again.");
+        return;
+    }
     writingMessage(result?.writeJSON?.path ?? "");
 };
 export const createMulmoScriptFromFile = async (fileName, { templateName, outDirPath, filename, cacheDirPath, llm, llm_model, verbose }) => {
@@ -263,5 +271,9 @@ export const createMulmoScriptFromFile = async (fileName, { templateName, outDir
         graph.registerCallback(cliLoadingPlugin({ nodeId: "mulmoScript", message: "Generating script..." }));
     }
     const result = await graph.run();
+    if (!result?.writeJSON?.path) {
+        showErrorMessage("Script generation failed. Please try again.");
+        return;
+    }
     writingMessage(result?.writeJSON?.path ?? "");
 };

package/lib/tools/create_mulmo_script_interactively.js

@@ -8,7 +8,8 @@ import { geminiAgent } from "@graphai/gemini_agent";
 import { groqAgent } from "@graphai/groq_agent";
 import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { readTemplatePrompt, mkdir } from "../utils/file.js";
+import { mkdir } from "../utils/file.js";
+import { readTemplatePrompt } from "../utils/templates.js";
 import { browserlessCacheGenerator } from "../utils/filters.js";
 import { mulmoScriptSchema } from "../types/index.js";
 import { browserlessAgent } from "@graphai/browserless_agent";

package/lib/tools/dump_prompt.js

@@ -1,5 +1,5 @@
 import { GraphAILogger } from "graphai";
-import { readTemplatePrompt } from "../utils/file.js";
+import { readTemplatePrompt } from "../utils/templates.js";
 import clipboardy from "clipboardy";
 export const dumpPromptFromTemplate = async ({ templateName }) => {
     const prompt = readTemplatePrompt(templateName);

package/lib/tools/story_to_script.js

@@ -1,5 +1,6 @@
-import { getTemplateFilePath, readAndParseJson, readScriptTemplateFile, writingMessage } from "../utils/file.js";
-import { mulmoScriptSchema, mulmoScriptTemplateSchema } from "../types/schema.js";
+import { getPromptTemplateFilePath, readAndParseJson, writingMessage } from "../utils/file.js";
+import { readScriptTemplateFile } from "../utils/templates.js";
+import { mulmoScriptSchema, mulmoPromptTemplateSchema } from "../types/schema.js";
 import { GraphAI, GraphAILogger } from "graphai";
 import { openAIAgent } from "@graphai/openai_agent";
 import { anthropicAgent } from "@graphai/anthropic_agent";
@@ -228,7 +229,7 @@ const oneStepGraphData = {
 };
 const generateBeatsPrompt = async (template, beatsPerScene, story) => {
     const allScenes = story.scenes.map((scene) => scene.description).join("\n");
-    const sampleBeats = template.scriptName ? readScriptTemplateFile(template.scriptName).beats : [];
+    const sampleBeats = template.scriptName ? (readScriptTemplateFile(template.scriptName).beats ?? []) : [];
     return sceneToBeatsPrompt({ sampleBeats, beatsPerScene, allScenes });
 };
 const generateScriptInfoPrompt = async (template, story) => {
@@ -249,7 +250,7 @@ const generateScriptPrompt = async (template, beatsPerScene, story) => {
     return storyToScriptPrompt(script, beatsPerScene, story);
 };
 export const storyToScript = async ({ story, beatsPerScene, templateName, outdir, fileName, llm, llmModel, generateMode, }) => {
-    const template = readAndParseJson(getTemplateFilePath(templateName), mulmoScriptTemplateSchema);
+    const template = readAndParseJson(getPromptTemplateFilePath(templateName), mulmoPromptTemplateSchema);
     const { agent, model, max_tokens } = llmPair(llm, llmModel);
     const beatsPrompt = await generateBeatsPrompt(template, beatsPerScene, story);
     const scriptInfoPrompt = await generateScriptInfoPrompt(template, story);

package/lib/types/agent.d.ts

@@ -66,9 +66,19 @@ export type SoundEffectAgentInputs = AgentPromptInputs & {
     soundEffectFile: string;
     movieFile: string;
 };
+export type ReplicateLipSyncAgentParams = {
+    model: `${string}/${string}` | undefined;
+    duration?: number;
+};
+export type LipSyncAgentInputs = {
+    lipSyncFile: string;
+    movieFile: string;
+    audioFile: string;
+};
 export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
 export type ReplicateMovieAgentConfig = AgentConfig;
 export type ReplicateSoundEffectAgentConfig = AgentConfig;
+export type ReplicateLipSyncAgentConfig = AgentConfig;
 export type TTSAgentParams = {
     suppressError: boolean;
     voice: string;