mulmocast 1.1.0 → 1.1.1

package/lib/data/scriptTemplates.js

@@ -0,0 +1,580 @@
+export const scriptTemplates = [
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "business",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "children_book",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "coding",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "html",
+        htmlImageParams: {
+            model: "claude-3-7-sonnet-20250219",
+            provider: "anthropic",
+        },
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "image_prompt_only_template",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "image_prompts_template",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "image_refs",
+        imageParams: {
+            images: {
+                "[CHARACTER_ID_1]": {
+                    prompt: "[IMAGE PROMPT FOR THIS CHARACTER]",
+                    type: "imagePrompt",
+                },
+                "[CHARACTER_ID_2]": {
+                    prompt: "[IMAGE PROMPT FOR THIS CHARACTER]",
+                    type: "imagePrompt",
+                },
+            },
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "movie_prompts_no_text_template",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "google",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "movie_prompts_template",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "google",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "presentation",
+        htmlImageParams: {
+            model: "claude-3-7-sonnet-20250219",
+            provider: "anthropic",
+        },
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "sensei_and_taro",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "shorts_template",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "google",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            credit: "closing",
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 720,
+            width: 1280,
+        },
+        filename: "text_only_template",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+    {
+        $mulmocast: {
+            version: "1.1",
+        },
+        audioParams: {
+            audioVolume: 1,
+            bgmVolume: 0.2,
+            closingPadding: 0.8,
+            introPadding: 1,
+            outroPadding: 1,
+            padding: 0.3,
+            suppressSpeech: false,
+        },
+        canvasSize: {
+            height: 2064,
+            width: 1552,
+        },
+        captionParams: {
+            lang: "en",
+            styles: [],
+        },
+        filename: "voice_over",
+        imageParams: {
+            images: {},
+            provider: "openai",
+        },
+        movieParams: {
+            provider: "replicate",
+        },
+        soundEffectParams: {
+            provider: "replicate",
+        },
+        speechParams: {
+            speakers: {
+                Presenter: {
+                    displayName: {
+                        en: "Presenter",
+                    },
+                    voiceId: "shimmer",
+                },
+            },
+        },
+    },
+];

package/lib/index.browser.d.ts

@@ -1,2 +1,3 @@
 export * from "./index.common.js";
-export * from "./agents/validate_schema_agent.js";
+import validateSchemaAgent from "./agents/validate_schema_agent.js";
+export { validateSchemaAgent };

package/lib/index.browser.js

@@ -1,3 +1,4 @@
 // Entry point for browser code
 export * from "./index.common.js";
-export * from "./agents/validate_schema_agent.js";
+import validateSchemaAgent from "./agents/validate_schema_agent.js";
+export { validateSchemaAgent };

package/lib/methods/mulmo_presentation_style.d.ts

@@ -32,6 +32,14 @@ export declare const MulmoPresentationStyleMethods: {
             identifier?: `${string}/${string}:${string}`;
         }>;
     };
+    getLipSyncAgentInfo(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): {
+        agentName: string;
+        defaultModel: import("../utils/provider2agent.js").ReplicateModel;
+        models: import("../utils/provider2agent.js").ReplicateModel[];
+        modelParams: Record<import("../utils/provider2agent.js").ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+        }>;
+    };
     getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;
     getHtmlImageAgentInfo(presentationStyle: MulmoPresentationStyle): Text2HtmlAgentInfo;
     getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;

package/lib/methods/mulmo_presentation_style.js

@@ -2,7 +2,7 @@ import "dotenv/config";
 import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, } from "../types/schema.js";
-import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2SoundEffectAgent, defaultProviders, } from "../utils/provider2agent.js";
+import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, } from "../utils/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -93,6 +93,13 @@ export const MulmoPresentationStyleMethods = {
         const agentInfo = provider2SoundEffectAgent[soundEffectProvider];
         return agentInfo;
     },
+    getLipSyncAgentInfo(presentationStyle, beat) {
+        const lipSyncProvider = (beat.lipSyncParams?.provider ??
+            presentationStyle.lipSyncParams?.provider ??
+            defaultProviders.lipSync);
+        const agentInfo = provider2LipSyncAgent[lipSyncProvider];
+        return agentInfo;
+    },
     getConcurrency(presentationStyle) {
         const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(presentationStyle);
         if (imageAgentInfo.imageParams.provider === "openai") {

package/lib/tools/story_to_script.js

@@ -1,4 +1,4 @@
-import { getTemplateFilePath, readAndParseJson, readScriptTemplateFile, writingMessage } from "../utils/file.js";
+import { getPromptTemplateFilePath, readAndParseJson, readScriptTemplateFile, writingMessage } from "../utils/file.js";
 import { mulmoScriptSchema, mulmoScriptTemplateSchema } from "../types/schema.js";
 import { GraphAI, GraphAILogger } from "graphai";
 import { openAIAgent } from "@graphai/openai_agent";
@@ -249,7 +249,7 @@ const generateScriptPrompt = async (template, beatsPerScene, story) => {
     return storyToScriptPrompt(script, beatsPerScene, story);
 };
 export const storyToScript = async ({ story, beatsPerScene, templateName, outdir, fileName, llm, llmModel, generateMode, }) => {
-    const template = readAndParseJson(getTemplateFilePath(templateName), mulmoScriptTemplateSchema);
+    const template = readAndParseJson(getPromptTemplateFilePath(templateName), mulmoScriptTemplateSchema);
     const { agent, model, max_tokens } = llmPair(llm, llmModel);
     const beatsPrompt = await generateBeatsPrompt(template, beatsPerScene, story);
     const scriptInfoPrompt = await generateScriptInfoPrompt(template, story);

package/lib/types/agent.d.ts

@@ -66,9 +66,19 @@ export type SoundEffectAgentInputs = AgentPromptInputs & {
     soundEffectFile: string;
     movieFile: string;
 };
+export type ReplicateLipSyncAgentParams = {
+    model: `${string}/${string}` | undefined;
+    duration?: number;
+};
+export type LipSyncAgentInputs = {
+    lipSyncFile: string;
+    movieFile: string;
+    audioFile: string;
+};
 export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
 export type ReplicateMovieAgentConfig = AgentConfig;
 export type ReplicateSoundEffectAgentConfig = AgentConfig;
+export type ReplicateLipSyncAgentConfig = AgentConfig;
 export type TTSAgentParams = {
     suppressError: boolean;
     voice: string;