mulmocast 0.1.7 → 1.1.1

package/lib/methods/mulmo_presentation_style.js

@@ -1,7 +1,8 @@
 import "dotenv/config";
+import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, } from "../types/schema.js";
-import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent } from "../utils/provider2agent.js";
+import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, } from "../utils/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -20,14 +21,10 @@ export const MulmoPresentationStyleMethods = {
     getCanvasSize(presentationStyle) {
         return mulmoCanvasDimensionSchema.parse(presentationStyle.canvasSize);
     },
-    getSpeechProvider(presentationStyle) {
-        return text2SpeechProviderSchema.parse(presentationStyle.speechParams?.provider);
-    },
     getAllSpeechProviders(presentationStyle) {
         const providers = new Set();
-        const defaultProvider = this.getSpeechProvider(presentationStyle);
         Object.values(presentationStyle.speechParams.speakers).forEach((speaker) => {
-            const provider = speaker.provider ?? defaultProvider;
+            const provider = text2SpeechProviderSchema.parse(speaker.provider);
             providers.add(provider);
         });
         return providers;
@@ -39,27 +36,27 @@ export const MulmoPresentationStyleMethods = {
         // This code allows us to support both string and array of strings for cssStyles
         return [...defaultTextSlideStyles, ...[styles], ...[extraStyles]].flat().join("\n");
     },
-    getSpeechOptions(presentationStyle, beat) {
-        return { ...presentationStyle.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };
+    getDefaultSpeaker(presentationStyle) {
+        const speakers = presentationStyle.speechParams.speakers ?? {};
+        const keys = Object.keys(speakers).sort();
+        userAssert(keys.length !== 0, "presentationStyle.speechParams.speakers is not set!!");
+        const defaultSpeaker = keys.find((key) => speakers[key].isDefault);
+        if (!isNull(defaultSpeaker)) {
+            return defaultSpeaker;
+        }
+        return keys[0];
     },
     getSpeaker(presentationStyle, beat) {
         userAssert(!!presentationStyle?.speechParams?.speakers, "presentationStyle.speechParams.speakers is not set!!");
-        userAssert(!!beat?.speaker, "beat.speaker is not set");
-        const speaker = presentationStyle.speechParams.speakers[beat.speaker];
-        userAssert(!!speaker, `speaker is not set: speaker "${beat.speaker}"`);
+        const speakerId = beat?.speaker ?? MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle);
+        userAssert(!!speakerId, "beat.speaker and default speaker is not set");
+        const speaker = presentationStyle.speechParams.speakers[speakerId];
+        userAssert(!!speaker, `speaker is not set: speaker "${speakerId}"`);
         return speaker;
     },
-    getTTSProvider(presentationStyle, beat) {
-        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.provider ?? presentationStyle.speechParams.provider;
-    },
     getTTSModel(presentationStyle, beat) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.model ?? presentationStyle.speechParams.model;
-    },
-    getVoiceId(presentationStyle, beat) {
-        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.voiceId;
+        return speaker.model;
     },
     getText2ImageProvider(provider) {
         return text2ImageProviderSchema.parse(provider);
@@ -89,6 +86,20 @@ export const MulmoPresentationStyleMethods = {
             movieParams,
         };
     },
+    getSoundEffectAgentInfo(presentationStyle, beat) {
+        const soundEffectProvider = (beat.soundEffectParams?.provider ??
+            presentationStyle.soundEffectParams?.provider ??
+            defaultProviders.soundEffect);
+        const agentInfo = provider2SoundEffectAgent[soundEffectProvider];
+        return agentInfo;
+    },
+    getLipSyncAgentInfo(presentationStyle, beat) {
+        const lipSyncProvider = (beat.lipSyncParams?.provider ??
+            presentationStyle.lipSyncParams?.provider ??
+            defaultProviders.lipSync);
+        const agentInfo = provider2LipSyncAgent[lipSyncProvider];
+        return agentInfo;
+    },
     getConcurrency(presentationStyle) {
         const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(presentationStyle);
         if (imageAgentInfo.imageParams.provider === "openai") {

package/lib/methods/mulmo_script.d.ts

@@ -0,0 +1,4 @@
+import { MulmoScript } from "../types/index.js";
+export declare const MulmoScriptMethods: {
+    validate(script: any): MulmoScript;
+};

package/lib/methods/mulmo_script.js

@@ -0,0 +1,31 @@
+import { mulmoScriptSchema } from "../types/index.js";
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const validate_1_0 = (script) => {
+    if (script.speechParams?.provider) {
+        if (typeof script.speechParams.speakers === "object") {
+            Object.keys(script.speechParams.speakers).forEach((speakerId) => {
+                const speaker = script.speechParams.speakers[speakerId];
+                if (!speaker.provider) {
+                    speaker.provider = script.speechParams.provider;
+                }
+            });
+        }
+        delete script.speechParams.provider;
+    }
+    return script;
+};
+const validators = [{ from: "1.0", to: "1.1", validator: validate_1_0 }];
+export const MulmoScriptMethods = {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    validate(script) {
+        const validatedScript = validators.reduce((acc, validator) => {
+            if (acc.$mulmocast.version === validator.from) {
+                const validated = validator.validator(acc);
+                validated.$mulmocast.version = validator.to;
+                return validated;
+            }
+            return acc;
+        }, script);
+        return mulmoScriptSchema.parse(validatedScript);
+    },
+};

package/lib/tools/story_to_script.js

@@ -1,4 +1,4 @@
-import { getTemplateFilePath, readAndParseJson, readScriptTemplateFile, writingMessage } from "../utils/file.js";
+import { getPromptTemplateFilePath, readAndParseJson, readScriptTemplateFile, writingMessage } from "../utils/file.js";
 import { mulmoScriptSchema, mulmoScriptTemplateSchema } from "../types/schema.js";
 import { GraphAI, GraphAILogger } from "graphai";
 import { openAIAgent } from "@graphai/openai_agent";
@@ -249,7 +249,7 @@ const generateScriptPrompt = async (template, beatsPerScene, story) => {
     return storyToScriptPrompt(script, beatsPerScene, story);
 };
 export const storyToScript = async ({ story, beatsPerScene, templateName, outdir, fileName, llm, llmModel, generateMode, }) => {
-    const template = readAndParseJson(getTemplateFilePath(templateName), mulmoScriptTemplateSchema);
+    const template = readAndParseJson(getPromptTemplateFilePath(templateName), mulmoScriptTemplateSchema);
     const { agent, model, max_tokens } = llmPair(llm, llmModel);
     const beatsPrompt = await generateBeatsPrompt(template, beatsPerScene, story);
     const scriptInfoPrompt = await generateScriptInfoPrompt(template, story);

package/lib/types/agent.d.ts

@@ -58,8 +58,27 @@ export type ReplicateMovieAgentParams = {
     };
     duration?: number;
 };
+export type ReplicateSoundEffectAgentParams = {
+    model: `${string}/${string}` | undefined;
+    duration?: number;
+};
+export type SoundEffectAgentInputs = AgentPromptInputs & {
+    soundEffectFile: string;
+    movieFile: string;
+};
+export type ReplicateLipSyncAgentParams = {
+    model: `${string}/${string}` | undefined;
+    duration?: number;
+};
+export type LipSyncAgentInputs = {
+    lipSyncFile: string;
+    movieFile: string;
+    audioFile: string;
+};
 export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
 export type ReplicateMovieAgentConfig = AgentConfig;
+export type ReplicateSoundEffectAgentConfig = AgentConfig;
+export type ReplicateLipSyncAgentConfig = AgentConfig;
 export type TTSAgentParams = {
     suppressError: boolean;
     voice: string;