mulmocast 0.1.7 → 1.1.0

package/lib/agents/sound_effect_replicate_agent.js

@@ -0,0 +1,59 @@
+import { readFileSync } from "fs";
+import { GraphAILogger } from "graphai";
+import Replicate from "replicate";
+import { provider2SoundEffectAgent } from "../utils/provider2agent.js";
+export const soundEffectReplicateAgent = async ({ namedInputs, params, config }) => {
+    const { prompt, movieFile } = namedInputs;
+    const apiKey = config?.apiKey;
+    const model = params.model ?? provider2SoundEffectAgent.replicate.defaultModel;
+    if (!apiKey) {
+        throw new Error("REPLICATE_API_TOKEN environment variable is required");
+    }
+    const replicate = new Replicate({
+        auth: apiKey,
+    });
+    const buffer = readFileSync(movieFile);
+    const uri = `data:video/quicktime;base64,${buffer.toString("base64")}`;
+    const input = {
+        video: uri,
+        prompt,
+        duration: params.duration,
+        // seed: -1,
+        // num_steps: 25,
+        // cfg_strength: 4.5,
+        // negative_prompt: "music"
+    };
+    try {
+        const model_identifier = provider2SoundEffectAgent.replicate.modelParams[model]?.identifier ?? model;
+        const output = await replicate.run(model_identifier, {
+            input,
+        });
+        if (output && typeof output === "object" && "url" in output) {
+            const videoUrl = output.url();
+            const videoResponse = await fetch(videoUrl);
+            if (!videoResponse.ok) {
+                throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`);
+            }
+            const arrayBuffer = await videoResponse.arrayBuffer();
+            return { buffer: Buffer.from(arrayBuffer) };
+        }
+        return undefined;
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate sound effect:", error.message);
+        throw error;
+    }
+};
+const soundEffectReplicateAgentInfo = {
+    name: "soundEffectReplicateAgent",
+    agent: soundEffectReplicateAgent,
+    mock: soundEffectReplicateAgent,
+    samples: [],
+    description: "Replicate Sound Effect agent (movie to movie)",
+    category: ["movie"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["REPLICATE_API_TOKEN"],
+};
+export default soundEffectReplicateAgentInfo;

package/lib/mcp/server.js

@@ -11,7 +11,7 @@ import { audio, images, movie, captions, pdf } from "../actions/index.js";
 import { initializeContext, runTranslateIfNeeded } from "../cli/helpers.js";
 import { outDirName } from "../utils/const.js";
 import { resolveDirPath, mkdir, generateTimestampedFileName } from "../utils/file.js";
-import { mulmoScriptSchema } from "../types/schema.js";
+import { MulmoScriptMethods } from "../methods/index.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 // Load MulmoScript JSON Schema from file
@@ -83,7 +83,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         }
         const { cmd, mulmoScript, options = {}, } = args;
         // Validate MulmoScript schema
-        const validatedScript = mulmoScriptSchema.parse(mulmoScript);
+        const validatedScript = MulmoScriptMethods.validate(mulmoScript);
         // Save MulmoScript to output directory
         const filePath = await saveMulmoScriptToOutput(validatedScript);
         // Create argv-like object for CLI compatibility

package/lib/methods/index.d.ts

@@ -3,3 +3,4 @@ export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
 export * from "./mulmo_beat.js";
+export * from "./mulmo_script.js";

package/lib/methods/index.js

@@ -3,3 +3,4 @@ export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
 export * from "./mulmo_beat.js";
+export * from "./mulmo_script.js";

package/lib/methods/mulmo_presentation_style.d.ts

@@ -1,15 +1,12 @@
 import "dotenv/config";
-import { MulmoCanvasDimension, MulmoBeat, SpeechOptions, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
-    getSpeechProvider(presentationStyle: MulmoPresentationStyle): Text2SpeechProvider;
     getAllSpeechProviders(presentationStyle: MulmoPresentationStyle): Set<Text2SpeechProvider>;
     getTextSlideStyle(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
-    getSpeechOptions(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeechOptions | undefined;
+    getDefaultSpeaker(presentationStyle: MulmoPresentationStyle): string;
     getSpeaker(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeakerData;
-    getTTSProvider(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): Text2SpeechProvider;
     getTTSModel(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string | undefined;
-    getVoiceId(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;
     getMovieAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): {
@@ -27,6 +24,14 @@ export declare const MulmoPresentationStyleMethods: {
             } | undefined;
         };
     };
+    getSoundEffectAgentInfo(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): {
+        agentName: string;
+        defaultModel: import("../utils/provider2agent.js").ReplicateModel;
+        models: import("../utils/provider2agent.js").ReplicateModel[];
+        modelParams: Record<import("../utils/provider2agent.js").ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+        }>;
+    };
     getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;
     getHtmlImageAgentInfo(presentationStyle: MulmoPresentationStyle): Text2HtmlAgentInfo;
     getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;

package/lib/methods/mulmo_presentation_style.js

@@ -1,7 +1,8 @@
 import "dotenv/config";
+import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, } from "../types/schema.js";
-import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent } from "../utils/provider2agent.js";
+import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2SoundEffectAgent, defaultProviders, } from "../utils/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -20,14 +21,10 @@ export const MulmoPresentationStyleMethods = {
     getCanvasSize(presentationStyle) {
         return mulmoCanvasDimensionSchema.parse(presentationStyle.canvasSize);
     },
-    getSpeechProvider(presentationStyle) {
-        return text2SpeechProviderSchema.parse(presentationStyle.speechParams?.provider);
-    },
     getAllSpeechProviders(presentationStyle) {
         const providers = new Set();
-        const defaultProvider = this.getSpeechProvider(presentationStyle);
         Object.values(presentationStyle.speechParams.speakers).forEach((speaker) => {
-            const provider = speaker.provider ?? defaultProvider;
+            const provider = text2SpeechProviderSchema.parse(speaker.provider);
             providers.add(provider);
         });
         return providers;
@@ -39,27 +36,27 @@ export const MulmoPresentationStyleMethods = {
         // This code allows us to support both string and array of strings for cssStyles
         return [...defaultTextSlideStyles, ...[styles], ...[extraStyles]].flat().join("\n");
     },
-    getSpeechOptions(presentationStyle, beat) {
-        return { ...presentationStyle.speechParams.speakers[beat.speaker].speechOptions, ...beat.speechOptions };
+    getDefaultSpeaker(presentationStyle) {
+        const speakers = presentationStyle.speechParams.speakers ?? {};
+        const keys = Object.keys(speakers).sort();
+        userAssert(keys.length !== 0, "presentationStyle.speechParams.speakers is not set!!");
+        const defaultSpeaker = keys.find((key) => speakers[key].isDefault);
+        if (!isNull(defaultSpeaker)) {
+            return defaultSpeaker;
+        }
+        return keys[0];
     },
     getSpeaker(presentationStyle, beat) {
         userAssert(!!presentationStyle?.speechParams?.speakers, "presentationStyle.speechParams.speakers is not set!!");
-        userAssert(!!beat?.speaker, "beat.speaker is not set");
-        const speaker = presentationStyle.speechParams.speakers[beat.speaker];
-        userAssert(!!speaker, `speaker is not set: speaker "${beat.speaker}"`);
+        const speakerId = beat?.speaker ?? MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle);
+        userAssert(!!speakerId, "beat.speaker and default speaker is not set");
+        const speaker = presentationStyle.speechParams.speakers[speakerId];
+        userAssert(!!speaker, `speaker is not set: speaker "${speakerId}"`);
         return speaker;
     },
-    getTTSProvider(presentationStyle, beat) {
-        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.provider ?? presentationStyle.speechParams.provider;
-    },
     getTTSModel(presentationStyle, beat) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.model ?? presentationStyle.speechParams.model;
-    },
-    getVoiceId(presentationStyle, beat) {
-        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
-        return speaker.voiceId;
+        return speaker.model;
     },
     getText2ImageProvider(provider) {
         return text2ImageProviderSchema.parse(provider);
@@ -89,6 +86,13 @@ export const MulmoPresentationStyleMethods = {
             movieParams,
         };
     },
+    getSoundEffectAgentInfo(presentationStyle, beat) {
+        const soundEffectProvider = (beat.soundEffectParams?.provider ??
+            presentationStyle.soundEffectParams?.provider ??
+            defaultProviders.soundEffect);
+        const agentInfo = provider2SoundEffectAgent[soundEffectProvider];
+        return agentInfo;
+    },
     getConcurrency(presentationStyle) {
         const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(presentationStyle);
         if (imageAgentInfo.imageParams.provider === "openai") {

package/lib/methods/mulmo_script.d.ts

@@ -0,0 +1,4 @@
+import { MulmoScript } from "../types/index.js";
+export declare const MulmoScriptMethods: {
+    validate(script: any): MulmoScript;
+};

package/lib/methods/mulmo_script.js

@@ -0,0 +1,31 @@
+import { mulmoScriptSchema } from "../types/index.js";
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const validate_1_0 = (script) => {
+    if (script.speechParams?.provider) {
+        if (typeof script.speechParams.speakers === "object") {
+            Object.keys(script.speechParams.speakers).forEach((speakerId) => {
+                const speaker = script.speechParams.speakers[speakerId];
+                if (!speaker.provider) {
+                    speaker.provider = script.speechParams.provider;
+                }
+            });
+        }
+        delete script.speechParams.provider;
+    }
+    return script;
+};
+const validators = [{ from: "1.0", to: "1.1", validator: validate_1_0 }];
+export const MulmoScriptMethods = {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    validate(script) {
+        const validatedScript = validators.reduce((acc, validator) => {
+            if (acc.$mulmocast.version === validator.from) {
+                const validated = validator.validator(acc);
+                validated.$mulmocast.version = validator.to;
+                return validated;
+            }
+            return acc;
+        }, script);
+        return mulmoScriptSchema.parse(validatedScript);
+    },
+};

package/lib/types/agent.d.ts

@@ -58,8 +58,17 @@ export type ReplicateMovieAgentParams = {
     };
     duration?: number;
 };
+export type ReplicateSoundEffectAgentParams = {
+    model: `${string}/${string}` | undefined;
+    duration?: number;
+};
+export type SoundEffectAgentInputs = AgentPromptInputs & {
+    soundEffectFile: string;
+    movieFile: string;
+};
 export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
 export type ReplicateMovieAgentConfig = AgentConfig;
+export type ReplicateSoundEffectAgentConfig = AgentConfig;
 export type TTSAgentParams = {
     suppressError: boolean;
     voice: string;