mulmocast 0.1.7 → 1.1.1

package/lib/agents/lipsync_replicate_agent.js

@@ -0,0 +1,57 @@
+import { readFileSync } from "fs";
+import { GraphAILogger } from "graphai";
+import Replicate from "replicate";
+import { provider2LipSyncAgent } from "../utils/provider2agent.js";
+export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) => {
+    const { movieFile, audioFile } = namedInputs;
+    const apiKey = config?.apiKey;
+    const model = params.model ?? provider2LipSyncAgent.replicate.defaultModel;
+    if (!apiKey) {
+        throw new Error("REPLICATE_API_TOKEN environment variable is required");
+    }
+    const replicate = new Replicate({
+        auth: apiKey,
+    });
+    const videoBuffer = readFileSync(movieFile);
+    const audioBuffer = readFileSync(audioFile);
+    const videoUri = `data:video/quicktime;base64,${videoBuffer.toString("base64")}`;
+    const audioUri = `data:audio/wav;base64,${audioBuffer.toString("base64")}`;
+    const input = {
+        video: videoUri,
+        audio: audioUri,
+        duration: params.duration,
+    };
+    try {
+        const model_identifier = provider2LipSyncAgent.replicate.modelParams[model]?.identifier ?? model;
+        const output = await replicate.run(model_identifier, {
+            input,
+        });
+        if (output && typeof output === "object" && "url" in output) {
+            const videoUrl = output.url();
+            const videoResponse = await fetch(videoUrl);
+            if (!videoResponse.ok) {
+                throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`);
+            }
+            const arrayBuffer = await videoResponse.arrayBuffer();
+            return { buffer: Buffer.from(arrayBuffer) };
+        }
+        return undefined;
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate lip sync:", error.message);
+        throw error;
+    }
+};
+const lipSyncReplicateAgentInfo = {
+    name: "lipSyncReplicateAgent",
+    agent: lipSyncReplicateAgent,
+    mock: lipSyncReplicateAgent,
+    samples: [],
+    description: "Replicate Lip Sync agent (video + audio to video)",
+    category: ["movie"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["REPLICATE_API_TOKEN"],
+};
+export default lipSyncReplicateAgentInfo;

package/lib/agents/movie_replicate_agent.js

@@ -11,6 +11,7 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
         duration,
         image: undefined,
         start_image: undefined,
+        first_frame_image: undefined,
         aspect_ratio: aspectRatio, // only for bytedance/seedance-1-lite
         // resolution: "720p", // only for bytedance/seedance-1-lite
         // fps: 24, // only for bytedance/seedance-1-lite
@@ -22,15 +23,19 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
     if (imagePath) {
         const buffer = readFileSync(imagePath);
         const base64Image = `data:image/png;base64,${buffer.toString("base64")}`;
-        if (model === "kwaivgi/kling-v2.1" || model === "kwaivgi/kling-v1.6-pro" || model === "minimax/hailuo-02") {
-            input.start_image = base64Image;
+        const start_image = provider2MovieAgent.replicate.modelParams[model]?.start_image;
+        if (start_image === "first_frame_image" || start_image === "image" || start_image === "start_image") {
+            input[start_image] = base64Image;
+        }
+        else if (start_image === undefined) {
+            throw new Error(`Model ${model} does not support image-to-video generation`);
         }
         else {
             input.image = base64Image;
         }
     }
     try {
-        const output = await replicate.run(model ?? provider2MovieAgent.replicate.defaultModel, { input });
+        const output = await replicate.run(model, { input });
         // Download the generated video
         if (output && typeof output === "object" && "url" in output) {
             const videoUrl = output.url();
@@ -62,13 +67,20 @@ export const getAspectRatio = (canvasSize) => {
 export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, imagePath } = namedInputs;
     const aspectRatio = getAspectRatio(params.canvasSize);
-    const duration = params.duration ?? 5;
+    const model = params.model ?? provider2MovieAgent.replicate.defaultModel;
+    if (!provider2MovieAgent.replicate.modelParams[model]) {
+        throw new Error(`Model ${model} is not supported`);
+    }
+    const duration = params.duration ?? provider2MovieAgent.replicate.modelParams[model].durations[0] ?? 5;
+    if (!provider2MovieAgent.replicate.modelParams[model].durations.includes(duration)) {
+        throw new Error(`Duration ${duration} is not supported for model ${model}. Supported durations: ${provider2MovieAgent.replicate.modelParams[model].durations.join(", ")}`);
+    }
     const apiKey = config?.apiKey;
     if (!apiKey) {
         throw new Error("REPLICATE_API_TOKEN environment variable is required");
     }
     try {
-        const buffer = await generateMovie(params.model, apiKey, prompt, imagePath, aspectRatio, duration);
+        const buffer = await generateMovie(model, apiKey, prompt, imagePath, aspectRatio, duration);
         if (buffer) {
             return { buffer };
         }

package/lib/agents/sound_effect_replicate_agent.d.ts

@@ -0,0 +1,5 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { AgentBufferResult, SoundEffectAgentInputs, ReplicateSoundEffectAgentParams, ReplicateSoundEffectAgentConfig } from "../types/agent.js";
+export declare const soundEffectReplicateAgent: AgentFunction<ReplicateSoundEffectAgentParams, AgentBufferResult, SoundEffectAgentInputs, ReplicateSoundEffectAgentConfig>;
+declare const soundEffectReplicateAgentInfo: AgentFunctionInfo;
+export default soundEffectReplicateAgentInfo;

package/lib/agents/sound_effect_replicate_agent.js

@@ -0,0 +1,59 @@
+import { readFileSync } from "fs";
+import { GraphAILogger } from "graphai";
+import Replicate from "replicate";
+import { provider2SoundEffectAgent } from "../utils/provider2agent.js";
+export const soundEffectReplicateAgent = async ({ namedInputs, params, config }) => {
+    const { prompt, movieFile } = namedInputs;
+    const apiKey = config?.apiKey;
+    const model = params.model ?? provider2SoundEffectAgent.replicate.defaultModel;
+    if (!apiKey) {
+        throw new Error("REPLICATE_API_TOKEN environment variable is required");
+    }
+    const replicate = new Replicate({
+        auth: apiKey,
+    });
+    const buffer = readFileSync(movieFile);
+    const uri = `data:video/quicktime;base64,${buffer.toString("base64")}`;
+    const input = {
+        video: uri,
+        prompt,
+        duration: params.duration,
+        // seed: -1,
+        // num_steps: 25,
+        // cfg_strength: 4.5,
+        // negative_prompt: "music"
+    };
+    try {
+        const model_identifier = provider2SoundEffectAgent.replicate.modelParams[model]?.identifier ?? model;
+        const output = await replicate.run(model_identifier, {
+            input,
+        });
+        if (output && typeof output === "object" && "url" in output) {
+            const videoUrl = output.url();
+            const videoResponse = await fetch(videoUrl);
+            if (!videoResponse.ok) {
+                throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`);
+            }
+            const arrayBuffer = await videoResponse.arrayBuffer();
+            return { buffer: Buffer.from(arrayBuffer) };
+        }
+        return undefined;
+    }
+    catch (error) {
+        GraphAILogger.info("Failed to generate sound effect:", error.message);
+        throw error;
+    }
+};
+const soundEffectReplicateAgentInfo = {
+    name: "soundEffectReplicateAgent",
+    agent: soundEffectReplicateAgent,
+    mock: soundEffectReplicateAgent,
+    samples: [],
+    description: "Replicate Sound Effect agent (movie to movie)",
+    category: ["movie"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["REPLICATE_API_TOKEN"],
+};
+export default soundEffectReplicateAgentInfo;

package/lib/data/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./promptTemplates.js";
+export * from "./scriptTemplates.js";

package/lib/data/index.js

@@ -0,0 +1,2 @@
+export * from "./promptTemplates.js";
+export * from "./scriptTemplates.js";