mulmocast 0.1.2 → 0.1.4

package/lib/agents/movie_google_agent.js

@@ -5,13 +5,13 @@ async function generateMovie(projectId, model, token, prompt, imagePath, aspectR
     const payload = {
         instances: [
             {
-                prompt: prompt,
+                prompt,
                 image: undefined,
             },
         ],
         parameters: {
             sampleCount: 1,
-            aspectRatio: aspectRatio,
+            aspectRatio,
             safetySetting: "block_only_high",
             personGeneration: "allow_all",
             durationSeconds: duration,
@@ -46,7 +46,7 @@ async function generateMovie(projectId, model, token, prompt, imagePath, aspectR
         while (true) {
             GraphAILogger.info("...waiting for movie generation...");
             await sleep(3000);
-            const response = await fetch(`${GOOGLE_IMAGEN_ENDPOINT}:fetchPredictOperation`, {
+            const operationResponse = await fetch(`${GOOGLE_IMAGEN_ENDPOINT}:fetchPredictOperation`, {
                 method: "POST",
                 headers: {
                     Authorization: `Bearer ${token}`,
@@ -54,10 +54,10 @@ async function generateMovie(projectId, model, token, prompt, imagePath, aspectR
                 },
                 body: JSON.stringify(fetchBody),
             });
-            if (!response.ok) {
-                throw new Error(`Error: ${response.status} - ${response.statusText}`);
+            if (!operationResponse.ok) {
+                throw new Error(`Error: ${operationResponse.status} - ${operationResponse.statusText}`);
             }
-            const responseData = await response.json();
+            const responseData = await operationResponse.json();
             if (responseData.done) {
                 if (responseData.error) {
                     GraphAILogger.info("Prompt: ", prompt);
@@ -87,7 +87,7 @@ export const getAspectRatio = (canvasSize) => {
         return "1:1";
     }
 };
-export const movieGoogleAgent = async ({ namedInputs, params, config }) => {
+export const movieGoogleAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, imagePath } = namedInputs;
     const aspectRatio = getAspectRatio(params.canvasSize);
     const model = params.model ?? "veo-2.0-generate-001"; // "veo-3.0-generate-preview";

package/lib/agents/movie_replicate_agent.d.ts

@@ -1,23 +1,9 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import type { AgentBufferResult, MovieAgentInputs, ReplicateMovieAgentParams, ReplicateMovieAgentConfig } from "../types/agent.js";
 export declare const getAspectRatio: (canvasSize: {
     width: number;
     height: number;
 }) => string;
-export type MovieReplicateConfig = {
-    apiKey?: string;
-};
-export declare const movieReplicateAgent: AgentFunction<{
-    model: `${string}/${string}` | undefined;
-    canvasSize: {
-        width: number;
-        height: number;
-    };
-    duration?: number;
-}, {
-    buffer: Buffer;
-}, {
-    prompt: string;
-    imagePath?: string;
-}, MovieReplicateConfig>;
+export declare const movieReplicateAgent: AgentFunction<ReplicateMovieAgentParams, AgentBufferResult, MovieAgentInputs, ReplicateMovieAgentConfig>;
 declare const movieReplicateAgentInfo: AgentFunctionInfo;
 export default movieReplicateAgentInfo;

package/lib/agents/movie_replicate_agent.js

@@ -6,8 +6,8 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
         auth: apiKey,
     });
     const input = {
-        prompt: prompt,
-        duration: duration,
+        prompt,
+        duration,
         image: undefined,
         start_image: undefined,
         aspect_ratio: aspectRatio, // only for bytedance/seedance-1-lite
@@ -21,7 +21,7 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
     if (imagePath) {
         const buffer = readFileSync(imagePath);
         const base64Image = `data:image/png;base64,${buffer.toString("base64")}`;
-        if (model === "kwaivgi/kling-v2.1") {
+        if (model === "kwaivgi/kling-v2.1" || model === "kwaivgi/kling-v1.6-pro") {
             input.start_image = base64Image;
         }
         else {
@@ -58,7 +58,7 @@ export const getAspectRatio = (canvasSize) => {
         return "1:1";
     }
 };
-export const movieReplicateAgent = async ({ namedInputs, params, config }) => {
+export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, imagePath } = namedInputs;
     const aspectRatio = getAspectRatio(params.canvasSize);
     const duration = params.duration ?? 5;

package/lib/agents/tts_google_agent.d.ts

@@ -1,4 +1,12 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
-export declare const ttsGoogleAgent: AgentFunction;
+export declare const ttsGoogleAgent: AgentFunction<{
+    voice: string;
+    speed: number;
+    suppressError: boolean;
+}, {
+    buffer?: Buffer | null;
+}, {
+    text: string;
+}>;
 declare const ttsGoogleAgentInfo: AgentFunctionInfo;
 export default ttsGoogleAgentInfo;

package/lib/agents/tts_google_agent.js

@@ -14,7 +14,7 @@ export const ttsGoogleAgent = async ({ namedInputs, params }) => {
     }
     // Construct the request
     const request = {
-        input: { text: text },
+        input: { text },
         voice: voiceParams,
         audioConfig: {
             audioEncoding: "MP3",
@@ -46,6 +46,6 @@ const ttsGoogleAgentInfo = {
     author: "Receptron Team",
     repository: "https://github.com/receptron/mulmocast-cli/",
     license: "MIT",
-    environmentVariables: ["OPENAI_API_KEY"],
+    environmentVariables: ["GOOGLE_GENAI_API_KEY"],
 };
 export default ttsGoogleAgentInfo;

package/lib/agents/tts_nijivoice_agent.js

@@ -21,7 +21,7 @@ export const ttsNijivoiceAgent = async ({ params, namedInputs, config }) => {
         },
         body: JSON.stringify({
             format: "mp3",
-            speed: speed ? "" + speed : speed_global ? "" + speed_global : "1.0",
+            speed: String(speed ?? speed_global ?? "1.0"),
             script: text,
         }),
     };

package/lib/agents/tts_openai_agent.d.ts

@@ -1,4 +1,16 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
-export declare const ttsOpenaiAgent: AgentFunction;
+export declare const ttsOpenaiAgent: AgentFunction<{
+    model: string;
+    voice: string;
+    instructions: string;
+    suppressError: boolean;
+}, {
+    buffer?: Buffer;
+}, {
+    text: string;
+}, {
+    baseURL?: string;
+    apiKey?: string;
+}>;
 declare const ttsOpenaiAgentInfo: AgentFunctionInfo;
 export default ttsOpenaiAgentInfo;

package/lib/agents/tts_openai_agent.js

@@ -3,8 +3,8 @@ import OpenAI from "openai";
 export const ttsOpenaiAgent = async ({ namedInputs, params, config }) => {
     const { text } = namedInputs;
     const { model, voice, suppressError, instructions } = params;
-    const { apiKey } = config ?? {};
-    const openai = new OpenAI({ apiKey });
+    const { apiKey, baseURL } = config ?? {};
+    const openai = new OpenAI({ apiKey, baseURL });
     try {
         const tts_options = {
             model: model ?? "gpt-4o-mini-tts", // "tts-1",

package/lib/cli/helpers.js

@@ -34,16 +34,16 @@ export const getFileObject = (args) => {
     const { fileOrUrl, fileName } = (() => {
         if (file === "__clipboard") {
             // We generate a new unique script file from clipboard text in the output directory
-            const fileName = generateTimestampedFileName("script");
+            const generatedFileName = generateTimestampedFileName("script");
             const clipboardText = clipboardy.readSync();
-            const fileOrUrl = resolveDirPath(outDirPath, `${fileName}.json`);
+            const resolvedFilePath = resolveDirPath(outDirPath, `${generatedFileName}.json`);
             mkdir(outDirPath);
-            fs.writeFileSync(fileOrUrl, clipboardText, "utf8");
-            return { fileOrUrl, fileName };
+            fs.writeFileSync(resolvedFilePath, clipboardText, "utf8");
+            return { fileOrUrl: resolvedFilePath, fileName: generatedFileName };
         }
-        const fileOrUrl = file ?? "";
-        const fileName = path.parse(fileOrUrl).name;
-        return { fileOrUrl, fileName };
+        const resolvedFileOrUrl = file ?? "";
+        const parsedFileName = path.parse(resolvedFileOrUrl).name;
+        return { fileOrUrl: resolvedFileOrUrl, fileName: parsedFileName };
     })();
     const isHttpPath = isHttp(fileOrUrl);
     const mulmoFilePath = isHttpPath ? "" : getFullPath(baseDirPath, fileOrUrl);

package/lib/index.d.ts

@@ -2,6 +2,7 @@ export * from "./actions/index.js";
 export * from "./cli/helpers.js";
 export * from "./utils/file.js";
 export * from "./utils/ffmpeg_utils.js";
+export * from "./utils/provider2agent.js";
 export * from "./methods/index.js";
 export * from "./agents/index.js";
 export * from "./types/index.js";

package/lib/index.js

@@ -2,6 +2,7 @@ export * from "./actions/index.js";
 export * from "./cli/helpers.js";
 export * from "./utils/file.js";
 export * from "./utils/ffmpeg_utils.js";
+export * from "./utils/provider2agent.js";
 export * from "./methods/index.js";
 export * from "./agents/index.js";
 export * from "./types/index.js";

package/lib/methods/index.d.ts

@@ -2,3 +2,4 @@ export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_beat.js";

package/lib/methods/index.js

@@ -2,3 +2,4 @@ export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_beat.js";

package/lib/methods/mulmo_beat.d.ts

@@ -0,0 +1,6 @@
+import { MulmoBeat } from "../types/index.js";
+export declare const MulmoBeatMethods: {
+    getHtmlPrompt(beat: MulmoBeat): string | undefined;
+    getPlugin(beat: MulmoBeat): typeof import("../utils/image_plugins/text_slide.js") | typeof import("../utils/image_plugins/markdown.js") | typeof import("../utils/image_plugins/chart.js") | typeof import("../utils/image_plugins/mermaid.js") | typeof import("../utils/image_plugins/html_tailwind.js") | typeof import("../utils/image_plugins/image.js") | typeof import("../utils/image_plugins/movie.js") | typeof import("../utils/image_plugins/beat.js") | typeof import("../utils/image_plugins/voice_over.js");
+    getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
+};

package/lib/methods/mulmo_beat.js

@@ -0,0 +1,21 @@
+import { findImagePlugin } from "../utils/image_plugins/index.js";
+export const MulmoBeatMethods = {
+    getHtmlPrompt(beat) {
+        if (beat?.htmlPrompt?.data) {
+            return beat.htmlPrompt.prompt + "\n\n data\n" + JSON.stringify(beat.htmlPrompt.data, null, 2);
+        }
+        return beat?.htmlPrompt?.prompt;
+    },
+    getPlugin(beat) {
+        const plugin = findImagePlugin(beat?.image?.type);
+        if (!plugin) {
+            throw new Error(`invalid beat image type: ${beat.image}`);
+        }
+        return plugin;
+    },
+    getImageReferenceForImageGenerator(beat, imageRefs) {
+        const imageNames = beat.imageNames ?? Object.keys(imageRefs); // use all images if imageNames is not specified
+        const sources = imageNames.map((name) => imageRefs[name]);
+        return sources.filter((source) => source !== undefined);
+    },
+};

package/lib/methods/mulmo_presentation_style.d.ts

@@ -7,10 +7,12 @@ export declare const MulmoPresentationStyleMethods: {
     getTextSlideStyle(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getSpeechOptions(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeechOptions | undefined;
     getSpeaker(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): SpeakerData;
-    getProvider(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): Text2SpeechProvider;
+    getTTSProvider(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): Text2SpeechProvider;
     getVoiceId(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;
+    getMovieAgent(presentationStyle: MulmoPresentationStyle): string;
+    getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;
     getHtmlImageAgentInfo(presentationStyle: MulmoPresentationStyle): Text2HtmlAgentInfo;
     getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;
 };

package/lib/methods/mulmo_presentation_style.js

@@ -1,7 +1,7 @@
 import "dotenv/config";
-import { userAssert, llmConfig } from "../utils/utils.js";
+import { userAssert } from "../utils/utils.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
-import { defaultOpenAIImageModel } from "../utils/const.js";
+import { defaultProviders, provider2ImageAgent, provider2MovieAgent, provider2LLMAgent } from "../utils/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -49,7 +49,7 @@ export const MulmoPresentationStyleMethods = {
         userAssert(!!speaker, `speaker is not set: speaker "${beat.speaker}"`);
         return speaker;
     },
-    getProvider(presentationStyle, beat) {
+    getTTSProvider(presentationStyle, beat) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
         return speaker.provider ?? presentationStyle.speechParams.provider;
     },
@@ -65,22 +65,46 @@ export const MulmoPresentationStyleMethods = {
         // provider and model appropriately.
         const imageParams = { ...presentationStyle.imageParams, ...beat?.imageParams };
         const provider = MulmoPresentationStyleMethods.getText2ImageProvider(imageParams?.provider);
+        const agentInfo = provider2ImageAgent[provider];
+        // The default text2image model is gpt-image-1 from OpenAI, and to use it you must have an OpenAI account and have verified your identity. If this is not possible, please specify dall-e-3 as the model.
         const defaultImageParams = {
             provider,
-            model: provider === "openai" ? (process.env.DEFAULT_OPENAI_IMAGE_MODEL ?? defaultOpenAIImageModel) : undefined,
+            model: agentInfo.defaultModel,
         };
         return {
-            agent: provider === "google" ? "imageGoogleAgent" : "imageOpenaiAgent",
+            agent: agentInfo.agentName,
             imageParams: { ...defaultImageParams, ...imageParams },
         };
     },
+    // Determine movie agent based on provider
+    getMovieAgent(presentationStyle) {
+        const movieProvider = (presentationStyle.movieParams?.provider ?? defaultProviders.text2movie);
+        return provider2MovieAgent[movieProvider].agentName;
+    },
+    getConcurrency(presentationStyle) {
+        /*
+        if (presentationStyle.movieParams?.provider === "replicate") {
+          return 4;
+        }
+        */
+        const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(presentationStyle);
+        if (imageAgentInfo.imageParams.provider === "openai") {
+            // NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).
+            // dall-e-3: 7,500 RPM、15 images per minute (4 images for max resolution)
+            // gpt-image-1：3,000,000 TPM、150 images per minute
+            if (imageAgentInfo.imageParams.model === provider2ImageAgent.openai.defaultModel) {
+                return 16;
+            }
+        }
+        return 4;
+    },
     getHtmlImageAgentInfo(presentationStyle) {
         const provider = text2HtmlImageProviderSchema.parse(presentationStyle.htmlImageParams?.provider);
-        const defaultConfig = llmConfig[provider];
+        const defaultConfig = provider2LLMAgent[provider];
         const model = presentationStyle.htmlImageParams?.model ? presentationStyle.htmlImageParams?.model : defaultConfig.defaultModel;
         return {
             provider,
-            agent: defaultConfig.agent,
+            agent: defaultConfig.agentName,
             model,
             max_tokens: defaultConfig.max_tokens,
         };

package/lib/methods/mulmo_studio_context.js

@@ -52,6 +52,9 @@ export const MulmoStudioContextMethods = {
     },
     setBeatSessionState(context, sessionType, index, value) {
         if (value) {
+            if (!context.sessionState.inBeatSession[sessionType]) {
+                context.sessionState.inBeatSession[sessionType] = {};
+            }
             context.sessionState.inBeatSession[sessionType][index] = true;
         }
         else {

package/lib/tools/story_to_script.js

@@ -28,7 +28,7 @@ const createValidatedScriptGraphData = ({ systemPrompt, prompt, schema, llmAgent
                 inputs: {
                     model: llmModel,
                     system: systemPrompt,
-                    prompt: prompt,
+                    prompt,
                     max_tokens: maxTokens,
                 },
             },
@@ -36,7 +36,7 @@ const createValidatedScriptGraphData = ({ systemPrompt, prompt, schema, llmAgent
                 agent: "validateSchemaAgent",
                 inputs: {
                     text: ":llm.text.codeBlock()",
-                    schema: schema,
+                    schema,
                 },
                 isResult: true,
             },

package/lib/types/agent.d.ts

@@ -0,0 +1,55 @@
+export type OpenAIImageSize = "1792x1024" | "1024x1792" | "1024x1024" | "1536x1024" | "1024x1536";
+export type OpenAIImageModeration = "low" | "auto";
+export type OpenAIImageOptions = {
+    model: string;
+    prompt: string;
+    n: number;
+    size: OpenAIImageSize;
+    moderation?: OpenAIImageModeration;
+};
+export type AgentBufferResult = {
+    buffer: Buffer;
+};
+export type AgentPromptInputs = {
+    prompt: string;
+};
+export type ImageAgentInputs = AgentPromptInputs;
+export type OpenAIImageAgentInputs = AgentPromptInputs & {
+    referenceImages: string[] | null | undefined;
+};
+export type ImageAgentParams = {
+    model: string;
+    canvasSize: {
+        width: number;
+        height: number;
+    };
+};
+export type OpenAIImageAgentParams = ImageAgentParams & {
+    moderation: OpenAIImageModeration | null | undefined;
+};
+export type OpenAIImageAgentConfig = {
+    baseURL?: string;
+    apiKey?: string;
+};
+export type GoogleImageAgentConfig = {
+    projectId?: string;
+    token?: string;
+};
+export type MovieAgentInputs = AgentPromptInputs & {
+    imagePath?: string;
+};
+export type GoogleMovieAgentParams = ImageAgentParams & {
+    duration?: number;
+};
+export type ReplicateMovieAgentParams = {
+    model: `${string}/${string}` | undefined;
+    canvasSize: {
+        width: number;
+        height: number;
+    };
+    duration?: number;
+};
+export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
+export type ReplicateMovieAgentConfig = {
+    apiKey?: string;
+};

package/lib/types/agent.js

@@ -0,0 +1,3 @@
+// for image agent
+export {};
+// end of image agent