mulmocast 0.1.1 → 0.1.3

package/lib/agents/movie_replicate_agent.js

@@ -6,8 +6,8 @@ async function generateMovie(model, apiKey, prompt, imagePath, aspectRatio, dura
         auth: apiKey,
     });
     const input = {
-        prompt: prompt,
-        duration: duration,
+        prompt,
+        duration,
         image: undefined,
         start_image: undefined,
         aspect_ratio: aspectRatio, // only for bytedance/seedance-1-lite
@@ -58,7 +58,7 @@ export const getAspectRatio = (canvasSize) => {
         return "1:1";
     }
 };
-export const movieReplicateAgent = async ({ namedInputs, params, config }) => {
+export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, imagePath } = namedInputs;
     const aspectRatio = getAspectRatio(params.canvasSize);
     const duration = params.duration ?? 5;

package/lib/agents/tts_google_agent.d.ts

@@ -1,4 +1,12 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
-export declare const ttsGoogleAgent: AgentFunction;
+export declare const ttsGoogleAgent: AgentFunction<{
+    voice: string;
+    speed: number;
+    suppressError: boolean;
+}, {
+    buffer?: Buffer | null;
+}, {
+    text: string;
+}>;
 declare const ttsGoogleAgentInfo: AgentFunctionInfo;
 export default ttsGoogleAgentInfo;

package/lib/agents/tts_google_agent.js

@@ -14,7 +14,7 @@ export const ttsGoogleAgent = async ({ namedInputs, params }) => {
     }
     // Construct the request
     const request = {
-        input: { text: text },
+        input: { text },
         voice: voiceParams,
         audioConfig: {
             audioEncoding: "MP3",
@@ -46,6 +46,6 @@ const ttsGoogleAgentInfo = {
     author: "Receptron Team",
     repository: "https://github.com/receptron/mulmocast-cli/",
     license: "MIT",
-    environmentVariables: ["OPENAI_API_KEY"],
+    environmentVariables: ["GOOGLE_GENAI_API_KEY"],
 };
 export default ttsGoogleAgentInfo;

package/lib/agents/tts_nijivoice_agent.js

@@ -21,7 +21,7 @@ export const ttsNijivoiceAgent = async ({ params, namedInputs, config }) => {
         },
         body: JSON.stringify({
             format: "mp3",
-            speed: speed ? "" + speed : speed_global ? "" + speed_global : "1.0",
+            speed: String(speed ?? speed_global ?? "1.0"),
             script: text,
         }),
     };

package/lib/agents/tts_openai_agent.d.ts

@@ -1,4 +1,16 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
-export declare const ttsOpenaiAgent: AgentFunction;
+export declare const ttsOpenaiAgent: AgentFunction<{
+    model: string;
+    voice: string;
+    instructions: string;
+    suppressError: boolean;
+}, {
+    buffer?: Buffer;
+}, {
+    text: string;
+}, {
+    baseURL?: string;
+    apiKey?: string;
+}>;
 declare const ttsOpenaiAgentInfo: AgentFunctionInfo;
 export default ttsOpenaiAgentInfo;

package/lib/agents/tts_openai_agent.js

@@ -3,8 +3,8 @@ import OpenAI from "openai";
 export const ttsOpenaiAgent = async ({ namedInputs, params, config }) => {
     const { text } = namedInputs;
     const { model, voice, suppressError, instructions } = params;
-    const { apiKey } = config ?? {};
-    const openai = new OpenAI({ apiKey });
+    const { apiKey, baseURL } = config ?? {};
+    const openai = new OpenAI({ apiKey, baseURL });
     try {
         const tts_options = {
             model: model ?? "gpt-4o-mini-tts", // "tts-1",

package/lib/cli/helpers.js

@@ -34,16 +34,16 @@ export const getFileObject = (args) => {
     const { fileOrUrl, fileName } = (() => {
         if (file === "__clipboard") {
             // We generate a new unique script file from clipboard text in the output directory
-            const fileName = generateTimestampedFileName("script");
+            const generatedFileName = generateTimestampedFileName("script");
             const clipboardText = clipboardy.readSync();
-            const fileOrUrl = resolveDirPath(outDirPath, `${fileName}.json`);
+            const resolvedFilePath = resolveDirPath(outDirPath, `${generatedFileName}.json`);
             mkdir(outDirPath);
-            fs.writeFileSync(fileOrUrl, clipboardText, "utf8");
-            return { fileOrUrl, fileName };
+            fs.writeFileSync(resolvedFilePath, clipboardText, "utf8");
+            return { fileOrUrl: resolvedFilePath, fileName: generatedFileName };
         }
-        const fileOrUrl = file ?? "";
-        const fileName = path.parse(fileOrUrl).name;
-        return { fileOrUrl, fileName };
+        const resolvedFileOrUrl = file ?? "";
+        const parsedFileName = path.parse(resolvedFileOrUrl).name;
+        return { fileOrUrl: resolvedFileOrUrl, fileName: parsedFileName };
     })();
     const isHttpPath = isHttp(fileOrUrl);
     const mulmoFilePath = isHttpPath ? "" : getFullPath(baseDirPath, fileOrUrl);

package/lib/methods/index.d.ts

@@ -2,3 +2,4 @@ export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_beat.js";

package/lib/methods/index.js

@@ -2,3 +2,4 @@ export * from "./mulmo_presentation_style.js";
 export * from "./mulmo_script_template.js";
 export * from "./mulmo_studio_context.js";
 export * from "./mulmo_media_source.js";
+export * from "./mulmo_beat.js";

package/lib/methods/mulmo_beat.d.ts

@@ -0,0 +1,6 @@
+import { MulmoBeat } from "../types/index.js";
+export declare const MulmoBeatMethods: {
+    getHtmlPrompt(beat: MulmoBeat): string | undefined;
+    getPlugin(beat: MulmoBeat): typeof import("../utils/image_plugins/text_slide.js") | typeof import("../utils/image_plugins/markdown.js") | typeof import("../utils/image_plugins/chart.js") | typeof import("../utils/image_plugins/mermaid.js") | typeof import("../utils/image_plugins/html_tailwind.js") | typeof import("../utils/image_plugins/image.js") | typeof import("../utils/image_plugins/movie.js") | typeof import("../utils/image_plugins/beat.js") | typeof import("../utils/image_plugins/voice_over.js");
+    getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
+};

package/lib/methods/mulmo_beat.js

@@ -0,0 +1,21 @@
+import { findImagePlugin } from "../utils/image_plugins/index.js";
+export const MulmoBeatMethods = {
+    getHtmlPrompt(beat) {
+        if (beat?.htmlPrompt?.data) {
+            return beat.htmlPrompt.prompt + "\n\n data\n" + JSON.stringify(beat.htmlPrompt.data, null, 2);
+        }
+        return beat?.htmlPrompt?.prompt;
+    },
+    getPlugin(beat) {
+        const plugin = findImagePlugin(beat?.image?.type);
+        if (!plugin) {
+            throw new Error(`invalid beat image type: ${beat.image}`);
+        }
+        return plugin;
+    },
+    getImageReferenceForImageGenerator(beat, imageRefs) {
+        const imageNames = beat.imageNames ?? Object.keys(imageRefs); // use all images if imageNames is not specified
+        const sources = imageNames.map((name) => imageRefs[name]);
+        return sources.filter((source) => source !== undefined);
+    },
+};

package/lib/methods/mulmo_presentation_style.d.ts

@@ -11,6 +11,8 @@ export declare const MulmoPresentationStyleMethods: {
     getVoiceId(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat): string;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;
+    getMovieAgent(presentationStyle: MulmoPresentationStyle): string;
+    getConcurrency(presentationStyle: MulmoPresentationStyle): 4 | 16;
     getHtmlImageAgentInfo(presentationStyle: MulmoPresentationStyle): Text2HtmlAgentInfo;
     getImageType(_: MulmoPresentationStyle, beat: MulmoBeat): BeatMediaType;
 };

package/lib/methods/mulmo_presentation_style.js

@@ -74,6 +74,30 @@ export const MulmoPresentationStyleMethods = {
             imageParams: { ...defaultImageParams, ...imageParams },
         };
     },
+    // Determine movie agent based on provider
+    getMovieAgent(presentationStyle) {
+        const movieProvider = presentationStyle.movieParams?.provider ?? "google";
+        switch (movieProvider) {
+            case "replicate":
+                return "movieReplicateAgent";
+            case "google":
+            default:
+                return "movieGoogleAgent";
+        }
+    },
+    getConcurrency(presentationStyle) {
+        if (presentationStyle.movieParams?.provider === "replicate") {
+            return 4;
+        }
+        const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(presentationStyle);
+        if (imageAgentInfo.imageParams.provider === "openai") {
+            // NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).
+            // dall-e-3: 7,500 RPM、15 images per minute (4 images for max resolution)
+            // gpt-image-1：3,000,000 TPM、150 images per minute
+            return imageAgentInfo.imageParams.model === defaultOpenAIImageModel ? 4 : 16;
+        }
+        return 4;
+    },
     getHtmlImageAgentInfo(presentationStyle) {
         const provider = text2HtmlImageProviderSchema.parse(presentationStyle.htmlImageParams?.provider);
         const defaultConfig = llmConfig[provider];

package/lib/methods/mulmo_studio_context.js

@@ -52,6 +52,9 @@ export const MulmoStudioContextMethods = {
     },
     setBeatSessionState(context, sessionType, index, value) {
         if (value) {
+            if (!context.sessionState.inBeatSession[sessionType]) {
+                context.sessionState.inBeatSession[sessionType] = {};
+            }
             context.sessionState.inBeatSession[sessionType][index] = true;
         }
         else {

package/lib/tools/story_to_script.js

@@ -28,7 +28,7 @@ const createValidatedScriptGraphData = ({ systemPrompt, prompt, schema, llmAgent
                 inputs: {
                     model: llmModel,
                     system: systemPrompt,
-                    prompt: prompt,
+                    prompt,
                     max_tokens: maxTokens,
                 },
             },
@@ -36,7 +36,7 @@ const createValidatedScriptGraphData = ({ systemPrompt, prompt, schema, llmAgent
                 agent: "validateSchemaAgent",
                 inputs: {
                     text: ":llm.text.codeBlock()",
-                    schema: schema,
+                    schema,
                 },
                 isResult: true,
             },

package/lib/types/agent.d.ts

@@ -0,0 +1,55 @@
+export type OpenAIImageSize = "1792x1024" | "1024x1792" | "1024x1024" | "1536x1024" | "1024x1536";
+export type OpenAIImageModeration = "low" | "auto";
+export type OpenAIImageOptions = {
+    model: string;
+    prompt: string;
+    n: number;
+    size: OpenAIImageSize;
+    moderation?: OpenAIImageModeration;
+};
+export type AgentBufferResult = {
+    buffer: Buffer;
+};
+export type AgentPromptInputs = {
+    prompt: string;
+};
+export type ImageAgentInputs = AgentPromptInputs;
+export type OpenAIImageAgentInputs = AgentPromptInputs & {
+    referenceImages: string[] | null | undefined;
+};
+export type ImageAgentParams = {
+    model: string;
+    canvasSize: {
+        width: number;
+        height: number;
+    };
+};
+export type OpenAIImageAgentParams = ImageAgentParams & {
+    moderation: OpenAIImageModeration | null | undefined;
+};
+export type OpenAIImageAgentConfig = {
+    baseURL?: string;
+    apiKey?: string;
+};
+export type GoogleImageAgentConfig = {
+    projectId?: string;
+    token?: string;
+};
+export type MovieAgentInputs = AgentPromptInputs & {
+    imagePath?: string;
+};
+export type GoogleMovieAgentParams = ImageAgentParams & {
+    duration?: number;
+};
+export type ReplicateMovieAgentParams = {
+    model: `${string}/${string}` | undefined;
+    canvasSize: {
+        width: number;
+        height: number;
+    };
+    duration?: number;
+};
+export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
+export type ReplicateMovieAgentConfig = {
+    apiKey?: string;
+};

package/lib/types/agent.js

@@ -0,0 +1,3 @@
+// for image agent
+export {};
+// end of image agent