mulmocast 0.1.3 → 0.1.4

package/lib/types/schema.js

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { htmlLLMProvider, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, defaultProviders } from "../utils/provider2agent.js";
 export const langSchema = z.string();
 const URLStringSchema = z.string().url();
 export const localizedTextSchema = z
@@ -20,7 +21,7 @@ export const speechOptionsSchema = z
 })
     .strict();
 const speakerIdSchema = z.string();
-export const text2SpeechProviderSchema = z.enum(["openai", "nijivoice", "google", "elevenlabs"]).default("openai");
+export const text2SpeechProviderSchema = z.enum(Object.keys(provider2TTSAgent)).default(defaultProviders.tts);
 export const speakerDataSchema = z
     .object({
     displayName: z.record(langSchema, z.string()).optional(),
@@ -165,19 +166,19 @@ export const mulmoFillOptionSchema = z
     style: z.enum(["aspectFit", "aspectFill"]).default("aspectFit"),
 })
     .describe("How to handle aspect ratio differences between image and canvas");
-export const text2ImageProviderSchema = z.enum(["openai", "google"]).default("openai");
+export const text2ImageProviderSchema = z.enum(Object.keys(provider2ImageAgent)).default(defaultProviders.text2image);
 // NOTE: This is for UI only. (until we figure out how to use it in mulmoImageParamsSchema)
 export const mulmoOpenAIImageModelSchema = z
     .object({
     provider: z.literal("openai"),
-    model: z.enum(["dall-e-3", "gpt-image-1"]).optional(),
+    model: z.enum(provider2ImageAgent["openai"].models).optional(),
 })
     .strict();
 // NOTE: This is for UI only. (until we figure out how to use it in mulmoImageParamsSchema)
 export const mulmoGoogleImageModelSchema = z
     .object({
     provider: z.literal("google"),
-    model: z.enum(["imagen-3.0-fast-generate-001", "imagen-3.0-generate-002", "imagen-3.0-capability-001"]).optional(),
+    model: z.enum(provider2ImageAgent["google"].models).optional(),
 })
     .strict();
 export const mulmoImageParamsSchema = z
@@ -238,6 +239,7 @@ export const mulmoBeatSchema = z
     audioParams: beatAudioParamsSchema.optional(), // beat specific parameters
     movieParams: z
         .object({
+        model: z.string().optional(),
         fillOption: mulmoFillOptionSchema.optional(),
         speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."),
     })
@@ -271,20 +273,20 @@ export const mulmoSpeechParamsSchema = z
     speakers: speakerDictionarySchema,
 })
     .strict();
-export const text2HtmlImageProviderSchema = z.enum(["openai", "anthropic"]).default("openai");
-export const text2MovieProviderSchema = z.enum(["google", "replicate"]).default("google");
+export const text2HtmlImageProviderSchema = z.enum(htmlLLMProvider).default(defaultProviders.text2Html);
+export const text2MovieProviderSchema = z.enum(Object.keys(provider2MovieAgent)).default(defaultProviders.text2movie);
 // NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
 export const mulmoGoogleMovieModelSchema = z
     .object({
     provider: z.literal("google"),
-    model: z.enum(["veo-2.0-generate-001"]).optional(),
+    model: z.enum(provider2MovieAgent.google.models).optional(),
 })
     .strict();
 // NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
 export const mulmoReplicateMovieModelSchema = z
     .object({
     provider: z.literal("replicate"),
-    model: z.enum(["bytedance/seedance-1-lite", "kwaivgi/kling-v2.1", "google/veo-3"]).optional(),
+    model: z.enum(provider2MovieAgent.replicate.models).optional(),
 })
     .strict();
 export const mulmoTransitionSchema = z.object({

package/lib/utils/const.d.ts

@@ -9,4 +9,3 @@ export declare const storyToScriptGenerateMode: {
     stepWise: string;
     oneStep: string;
 };
-export declare const defaultOpenAIImageModel = "dall-e-3";

package/lib/utils/const.js

@@ -9,4 +9,3 @@ export const storyToScriptGenerateMode = {
     stepWise: "step_wise",
     oneStep: "one_step",
 };
-export const defaultOpenAIImageModel = "dall-e-3";

package/lib/utils/context.d.ts

@@ -49,7 +49,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 height: number;
             };
             speechParams: {
-                provider: "openai" | "nijivoice" | "google" | "elevenlabs";
+                provider: string;
                 speakers: Record<string, {
                     voiceId: string;
                     displayName?: Record<string, string> | undefined;
@@ -57,7 +57,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         speed?: number | undefined;
                         instruction?: string | undefined;
                     } | undefined;
-                    provider?: "openai" | "nijivoice" | "google" | "elevenlabs" | undefined;
+                    provider?: string | undefined;
                 }>;
             };
             beats: {
@@ -196,7 +196,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 imagePrompt?: string | undefined;
                 description?: string | undefined;
                 imageParams?: {
-                    provider: "openai" | "google";
+                    provider: string;
                     style?: string | undefined;
                     model?: string | undefined;
                     moderation?: string | undefined;
@@ -225,6 +225,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
                 movieParams?: {
                     speed?: number | undefined;
+                    model?: string | undefined;
                     fillOption?: {
                         style: "aspectFit" | "aspectFill";
                     } | undefined;
@@ -252,7 +253,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             title?: string | undefined;
             description?: string | undefined;
             imageParams?: {
-                provider: "openai" | "google";
+                provider: string;
                 style?: string | undefined;
                 model?: string | undefined;
                 moderation?: string | undefined;
@@ -277,7 +278,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 }> | undefined;
             } | undefined;
             movieParams?: {
-                provider?: "google" | "replicate" | undefined;
+                provider?: string | undefined;
                 model?: string | undefined;
                 fillOption?: {
                     style: "aspectFit" | "aspectFill";
@@ -288,7 +289,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
             } | undefined;
             htmlImageParams?: {
-                provider: "openai" | "anthropic";
+                provider: string;
                 model?: string | undefined;
             } | undefined;
             textSlideParams?: {
@@ -363,7 +364,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             height: number;
         };
         speechParams: {
-            provider: "openai" | "nijivoice" | "google" | "elevenlabs";
+            provider: string;
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
@@ -371,11 +372,11 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     speed?: number | undefined;
                     instruction?: string | undefined;
                 } | undefined;
-                provider?: "openai" | "nijivoice" | "google" | "elevenlabs" | undefined;
+                provider?: string | undefined;
             }>;
         };
         imageParams?: {
-            provider: "openai" | "google";
+            provider: string;
             style?: string | undefined;
             model?: string | undefined;
             moderation?: string | undefined;
@@ -400,7 +401,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             }> | undefined;
         } | undefined;
         movieParams?: {
-            provider?: "google" | "replicate" | undefined;
+            provider?: string | undefined;
             model?: string | undefined;
             fillOption?: {
                 style: "aspectFit" | "aspectFill";
@@ -411,7 +412,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             } | undefined;
         } | undefined;
         htmlImageParams?: {
-            provider: "openai" | "anthropic";
+            provider: string;
             model?: string | undefined;
         } | undefined;
         textSlideParams?: {

package/lib/utils/preprocess.d.ts

@@ -44,7 +44,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             height: number;
         };
         speechParams: {
-            provider: "openai" | "nijivoice" | "google" | "elevenlabs";
+            provider: string;
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
@@ -52,7 +52,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                     speed?: number | undefined;
                     instruction?: string | undefined;
                 } | undefined;
-                provider?: "openai" | "nijivoice" | "google" | "elevenlabs" | undefined;
+                provider?: string | undefined;
             }>;
         };
         beats: {
@@ -191,7 +191,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             imagePrompt?: string | undefined;
             description?: string | undefined;
             imageParams?: {
-                provider: "openai" | "google";
+                provider: string;
                 style?: string | undefined;
                 model?: string | undefined;
                 moderation?: string | undefined;
@@ -220,6 +220,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
             movieParams?: {
                 speed?: number | undefined;
+                model?: string | undefined;
                 fillOption?: {
                     style: "aspectFit" | "aspectFill";
                 } | undefined;
@@ -247,7 +248,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         title?: string | undefined;
         description?: string | undefined;
         imageParams?: {
-            provider: "openai" | "google";
+            provider: string;
             style?: string | undefined;
             model?: string | undefined;
             moderation?: string | undefined;
@@ -272,7 +273,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             }> | undefined;
         } | undefined;
         movieParams?: {
-            provider?: "google" | "replicate" | undefined;
+            provider?: string | undefined;
             model?: string | undefined;
             fillOption?: {
                 style: "aspectFit" | "aspectFill";
@@ -283,7 +284,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
         } | undefined;
         htmlImageParams?: {
-            provider: "openai" | "anthropic";
+            provider: string;
             model?: string | undefined;
         } | undefined;
         textSlideParams?: {

package/lib/utils/provider2agent.d.ts

@@ -0,0 +1,72 @@
+export declare const defaultProviders: {
+    tts: string;
+    text2image: string;
+    text2movie: string;
+    text2Html: string;
+    llm: string;
+};
+export declare const provider2TTSAgent: {
+    nijivoice: {
+        agentName: string;
+        hasLimitedConcurrency: boolean;
+    };
+    openai: {
+        agentName: string;
+        hasLimitedConcurrency: boolean;
+    };
+    google: {
+        agentName: string;
+        hasLimitedConcurrency: boolean;
+    };
+    elevenlabs: {
+        agentName: string;
+        hasLimitedConcurrency: boolean;
+    };
+};
+export declare const provider2ImageAgent: {
+    openai: {
+        agentName: string;
+        defaultModel: string;
+        models: string[];
+    };
+    google: {
+        agentName: string;
+        defaultModel: string;
+        models: string[];
+    };
+};
+export declare const provider2MovieAgent: {
+    replicate: {
+        agentName: string;
+        models: string[];
+    };
+    google: {
+        agentName: string;
+        models: string[];
+    };
+};
+export declare const provider2LLMAgent: {
+    readonly openai: {
+        readonly agentName: "openAIAgent";
+        readonly defaultModel: "gpt-4o";
+        readonly max_tokens: 8192;
+    };
+    readonly anthropic: {
+        readonly agentName: "anthropicAgent";
+        readonly defaultModel: "claude-3-7-sonnet-20250219";
+        readonly max_tokens: 8192;
+    };
+    readonly gemini: {
+        readonly agentName: "geminiAgent";
+        readonly defaultModel: "gemini-1.5-flash";
+        readonly max_tokens: 8192;
+    };
+    readonly groq: {
+        readonly agentName: "groqAgent";
+        readonly defaultModel: "llama3-8b-8192";
+        readonly max_tokens: 4096;
+    };
+};
+export declare const llm: (keyof typeof provider2LLMAgent)[];
+export type LLM = keyof typeof provider2LLMAgent;
+export declare const htmlLLMProvider: string[];

package/lib/utils/provider2agent.js

@@ -0,0 +1,81 @@
+export const defaultProviders = {
+    tts: "openai",
+    text2image: "openai",
+    text2movie: "google",
+    text2Html: "openai",
+    llm: "openai",
+};
+export const provider2TTSAgent = {
+    nijivoice: {
+        agentName: "ttsNijivoiceAgent",
+        hasLimitedConcurrency: true,
+    },
+    openai: {
+        agentName: "ttsOpenaiAgent",
+        hasLimitedConcurrency: false,
+    },
+    google: {
+        agentName: "ttsGoogleAgent",
+        hasLimitedConcurrency: false,
+    },
+    elevenlabs: {
+        agentName: "ttsElevenlabsAgent",
+        hasLimitedConcurrency: true,
+    },
+};
+export const provider2ImageAgent = {
+    openai: {
+        agentName: "imageOpenaiAgent",
+        defaultModel: "gpt-image-1",
+        models: ["dall-e-3", "gpt-image-1"],
+    },
+    google: {
+        agentName: "imageGoogleAgent",
+        defaultModel: "imagen-3.0-fast-generate-001",
+        models: ["imagen-3.0-fast-generate-001", "imagen-3.0-generate-002", "imagen-3.0-capability-001"],
+    },
+};
+export const provider2MovieAgent = {
+    replicate: {
+        agentName: "movieReplicateAgent",
+        models: [
+            "bytedance/seedance-1-lite",
+            "bytedance/seedance-1-pro",
+            "kwaivgi/kling-v1.6-pro",
+            "kwaivgi/kling-v2.1",
+            "google/veo-2",
+            "google/veo-3",
+            "google/veo-3-fast",
+            "minimax/video-01",
+        ],
+    },
+    google: {
+        agentName: "movieGoogleAgent",
+        models: ["veo-2.0-generate-001"],
+    },
+};
+// : Record<LLM, { agent: string; defaultModel: string; max_tokens: number }>
+export const provider2LLMAgent = {
+    openai: {
+        agentName: "openAIAgent",
+        defaultModel: "gpt-4o",
+        max_tokens: 8192,
+    },
+    anthropic: {
+        agentName: "anthropicAgent",
+        defaultModel: "claude-3-7-sonnet-20250219",
+        max_tokens: 8192,
+    },
+    gemini: {
+        agentName: "geminiAgent",
+        defaultModel: "gemini-1.5-flash",
+        max_tokens: 8192,
+    },
+    groq: {
+        agentName: "groqAgent",
+        defaultModel: "llama3-8b-8192",
+        max_tokens: 4096,
+    },
+};
+export const llm = Object.keys(provider2LLMAgent);
+export const htmlLLMProvider = ["openai", "anthropic"];

package/lib/utils/utils.d.ts

@@ -1,16 +1,12 @@
-import { MulmoBeat, MulmoStudioMultiLingualData } from "../types/index.js";
 import type { ConfigDataDictionary, DefaultConfigData } from "graphai";
-export declare const llm: readonly ["openai", "anthropic", "gemini", "groq"];
-export type LLM = (typeof llm)[number];
-export declare const llmConfig: Record<LLM, {
-    agent: string;
-    defaultModel: string;
-    max_tokens: number;
-}>;
+import { MulmoBeat, MulmoStudioMultiLingualData } from "../types/index.js";
+import { llm } from "./provider2agent.js";
+import type { LLM } from "./provider2agent.js";
+export { LLM, llm };
 export declare const llmPair: (_llm?: LLM, _model?: string) => {
-    agent: string;
+    agent: "openAIAgent" | "anthropicAgent" | "geminiAgent" | "groqAgent";
     model: string;
-    max_tokens: number;
+    max_tokens: 8192 | 4096;
 };
 export declare const chunkArray: <T>(array: T[], size?: number) => T[][];
 export declare const isHttp: (fileOrUrl: string) => boolean;
@@ -26,4 +22,3 @@ type CleanableObject = {
     [key: string]: CleanableValue;
 };
 export declare const deepClean: <T extends CleanableValue>(input: T) => T | undefined;
-export {};

package/lib/utils/utils.js

@@ -1,32 +1,11 @@
 import * as crypto from "crypto";
-export const llm = ["openai", "anthropic", "gemini", "groq"];
-export const llmConfig = {
-    openai: {
-        agent: "openAIAgent",
-        defaultModel: "gpt-4o",
-        max_tokens: 8192,
-    },
-    anthropic: {
-        agent: "anthropicAgent",
-        defaultModel: "claude-3-7-sonnet-20250219",
-        max_tokens: 8192,
-    },
-    gemini: {
-        agent: "geminiAgent",
-        defaultModel: "gemini-1.5-flash",
-        max_tokens: 8192,
-    },
-    groq: {
-        agent: "groqAgent",
-        defaultModel: "llama3-8b-8192",
-        max_tokens: 4096,
-    },
-};
+import { provider2LLMAgent, llm } from "./provider2agent.js";
+export { llm };
 export const llmPair = (_llm, _model) => {
     const llmKey = _llm ?? "openai";
-    const agent = llmConfig[llmKey]?.agent ?? llmConfig.openai.agent;
-    const model = _model ?? llmConfig[llmKey]?.defaultModel ?? llmConfig.openai.defaultModel;
-    const max_tokens = llmConfig[llmKey]?.max_tokens ?? llmConfig.openai.max_tokens;
+    const agent = provider2LLMAgent[llmKey]?.agentName ?? provider2LLMAgent.openai.agentName;
+    const model = _model ?? provider2LLMAgent[llmKey]?.defaultModel ?? provider2LLMAgent.openai.defaultModel;
+    const max_tokens = provider2LLMAgent[llmKey]?.max_tokens ?? provider2LLMAgent.openai.max_tokens;
     return { agent, model, max_tokens };
 };
 export const chunkArray = (array, size = 3) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "",
   "type": "module",
   "main": "lib/index.js",