mulmocast 0.1.3 → 0.1.5

package/lib/methods/mulmo_presentation_style.js

@@ -1,7 +1,7 @@
 import "dotenv/config";
-import { userAssert, llmConfig } from "../utils/utils.js";
+import { userAssert } from "../utils/utils.js";
 import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema } from "../types/schema.js";
-import { defaultOpenAIImageModel } from "../utils/const.js";
+import { defaultProviders, provider2ImageAgent, provider2MovieAgent, provider2LLMAgent } from "../utils/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
     "body { margin: 60px; margin-top: 40px; color:#333; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -49,10 +49,14 @@ export const MulmoPresentationStyleMethods = {
         userAssert(!!speaker, `speaker is not set: speaker "${beat.speaker}"`);
         return speaker;
     },
-    getProvider(presentationStyle, beat) {
+    getTTSProvider(presentationStyle, beat) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
         return speaker.provider ?? presentationStyle.speechParams.provider;
     },
+    getTTSModel(presentationStyle, beat) {
+        const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
+        return speaker.model ?? presentationStyle.speechParams.model;
+    },
     getVoiceId(presentationStyle, beat) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(presentationStyle, beat);
         return speaker.voiceId;
@@ -65,46 +69,46 @@ export const MulmoPresentationStyleMethods = {
         // provider and model appropriately.
         const imageParams = { ...presentationStyle.imageParams, ...beat?.imageParams };
         const provider = MulmoPresentationStyleMethods.getText2ImageProvider(imageParams?.provider);
+        const agentInfo = provider2ImageAgent[provider];
+        // The default text2image model is gpt-image-1 from OpenAI, and to use it you must have an OpenAI account and have verified your identity. If this is not possible, please specify dall-e-3 as the model.
         const defaultImageParams = {
             provider,
-            model: provider === "openai" ? (process.env.DEFAULT_OPENAI_IMAGE_MODEL ?? defaultOpenAIImageModel) : undefined,
+            model: agentInfo.defaultModel,
         };
         return {
-            agent: provider === "google" ? "imageGoogleAgent" : "imageOpenaiAgent",
+            agent: agentInfo.agentName,
             imageParams: { ...defaultImageParams, ...imageParams },
         };
     },
     // Determine movie agent based on provider
     getMovieAgent(presentationStyle) {
-        const movieProvider = presentationStyle.movieParams?.provider ?? "google";
-        switch (movieProvider) {
-            case "replicate":
-                return "movieReplicateAgent";
-            case "google":
-            default:
-                return "movieGoogleAgent";
-        }
+        const movieProvider = (presentationStyle.movieParams?.provider ?? defaultProviders.text2movie);
+        return provider2MovieAgent[movieProvider].agentName;
     },
     getConcurrency(presentationStyle) {
+        /*
         if (presentationStyle.movieParams?.provider === "replicate") {
-            return 4;
+          return 4;
         }
+        */
         const imageAgentInfo = MulmoPresentationStyleMethods.getImageAgentInfo(presentationStyle);
         if (imageAgentInfo.imageParams.provider === "openai") {
             // NOTE: Here are the rate limits of OpenAI's text2image API (1token = 32x32 patch).
             // dall-e-3: 7,500 RPM、15 images per minute (4 images for max resolution)
             // gpt-image-1：3,000,000 TPM、150 images per minute
-            return imageAgentInfo.imageParams.model === defaultOpenAIImageModel ? 4 : 16;
+            if (imageAgentInfo.imageParams.model === provider2ImageAgent.openai.defaultModel) {
+                return 16;
+            }
         }
         return 4;
     },
     getHtmlImageAgentInfo(presentationStyle) {
         const provider = text2HtmlImageProviderSchema.parse(presentationStyle.htmlImageParams?.provider);
-        const defaultConfig = llmConfig[provider];
+        const defaultConfig = provider2LLMAgent[provider];
         const model = presentationStyle.htmlImageParams?.model ? presentationStyle.htmlImageParams?.model : defaultConfig.defaultModel;
         return {
             provider,
-            agent: defaultConfig.agent,
+            agent: defaultConfig.agentName,
             model,
             max_tokens: defaultConfig.max_tokens,
         };

package/lib/types/agent.d.ts

@@ -13,6 +13,15 @@ export type AgentBufferResult = {
 export type AgentPromptInputs = {
     prompt: string;
 };
+export type AgentTextInputs = {
+    text: string;
+};
+export type AgentErrorResult = {
+    error: unknown;
+};
+export type AgentConfig = {
+    apiKey?: string;
+};
 export type ImageAgentInputs = AgentPromptInputs;
 export type OpenAIImageAgentInputs = AgentPromptInputs & {
     referenceImages: string[] | null | undefined;
@@ -50,6 +59,24 @@ export type ReplicateMovieAgentParams = {
     duration?: number;
 };
 export type GoogleMovieAgentConfig = GoogleImageAgentConfig;
-export type ReplicateMovieAgentConfig = {
-    apiKey?: string;
+export type ReplicateMovieAgentConfig = AgentConfig;
+export type TTSAgentParams = {
+    suppressError: boolean;
+    voice: string;
+};
+export type OpenAITTSAgentParams = TTSAgentParams & {
+    instructions: string;
+    model: string;
+};
+export type NijivoiceTTSAgentParams = TTSAgentParams & {
+    speed: number;
+    speed_global: number;
+};
+export type GoogleTTSAgentParams = TTSAgentParams & {
+    speed: number;
+};
+export type ElevenlabsTTSAgentParams = TTSAgentParams & {
+    model: string;
+    stability: number;
+    similarityBoost: number;
 };

package/lib/types/agent.js

@@ -1,3 +1,2 @@
 // for image agent
 export {};
-// end of image agent