mulmocast 1.1.5 → 1.1.7

package/lib/types/schema.js

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import { htmlLLMProvider, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, defaultProviders, provider2SoundEffectAgent, } from "../utils/provider2agent.js";
+import { currentMulmoScriptVersion } from "../utils/const.js";
 export const langSchema = z.string();
 const URLStringSchema = z.string().url();
 export const localizedTextSchema = z
@@ -8,7 +9,8 @@ export const localizedTextSchema = z
     lang: langSchema,
     // caption: z.string(),
     texts: z.array(z.string()).optional(),
-    ttsTexts: z.array(z.string()).optional(),
+    ttsTexts: z.array(z.string()).optional(), // TODO remove
+    cacheKey: z.string(),
     duration: z.number().optional(), // generated // video duration time(ms)
     // filename: z.string().optional(), // generated //
 })
@@ -283,7 +285,7 @@ export const mulmoCanvasDimensionSchema = z
 // export const voiceMapSchema = z.record(speakerIdSchema, z.string())
 export const mulmoCastCreditSchema = z
     .object({
-    version: z.literal("1.1"),
+    version: z.literal(currentMulmoScriptVersion),
     credit: z.literal("closing").optional(),
 })
     .strict();
@@ -371,7 +373,7 @@ export const mulmoScriptSchema = mulmoPresentationStyleSchema
     title: z.string().optional(),
     description: z.string().optional(),
     references: z.array(mulmoReferenceSchema).optional(),
-    lang: langSchema.optional(), // default "en"
+    lang: langSchema, // required (default WAS "en")
     beats: z.array(mulmoBeatSchema).min(1),
     // TODO: Delete it later
     imagePath: z.string().optional(), // for keynote images movie ??
@@ -398,8 +400,13 @@ export const mulmoStudioBeatSchema = z
     .strict();
 export const mulmoStudioMultiLingualDataSchema = z.object({
     multiLingualTexts: multiLingualTextsSchema,
+    cacheKey: z.string().optional(),
 });
 export const mulmoStudioMultiLingualSchema = z.array(mulmoStudioMultiLingualDataSchema).min(1);
+export const mulmoStudioMultiLingualFileSchema = z.object({
+    version: z.literal(currentMulmoScriptVersion),
+    multiLingual: mulmoStudioMultiLingualSchema,
+});
 export const mulmoSessionStateSchema = z.object({
     inSession: z.object({
         audio: z.boolean(),

package/lib/types/type.d.ts

@@ -1,4 +1,4 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import type { LLM } from "../utils/provider2agent.js";
 import { z } from "zod";
@@ -28,6 +28,7 @@ export type MulmoPromptTemplate = z.infer<typeof mulmoPromptTemplateSchema>;
 export type MulmoPromptTemplateFile = z.infer<typeof mulmoPromptTemplateFileSchema>;
 export type MulmoStudioMultiLingual = z.infer<typeof mulmoStudioMultiLingualSchema>;
 export type MulmoStudioMultiLingualData = z.infer<typeof mulmoStudioMultiLingualDataSchema>;
+export type MulmoStudioMultiLingualFile = z.infer<typeof mulmoStudioMultiLingualFileSchema>;
 export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
 export type MulmoMovieParams = z.infer<typeof mulmoMovieParamsSchema>;
 export type MulmoOpenAIImageModel = z.infer<typeof mulmoOpenAIImageModelSchema>;
@@ -53,7 +54,7 @@ export type FileDirs = {
 export type MulmoStudioContext = {
     fileDirs: FileDirs;
     studio: MulmoStudio;
-    lang?: string;
+    lang: string;
     force: boolean;
     sessionState: MulmoSessionState;
     presentationStyle: MulmoPresentationStyle;

package/lib/utils/const.d.ts

@@ -1,3 +1,4 @@
+export declare const currentMulmoScriptVersion = "1.1";
 export declare const outDirName = "output";
 export declare const audioDirName = "audio";
 export declare const imageDirName = "images";

package/lib/utils/const.js

@@ -1,10 +1,11 @@
+export const currentMulmoScriptVersion = "1.1";
 export const outDirName = "output";
 export const audioDirName = "audio";
 export const imageDirName = "images";
 export const cacheDirName = "cache";
 export const pdf_modes = ["slide", "talk", "handout"];
 export const pdf_sizes = ["letter", "a4"];
-export const languages = ["en", "ja"];
+export const languages = ["en", "ja", "fr", "es"];
 export const storyToScriptGenerateMode = {
     stepWise: "step_wise",
     oneStep: "one_step",

package/lib/utils/context.d.ts

@@ -1,9 +1,350 @@
-import type { MulmoScript, MulmoPresentationStyle, MulmoStudioMultiLingual } from "../types/type.js";
-import { FileObject } from "../types/index.js";
+import type { MulmoScript, MulmoPresentationStyle, MulmoStudioMultiLingual, FileObject } from "../types/type.js";
+export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: string, videoCaptionLang?: string, presentationStyle?: MulmoPresentationStyle | null) => {
+    beats: {
+        duration?: number | undefined;
+        startAt?: number | undefined;
+        hash?: string | undefined;
+        audioDuration?: number | undefined;
+        movieDuration?: number | undefined;
+        silenceDuration?: number | undefined;
+        hasMovieAudio?: boolean | undefined;
+        audioFile?: string | undefined;
+        imageFile?: string | undefined;
+        movieFile?: string | undefined;
+        soundEffectFile?: string | undefined;
+        lipSyncFile?: string | undefined;
+        captionFile?: string | undefined;
+    }[];
+    script: {
+        lang: string;
+        imageParams: {
+            provider: string;
+            images?: Record<string, {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "imagePrompt";
+                prompt: string;
+            }> | undefined;
+            model?: string | undefined;
+            style?: string | undefined;
+            quality?: string | undefined;
+            moderation?: string | undefined;
+        };
+        audioParams: {
+            padding: number;
+            introPadding: number;
+            closingPadding: number;
+            outroPadding: number;
+            bgmVolume: number;
+            audioVolume: number;
+            suppressSpeech: boolean;
+            bgm?: {
+                url: string;
+                kind: "url";
+            } | {
+                kind: "base64";
+                data: string;
+            } | {
+                text: string;
+                kind: "text";
+            } | {
+                path: string;
+                kind: "path";
+            } | undefined;
+        };
+        movieParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
+        soundEffectParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        };
+        $mulmocast: {
+            version: "1.1";
+            credit?: "closing" | undefined;
+        };
+        canvasSize: {
+            width: number;
+            height: number;
+        };
+        speechParams: {
+            speakers: Record<string, {
+                voiceId: string;
+                lang?: Record<string, {
+                    voiceId: string;
+                    displayName?: Record<string, string> | undefined;
+                    isDefault?: boolean | undefined;
+                    speechOptions?: {
+                        speed?: number | undefined;
+                        instruction?: string | undefined;
+                    } | undefined;
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                }> | undefined;
+                displayName?: Record<string, string> | undefined;
+                isDefault?: boolean | undefined;
+                speechOptions?: {
+                    speed?: number | undefined;
+                    instruction?: string | undefined;
+                } | undefined;
+                provider?: string | undefined;
+                model?: string | undefined;
+            }>;
+        };
+        beats: {
+            text: string;
+            image?: {
+                type: "markdown";
+                markdown: string | string[];
+            } | {
+                type: "web";
+                url: string;
+            } | {
+                type: "pdf";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "svg";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "movie";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                slide: {
+                    title: string;
+                    subtitle?: string | undefined;
+                    bullets?: string[] | undefined;
+                };
+                type: "textSlide";
+            } | {
+                type: "chart";
+                title: string;
+                chartData: Record<string, any>;
+            } | {
+                code: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+                type: "mermaid";
+                title: string;
+                appendix?: string[] | undefined;
+            } | {
+                type: "html_tailwind";
+                html: string | string[];
+            } | {
+                type: "beat";
+                id?: string | undefined;
+            } | {
+                type: "voice_over";
+                startAt?: number | undefined;
+            } | undefined;
+            audio?: {
+                type: "audio";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "midi";
+                source: string;
+            } | undefined;
+            duration?: number | undefined;
+            speechOptions?: {
+                speed?: number | undefined;
+                instruction?: string | undefined;
+            } | undefined;
+            id?: string | undefined;
+            imagePrompt?: string | undefined;
+            speaker?: string | undefined;
+            description?: string | undefined;
+            imageParams?: {
+                provider: string;
+                images?: Record<string, {
+                    type: "image";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                } | {
+                    type: "imagePrompt";
+                    prompt: string;
+                }> | undefined;
+                model?: string | undefined;
+                style?: string | undefined;
+                quality?: string | undefined;
+                moderation?: string | undefined;
+            } | undefined;
+            audioParams?: {
+                movieVolume: number;
+                padding?: number | undefined;
+            } | undefined;
+            movieParams?: {
+                speed?: number | undefined;
+                provider?: string | undefined;
+                model?: string | undefined;
+                fillOption?: {
+                    style: "aspectFit" | "aspectFill";
+                } | undefined;
+            } | undefined;
+            soundEffectParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
+            lipSyncParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
+            htmlImageParams?: {
+                model?: string | undefined;
+            } | undefined;
+            textSlideParams?: {
+                cssStyles: string | string[];
+            } | undefined;
+            captionParams?: {
+                styles: string[];
+                lang?: string | undefined;
+            } | undefined;
+            imageNames?: string[] | undefined;
+            moviePrompt?: string | undefined;
+            soundEffectPrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                images?: Record<string, any> | undefined;
+                data?: any;
+                systemPrompt?: string | undefined;
+            } | undefined;
+            enableLipSync?: boolean | undefined;
+        }[];
+        title?: string | undefined;
+        description?: string | undefined;
+        lipSyncParams?: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        } | undefined;
+        htmlImageParams?: {
+            provider: string;
+            model?: string | undefined;
+        } | undefined;
+        textSlideParams?: {
+            cssStyles: string | string[];
+        } | undefined;
+        captionParams?: {
+            styles: string[];
+            lang?: string | undefined;
+        } | undefined;
+        references?: {
+            type: string;
+            url: string;
+            title?: string | undefined;
+            description?: string | undefined;
+        }[] | undefined;
+        imagePath?: string | undefined;
+        __test_invalid__?: boolean | undefined;
+    };
+    filename: string;
+};
 export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<MulmoScript | null>;
 export declare const getMultiLingual: (multilingualFilePath: string, studioBeatsLength: number) => MulmoStudioMultiLingual;
 export declare const getPresentationStyle: (presentationStylePath: string | undefined) => MulmoPresentationStyle | null;
-export declare const initializeContextFromFiles: (files: FileObject, raiseError: boolean, force?: boolean, caption?: string, lang?: string) => Promise<{
+export declare const initializeContextFromFiles: (files: FileObject, raiseError: boolean, force?: boolean, captionLang?: string, targetLang?: string) => Promise<{
     studio: {
         beats: {
             duration?: number | undefined;
@@ -21,12 +362,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             captionFile?: string | undefined;
         }[];
         script: {
+            lang: string;
             imageParams: {
                 provider: string;
-                model?: string | undefined;
-                style?: string | undefined;
-                quality?: string | undefined;
-                moderation?: string | undefined;
                 images?: Record<string, {
                     type: "image";
                     source: {
@@ -46,6 +384,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     type: "imagePrompt";
                     prompt: string;
                 }> | undefined;
+                model?: string | undefined;
+                style?: string | undefined;
+                quality?: string | undefined;
+                moderation?: string | undefined;
             };
             audioParams: {
                 padding: number;
@@ -185,12 +527,12 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         kind: "path";
                     };
                 } | {
-                    type: "textSlide";
                     slide: {
                         title: string;
                         subtitle?: string | undefined;
                         bullets?: string[] | undefined;
                     };
+                    type: "textSlide";
                 } | {
                     type: "chart";
                     title: string;
@@ -252,10 +594,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 description?: string | undefined;
                 imageParams?: {
                     provider: string;
-                    model?: string | undefined;
-                    style?: string | undefined;
-                    quality?: string | undefined;
-                    moderation?: string | undefined;
                     images?: Record<string, {
                         type: "image";
                         source: {
@@ -275,6 +613,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         type: "imagePrompt";
                         prompt: string;
                     }> | undefined;
+                    model?: string | undefined;
+                    style?: string | undefined;
+                    quality?: string | undefined;
+                    moderation?: string | undefined;
                 } | undefined;
                 audioParams?: {
                     movieVolume: number;
@@ -311,13 +653,12 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 soundEffectPrompt?: string | undefined;
                 htmlPrompt?: {
                     prompt: string;
-                    data?: any;
                     images?: Record<string, any> | undefined;
+                    data?: any;
                     systemPrompt?: string | undefined;
                 } | undefined;
                 enableLipSync?: boolean | undefined;
             }[];
-            lang?: string | undefined;
             title?: string | undefined;
             description?: string | undefined;
             lipSyncParams?: {
@@ -346,37 +687,21 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
         };
         filename: string;
     };
+    multiLingual: {
+        multiLingualTexts: Record<string, {
+            text: string;
+            lang: string;
+            cacheKey: string;
+            texts?: string[] | undefined;
+            ttsTexts?: string[] | undefined;
+            duration?: number | undefined;
+        }>;
+        cacheKey?: string | undefined;
+    }[];
     fileDirs: FileObject;
-    force: boolean;
-    lang: string | undefined;
-    sessionState: {
-        inSession: {
-            audio: boolean;
-            image: boolean;
-            video: boolean;
-            multiLingual: boolean;
-            caption: boolean;
-            pdf: boolean;
-        };
-        inBeatSession: {
-            audio: {};
-            image: {};
-            movie: {};
-            multiLingual: {};
-            caption: {};
-            html: {};
-            imageReference: {};
-            soundEffect: {};
-            lipSync: {};
-        };
-    };
     presentationStyle: {
         imageParams: {
             provider: string;
-            model?: string | undefined;
-            style?: string | undefined;
-            quality?: string | undefined;
-            moderation?: string | undefined;
             images?: Record<string, {
                 type: "image";
                 source: {
@@ -396,6 +721,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 type: "imagePrompt";
                 prompt: string;
             }> | undefined;
+            model?: string | undefined;
+            style?: string | undefined;
+            quality?: string | undefined;
+            moderation?: string | undefined;
         };
         audioParams: {
             padding: number;
@@ -482,13 +811,27 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             lang?: string | undefined;
         } | undefined;
     };
-    multiLingual: {
-        multiLingualTexts: Record<string, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }>;
-    }[];
+    sessionState: {
+        inSession: {
+            audio: boolean;
+            image: boolean;
+            video: boolean;
+            multiLingual: boolean;
+            caption: boolean;
+            pdf: boolean;
+        };
+        inBeatSession: {
+            audio: {};
+            image: {};
+            movie: {};
+            multiLingual: {};
+            caption: {};
+            html: {};
+            imageReference: {};
+            soundEffect: {};
+            lipSync: {};
+        };
+    };
+    force: boolean;
+    lang: string;
 } | null>;