mulmocast 0.1.7 → 1.1.1

package/lib/types/schema.js

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { htmlLLMProvider, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, defaultProviders } from "../utils/provider2agent.js";
+import { htmlLLMProvider, provider2TTSAgent, provider2ImageAgent, provider2MovieAgent, defaultProviders, provider2SoundEffectAgent, } from "../utils/provider2agent.js";
 export const langSchema = z.string();
 const URLStringSchema = z.string().url();
 export const localizedTextSchema = z
@@ -26,6 +26,7 @@ export const speakerDataSchema = z
     .object({
     displayName: z.record(langSchema, z.string()).optional(),
     voiceId: z.string(),
+    isDefault: z.boolean().optional(),
     speechOptions: speechOptionsSchema.optional(),
     provider: text2SpeechProviderSchema.optional(),
     model: z.string().optional().describe("TTS model to use for this speaker"),
@@ -227,10 +228,18 @@ export const htmlPromptParamsSchema = z
 })
     .strict();
 export const text2MovieProviderSchema = z.enum(Object.keys(provider2MovieAgent)).default(defaultProviders.text2movie);
-const defaultSpeaker = "Presenter";
+export const text2SoundEffectProviderSchema = z.enum(Object.keys(provider2SoundEffectAgent)).default(defaultProviders.soundEffect);
+export const mulmoSoundEffectParamsSchema = z.object({
+    provider: text2SoundEffectProviderSchema.optional(),
+    model: z.string().optional(), // default: provider specific
+});
+export const mulmoLipSyncParamsSchema = z.object({
+    provider: z.string().optional(), // lip sync provider
+    model: z.string().optional(), // default: provider specific
+});
 export const mulmoBeatSchema = z
     .object({
-    speaker: speakerIdSchema.default(defaultSpeaker),
+    speaker: speakerIdSchema.optional(),
     text: z.string().default("").describe("Text to be spoken. If empty, the audio is not generated."),
     id: z.string().optional().describe("Unique identifier for the beat."),
     description: z.string().optional(),
@@ -247,6 +256,8 @@ export const mulmoBeatSchema = z
         speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."), // for movie.ts
     })
         .optional(),
+    soundEffectParams: mulmoSoundEffectParamsSchema.optional(),
+    lipSyncParams: mulmoLipSyncParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
     speechOptions: speechOptionsSchema.optional(),
     textSlideParams: textSlideParamsSchema.optional(),
@@ -254,7 +265,9 @@ export const mulmoBeatSchema = z
     imageNames: z.array(imageIdSchema).optional(), // list of image names to use for image generation. The default is all images in the imageParams.images.
     imagePrompt: z.string().optional(),
     moviePrompt: z.string().optional(),
+    soundEffectPrompt: z.string().optional(),
     htmlPrompt: htmlPromptParamsSchema.optional(),
+    enableLipSync: z.boolean().optional().describe("Enable lip sync generation for this beat"),
 })
     .strict();
 export const mulmoCanvasDimensionSchema = z
@@ -266,17 +279,10 @@ export const mulmoCanvasDimensionSchema = z
 // export const voiceMapSchema = z.record(speakerIdSchema, z.string())
 export const mulmoCastCreditSchema = z
     .object({
-    version: z.literal("1.0"),
+    version: z.literal("1.1"),
     credit: z.literal("closing").optional(),
 })
     .strict();
-export const mulmoSpeechParamsSchema = z
-    .object({
-    provider: text2SpeechProviderSchema, // has default value
-    speakers: speakerDictionarySchema,
-    model: z.string().optional().describe("Default TTS model to use"),
-})
-    .strict();
 export const text2HtmlImageProviderSchema = z.enum(htmlLLMProvider).default(defaultProviders.text2Html);
 // NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
 export const mulmoGoogleMovieModelSchema = z
@@ -304,10 +310,15 @@ export const mulmoMovieParamsSchema = z
     fillOption: mulmoFillOptionSchema.optional(), // for movie.ts
 })
     .strict();
+const defaultSpeaker = "Presenter";
 export const mulmoPresentationStyleSchema = z.object({
     $mulmocast: mulmoCastCreditSchema,
     canvasSize: mulmoCanvasDimensionSchema, // has default value
-    speechParams: mulmoSpeechParamsSchema.default({
+    speechParams: z
+        .object({
+        speakers: speakerDictionarySchema,
+    })
+        .default({
         speakers: {
             [defaultSpeaker]: {
                 voiceId: "shimmer",
@@ -324,6 +335,10 @@ export const mulmoPresentationStyleSchema = z.object({
     movieParams: mulmoMovieParamsSchema.optional().default({
         provider: defaultProviders.text2movie,
     }),
+    soundEffectParams: mulmoSoundEffectParamsSchema.optional().default({
+        provider: defaultProviders.soundEffect,
+    }),
+    lipSyncParams: mulmoLipSyncParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema
         .extend({
         provider: text2HtmlImageProviderSchema,
@@ -372,6 +387,8 @@ export const mulmoStudioBeatSchema = z
     audioFile: z.string().optional(),
     imageFile: z.string().optional(), // path to the image
     movieFile: z.string().optional(), // path to the movie file
+    soundEffectFile: z.string().optional(), // path to the sound effect file
+    lipSyncFile: z.string().optional(), // path to the lip sync file
     captionFile: z.string().optional(), // path to the caption image
 })
     .strict();
@@ -396,6 +413,8 @@ export const mulmoSessionStateSchema = z.object({
         caption: z.record(z.number().int(), z.boolean()),
         html: z.record(z.number().int(), z.boolean()),
         imageReference: z.record(z.number().int(), z.boolean()),
+        soundEffect: z.record(z.number().int(), z.boolean()),
+        lipSync: z.record(z.number().int(), z.boolean()),
     }),
 });
 export const mulmoStudioSchema = z

package/lib/types/type.d.ts

@@ -1,11 +1,10 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import type { LLM } from "../utils/provider2agent.js";
 import { z } from "zod";
 export type LANG = z.infer<typeof langSchema>;
 export type MulmoBeat = z.infer<typeof mulmoBeatSchema>;
 export type SpeakerDictonary = z.infer<typeof speakerDictionarySchema>;
-export type MulmoSpeechParams = z.infer<typeof mulmoSpeechParamsSchema>;
 export type SpeechOptions = z.infer<typeof speechOptionsSchema>;
 export type SpeakerData = z.infer<typeof speakerDataSchema>;
 export type MulmoImageParams = z.infer<typeof mulmoImageParamsSchema>;
@@ -92,7 +91,7 @@ export type Text2HtmlAgentInfo = {
 export type BeatMediaType = "movie" | "image";
 export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];
 export type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
-export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference";
+export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference" | "soundEffect" | "lipSync";
 export type SessionProgressEvent = {
     kind: "session";
     sessionType: SessionType;

package/lib/utils/assets.d.ts

@@ -0,0 +1,18 @@
+type BgmAsset = {
+    name: string;
+    title: string;
+    url: string;
+    suno_url: string;
+    date: string;
+    duration: string;
+    account: string;
+    original_license: string;
+    prompt: string;
+    model: string;
+};
+export type BgmAssets = {
+    license: string;
+    bgms: BgmAsset[];
+};
+export declare const bgmAssets: BgmAssets;
+export {};

package/lib/utils/assets.js

@@ -0,0 +1,101 @@
+export const bgmAssets = {
+    license: "Free to distribute as the BMG of media generated by MulmoCast, including commercial use.",
+    bgms: [
+        {
+            name: "story001.mp3",
+            title: "Whispered Melody",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story001.mp3",
+            suno_url: "https://suno.com/s/v6zer50aQJu8Y0qA",
+            date: "2025-06-17",
+            duration: "03:17",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "instrumental, smooth, piano",
+            model: "v4.5 beta",
+        },
+        {
+            name: "story002.mp3",
+            title: "Rise and Shine",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story002.mp3",
+            suno_url: "https://suno.com/s/mJnvyu3UXnkdAPfQ",
+            date: "2025-06-17",
+            duration: "04:04",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "techno, inspiring, piano",
+            model: "v4.5 beta",
+        },
+        {
+            name: "story003.mp3",
+            title: "Chasing the Sunset",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story003.mp3",
+            suno_url: "https://suno.com/s/2zGjMQ9vURJbaMZA",
+            date: "2025-06-17",
+            duration: "02:49",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "piano, inspiring, sunset",
+            model: "v4.5 beta",
+        },
+        {
+            name: "story004.mp3",
+            title: "Whispering Keys",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story004.mp3",
+            suno_url: "https://suno.com/s/0SFoBRsBWsncw6tu",
+            date: "2025-06-17",
+            duration: "04:00",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Piano, classical, ambient",
+            model: "v4",
+        },
+        {
+            name: "story005.mp3",
+            title: "Whisper of Ivory",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/story005.mp3",
+            suno_url: "https://suno.com/s/0SFoBRsBWsncw6tu",
+            date: "2025-06-17",
+            duration: "04:00",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Piano solo, classical, ambient",
+            model: "v4",
+        },
+        {
+            name: "theme001.mp3",
+            title: "Rise of the Flame",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/theme001.mp3",
+            suno_url: "https://suno.com/s/WhYOf8oJYhBgSKET",
+            date: "2025-06-20",
+            duration: "03:23",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Olympic Theme, classical, emotional",
+            model: "v4",
+        },
+        {
+            name: "olympic001.mp3",
+            title: "Olympic-style Theme Music",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/olympic001.mp3",
+            suno_url: "https://suno.com/s/32wpnmCrkFVvkTSQ",
+            date: "2025-07-17",
+            duration: "02:54",
+            account: "@snakajima",
+            original_license: "Generated by Suno with commercial use rights (PRO Plan)",
+            prompt: "Epic orchestral fanfare in the style of John Williams' Olympic Fanfare and Theme. Bright brass fanfare, soaring strings, powerful percussion, and heroic French horn melodies. Triumphant and majestic mood, suitable for an opening ceremony or national celebration. Emphasize dynamic builds, rich harmonies, and cinematic grandeur.",
+            model: "v4.5+",
+        },
+        {
+            name: "morning001.mp3",
+            title: "Morning Dance",
+            url: "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/bgms/morning001.mp3",
+            suno_url: "https://suno.com/s/9MTkutZYqxeyBlwK",
+            date: "2025-07-17",
+            duration: "03:52",
+            account: "@snakajima",
+            original_license: "morning, piano solo, Japanese name, sexy",
+            prompt: "morning, piano solo, Japanese name, sexy",
+            model: "v4.5+",
+        },
+    ],
+};

package/lib/utils/context.d.ts

@@ -16,6 +16,8 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             audioFile?: string | undefined;
             imageFile?: string | undefined;
             movieFile?: string | undefined;
+            soundEffectFile?: string | undefined;
+            lipSyncFile?: string | undefined;
             captionFile?: string | undefined;
         }[];
         script: {
@@ -77,8 +79,12 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     duration: number;
                 } | undefined;
             };
+            soundEffectParams: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            };
             $mulmocast: {
-                version: "1.0";
+                version: "1.1";
                 credit?: "closing" | undefined;
             };
             canvasSize: {
@@ -86,10 +92,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 height: number;
             };
             speechParams: {
-                provider: string;
                 speakers: Record<string, {
                     voiceId: string;
                     displayName?: Record<string, string> | undefined;
+                    isDefault?: boolean | undefined;
                     speechOptions?: {
                         speed?: number | undefined;
                         instruction?: string | undefined;
@@ -97,16 +103,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     provider?: string | undefined;
                     model?: string | undefined;
                 }>;
-                model?: string | undefined;
             };
             beats: {
                 text: string;
-                speaker: string;
-                duration?: number | undefined;
-                speechOptions?: {
-                    speed?: number | undefined;
-                    instruction?: string | undefined;
-                } | undefined;
                 image?: {
                     type: "markdown";
                     markdown: string | string[];
@@ -211,6 +210,11 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     type: "voice_over";
                     startAt?: number | undefined;
                 } | undefined;
+                duration?: number | undefined;
+                speechOptions?: {
+                    speed?: number | undefined;
+                    instruction?: string | undefined;
+                } | undefined;
                 id?: string | undefined;
                 audio?: {
                     type: "audio";
@@ -232,6 +236,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     source: string;
                 } | undefined;
                 imagePrompt?: string | undefined;
+                speaker?: string | undefined;
                 description?: string | undefined;
                 imageParams?: {
                     provider: string;
@@ -270,6 +275,14 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         style: "aspectFit" | "aspectFill";
                     } | undefined;
                 } | undefined;
+                soundEffectParams?: {
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                } | undefined;
+                lipSyncParams?: {
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                } | undefined;
                 htmlImageParams?: {
                     model?: string | undefined;
                 } | undefined;
@@ -282,16 +295,22 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
                 imageNames?: string[] | undefined;
                 moviePrompt?: string | undefined;
+                soundEffectPrompt?: string | undefined;
                 htmlPrompt?: {
                     prompt: string;
                     data?: any;
                     images?: Record<string, any> | undefined;
                     systemPrompt?: string | undefined;
                 } | undefined;
+                enableLipSync?: boolean | undefined;
             }[];
             lang?: string | undefined;
             title?: string | undefined;
             description?: string | undefined;
+            lipSyncParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
             htmlImageParams?: {
                 provider: string;
                 model?: string | undefined;
@@ -334,6 +353,8 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             caption: {};
             html: {};
             imageReference: {};
+            soundEffect: {};
+            lipSync: {};
         };
     };
     presentationStyle: {
@@ -395,8 +416,12 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 duration: number;
             } | undefined;
         };
+        soundEffectParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        };
         $mulmocast: {
-            version: "1.0";
+            version: "1.1";
             credit?: "closing" | undefined;
         };
         canvasSize: {
@@ -404,10 +429,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             height: number;
         };
         speechParams: {
-            provider: string;
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
+                isDefault?: boolean | undefined;
                 speechOptions?: {
                     speed?: number | undefined;
                     instruction?: string | undefined;
@@ -415,8 +440,11 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 provider?: string | undefined;
                 model?: string | undefined;
             }>;
-            model?: string | undefined;
         };
+        lipSyncParams?: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/context.js

@@ -58,6 +58,8 @@ const initSessionState = () => {
             caption: {},
             html: {},
             imageReference: {},
+            soundEffect: {},
+            lipSync: {},
         },
     };
 };
@@ -84,7 +86,7 @@ export const initializeContextFromFiles = async (files, raiseError, force, capti
     const currentStudio = readMulmoScriptFile(outputStudioFilePath);
     try {
         // validate mulmoStudioSchema. skip if __test_invalid__ is true
-        const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName, caption);
+        const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName, caption, presentationStyle);
         const multiLingual = getMultiLingual(outputMultilingualFilePath, studio.beats.length);
         return buildContext(studio, files, presentationStyle, multiLingual, force, lang);
     }

package/lib/utils/file.d.ts

@@ -1,6 +1,6 @@
 import type { MulmoScript, MulmoScriptTemplateFile, MulmoStudioContext } from "../types/index.js";
 import { PDFMode } from "../types/index.js";
-import { ZodSchema } from "zod";
+import { ZodSchema, ZodType } from "zod";
 export declare const updateNpmRoot: (_npmRoot: string) => void;
 export declare function readMulmoScriptFile<T = MulmoScript>(path: string, errorMessage: string): {
     mulmoData: T;
@@ -25,11 +25,15 @@ export declare const getAudioArtifactFilePath: (outDirPath: string, fileName: st
 export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string, lang?: string, caption?: string) => string;
 export declare const imageSuffix = "p";
 export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: number) => string;
-export declare const getBeatMoviePath: (context: MulmoStudioContext, index: number) => string;
+export declare const getBeatMoviePaths: (context: MulmoStudioContext, index: number) => {
+    movieFile: string;
+    soundEffectFile: string;
+    lipSyncFile: string;
+};
 export declare const getReferenceImagePath: (context: MulmoStudioContext, key: string, extension: string) => string;
 export declare const getCaptionImagePath: (context: MulmoStudioContext, index: number) => string;
 export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;
-export declare const getTemplateFilePath: (templateName: string) => string;
+export declare const getPromptTemplateFilePath: (promptTemplateName: string) => string;
 export declare const mkdir: (dirPath: string) => void;
 export declare const silent60secPath: () => string;
 export declare const defaultBGMPath: () => string;
@@ -37,8 +41,12 @@ export declare const getHTMLFile: (filename: string) => string;
 export declare const getBaseDirPath: (basedir?: string) => string;
 export declare const getFullPath: (baseDirPath: string | undefined, file: string) => string;
 export declare const readScriptTemplateFile: (scriptName: string) => any;
-export declare const readTemplatePrompt: (templateName: string) => string;
+export declare const getScriptFromPromptTemplate: (promptTemplateName: string) => any;
+export declare const readTemplatePrompt: (promptTemplateName: string) => string;
 export declare const getAvailableTemplates: () => MulmoScriptTemplateFile[];
+export declare const getAvailablePromptTemplates: () => MulmoScriptTemplateFile[];
+export declare const getAvailableScriptTemplates: () => MulmoScriptTemplateFile[];
+export declare const getPromptTemplates: <T>(dirPath: string, schema: ZodType) => T[];
 export declare const writingMessage: (filePath: string) => void;
 export declare const readAndParseJson: <S extends ZodSchema<any>>(filePath: string, schema: S) => ReturnType<S["parse"]>;
 export declare const generateTimestampedFileName: (prefix: string) => string;

package/lib/utils/file.js

@@ -4,7 +4,9 @@ import { parse as yamlParse } from "yaml";
 import { fileURLToPath } from "url";
 import { GraphAILogger } from "graphai";
 import { MulmoScriptTemplateMethods, MulmoStudioContextMethods } from "../methods/index.js";
-import { mulmoScriptTemplateSchema } from "../types/schema.js";
+import { mulmoScriptTemplateSchema, mulmoPresentationStyleSchema } from "../types/schema.js";
+const promptTemplateDirName = "./assets/templates";
+const scriptTemplateDirName = "./scripts/templates";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 let npmRoot = path.resolve(__dirname, "../../");
@@ -87,13 +89,15 @@ export const getBeatPngImagePath = (context, index) => {
     }
     return `${imageProjectDirPath}/${index}${imageSuffix}.png`;
 };
-export const getBeatMoviePath = (context, index) => {
+export const getBeatMoviePaths = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
     const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
-    if (beat?.id) {
-        return `${imageProjectDirPath}/${beat.id}.mov`;
-    }
-    return `${imageProjectDirPath}/${index}.mov`;
+    const filename = beat?.id ? `${beat.id}` : `${index}`;
+    return {
+        movieFile: `${imageProjectDirPath}/${filename}.mov`,
+        soundEffectFile: `${imageProjectDirPath}/${filename}_sound.mov`,
+        lipSyncFile: `${imageProjectDirPath}/${filename}_lipsync.mov`,
+    };
 };
 export const getReferenceImagePath = (context, key, extension) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
@@ -110,8 +114,8 @@ export const getOutputPdfFilePath = (outDirPath, fileName, pdfMode, lang) => {
     }
     return path.resolve(outDirPath, `${fileName}_${pdfMode}.pdf`);
 };
-export const getTemplateFilePath = (templateName) => {
-    return path.resolve(npmRoot, "./assets/templates/" + templateName + ".json");
+export const getPromptTemplateFilePath = (promptTemplateName) => {
+    return path.resolve(npmRoot, promptTemplateDirName, promptTemplateName + ".json");
 };
 export const mkdir = (dirPath) => {
     if (!fs.existsSync(dirPath)) {
@@ -147,36 +151,56 @@ export const getFullPath = (baseDirPath, file) => {
     return path.resolve(file);
 };
 export const readScriptTemplateFile = (scriptName) => {
-    const scriptPath = path.resolve(npmRoot, "./scripts/templates", scriptName);
+    const scriptPath = path.resolve(npmRoot, scriptTemplateDirName, scriptName);
     const scriptData = fs.readFileSync(scriptPath, "utf-8");
     // NOTE: We don't want to schema parse the script here to eliminate default values.
     return JSON.parse(scriptData);
 };
-export const readTemplatePrompt = (templateName) => {
-    const templatePath = getTemplateFilePath(templateName);
-    const templateData = fs.readFileSync(templatePath, "utf-8");
+const readPromptTemplateFile = (promptTemplateName) => {
+    const promptTemplatePath = getPromptTemplateFilePath(promptTemplateName);
+    const promptTemplateData = fs.readFileSync(promptTemplatePath, "utf-8");
     // NOTE: We don't want to schema parse the template here to eliminate default values.
-    const template = JSON.parse(templateData);
-    const script = (() => {
-        if (template.scriptName) {
-            const scriptData = readScriptTemplateFile(template.scriptName);
-            return { ...scriptData, ...(template.presentationStyle ?? {}) };
-        }
-        return undefined;
-    })();
-    const prompt = MulmoScriptTemplateMethods.getSystemPrompt(template, script);
+    const promptTemplate = JSON.parse(promptTemplateData);
+    return promptTemplate;
+};
+const mulmoScriptTemplate2Script = (scriptTemplate) => {
+    if (scriptTemplate.scriptName) {
+        const scriptData = readScriptTemplateFile(scriptTemplate.scriptName);
+        return { ...scriptData, ...(scriptTemplate.presentationStyle ?? {}) };
+    }
+    return undefined;
+};
+export const getScriptFromPromptTemplate = (promptTemplateName) => {
+    const promptTemplate = readPromptTemplateFile(promptTemplateName);
+    return mulmoScriptTemplate2Script(promptTemplate);
+};
+export const readTemplatePrompt = (promptTemplateName) => {
+    const promptTemplate = readPromptTemplateFile(promptTemplateName);
+    const script = mulmoScriptTemplate2Script(promptTemplate);
+    const prompt = MulmoScriptTemplateMethods.getSystemPrompt(promptTemplate, script);
     return prompt;
 };
+// TODO: MulmoScriptTemplateFileは、実際はpromptTempate
+// TODO: remove it after update app
 export const getAvailableTemplates = () => {
-    const templatesDir = path.resolve(npmRoot, "./assets/templates");
+    return getAvailablePromptTemplates();
+};
+export const getAvailablePromptTemplates = () => {
+    return getPromptTemplates(promptTemplateDirName, mulmoScriptTemplateSchema);
+};
+export const getAvailableScriptTemplates = () => {
+    return getPromptTemplates(scriptTemplateDirName, mulmoPresentationStyleSchema);
+};
+export const getPromptTemplates = (dirPath, schema) => {
+    const templatesDir = path.resolve(npmRoot, dirPath);
     if (!fs.existsSync(templatesDir)) {
         return [];
     }
     const files = fs.readdirSync(templatesDir);
     return files.map((file) => {
-        const template = JSON.parse(fs.readFileSync(path.resolve(templatesDir, file), "utf-8"));
+        const promptTemplate = JSON.parse(fs.readFileSync(path.resolve(templatesDir, file), "utf-8"));
         return {
-            ...mulmoScriptTemplateSchema.parse(template),
+            ...schema.parse(promptTemplate),
             filename: file.replace(/\.json$/, ""),
         };
     });