mulmocast 1.1.0 → 1.1.2

package/lib/types/schema.js

@@ -233,6 +233,10 @@ export const mulmoSoundEffectParamsSchema = z.object({
     provider: text2SoundEffectProviderSchema.optional(),
     model: z.string().optional(), // default: provider specific
 });
+export const mulmoLipSyncParamsSchema = z.object({
+    provider: z.string().optional(), // lip sync provider
+    model: z.string().optional(), // default: provider specific
+});
 export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.optional(),
@@ -253,6 +257,7 @@ export const mulmoBeatSchema = z
     })
         .optional(),
     soundEffectParams: mulmoSoundEffectParamsSchema.optional(),
+    lipSyncParams: mulmoLipSyncParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
     speechOptions: speechOptionsSchema.optional(),
     textSlideParams: textSlideParamsSchema.optional(),
@@ -262,6 +267,7 @@ export const mulmoBeatSchema = z
     moviePrompt: z.string().optional(),
     soundEffectPrompt: z.string().optional(),
     htmlPrompt: htmlPromptParamsSchema.optional(),
+    enableLipSync: z.boolean().optional().describe("Enable lip sync generation for this beat"),
 })
     .strict();
 export const mulmoCanvasDimensionSchema = z
@@ -332,6 +338,7 @@ export const mulmoPresentationStyleSchema = z.object({
     soundEffectParams: mulmoSoundEffectParamsSchema.optional().default({
         provider: defaultProviders.soundEffect,
     }),
+    lipSyncParams: mulmoLipSyncParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema
         .extend({
         provider: text2HtmlImageProviderSchema,
@@ -381,6 +388,7 @@ export const mulmoStudioBeatSchema = z
     imageFile: z.string().optional(), // path to the image
     movieFile: z.string().optional(), // path to the movie file
     soundEffectFile: z.string().optional(), // path to the sound effect file
+    lipSyncFile: z.string().optional(), // path to the lip sync file
     captionFile: z.string().optional(), // path to the caption image
 })
     .strict();
@@ -406,6 +414,7 @@ export const mulmoSessionStateSchema = z.object({
         html: z.record(z.number().int(), z.boolean()),
         imageReference: z.record(z.number().int(), z.boolean()),
         soundEffect: z.record(z.number().int(), z.boolean()),
+        lipSync: z.record(z.number().int(), z.boolean()),
     }),
 });
 export const mulmoStudioSchema = z
@@ -415,7 +424,7 @@ export const mulmoStudioSchema = z
     beats: z.array(mulmoStudioBeatSchema).min(1),
 })
     .strict();
-export const mulmoScriptTemplateSchema = z
+export const mulmoPromptTemplateSchema = z
     .object({
     title: z.string(),
     description: z.string(),
@@ -424,7 +433,7 @@ export const mulmoScriptTemplateSchema = z
     presentationStyle: mulmoPresentationStyleSchema.optional(),
 })
     .strict();
-export const mulmoScriptTemplateFileSchema = mulmoScriptTemplateSchema.extend({
+export const mulmoPromptTemplateFileSchema = mulmoPromptTemplateSchema.extend({
     filename: z.string(),
 });
 export const mulmoStoryboardSceneSchema = z

package/lib/types/type.d.ts

@@ -1,4 +1,4 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import type { LLM } from "../utils/provider2agent.js";
 import { z } from "zod";
@@ -24,8 +24,8 @@ export type MulmoStoryboard = z.infer<typeof mulmoStoryboardSchema>;
 export type MulmoStudioBeat = z.infer<typeof mulmoStudioBeatSchema>;
 export type MulmoMediaSource = z.infer<typeof mediaSourceSchema>;
 export type MulmoStudio = z.infer<typeof mulmoStudioSchema>;
-export type MulmoScriptTemplate = z.infer<typeof mulmoScriptTemplateSchema>;
-export type MulmoScriptTemplateFile = z.infer<typeof mulmoScriptTemplateFileSchema>;
+export type MulmoPromptTemplate = z.infer<typeof mulmoPromptTemplateSchema>;
+export type MulmoPromptTemplateFile = z.infer<typeof mulmoPromptTemplateFileSchema>;
 export type MulmoStudioMultiLingual = z.infer<typeof mulmoStudioMultiLingualSchema>;
 export type MulmoStudioMultiLingualData = z.infer<typeof mulmoStudioMultiLingualDataSchema>;
 export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
@@ -91,7 +91,7 @@ export type Text2HtmlAgentInfo = {
 export type BeatMediaType = "movie" | "image";
 export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];
 export type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
-export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference" | "soundEffect";
+export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference" | "soundEffect" | "lipSync";
 export type SessionProgressEvent = {
     kind: "session";
     sessionType: SessionType;

package/lib/utils/context.d.ts

@@ -17,6 +17,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             imageFile?: string | undefined;
             movieFile?: string | undefined;
             soundEffectFile?: string | undefined;
+            lipSyncFile?: string | undefined;
             captionFile?: string | undefined;
         }[];
         script: {
@@ -209,12 +210,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     type: "voice_over";
                     startAt?: number | undefined;
                 } | undefined;
-                duration?: number | undefined;
-                speechOptions?: {
-                    speed?: number | undefined;
-                    instruction?: string | undefined;
-                } | undefined;
-                id?: string | undefined;
                 audio?: {
                     type: "audio";
                     source: {
@@ -234,6 +229,12 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     type: "midi";
                     source: string;
                 } | undefined;
+                duration?: number | undefined;
+                speechOptions?: {
+                    speed?: number | undefined;
+                    instruction?: string | undefined;
+                } | undefined;
+                id?: string | undefined;
                 imagePrompt?: string | undefined;
                 speaker?: string | undefined;
                 description?: string | undefined;
@@ -278,6 +279,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     provider?: string | undefined;
                     model?: string | undefined;
                 } | undefined;
+                lipSyncParams?: {
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                } | undefined;
                 htmlImageParams?: {
                     model?: string | undefined;
                 } | undefined;
@@ -297,10 +302,15 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     images?: Record<string, any> | undefined;
                     systemPrompt?: string | undefined;
                 } | undefined;
+                enableLipSync?: boolean | undefined;
             }[];
             lang?: string | undefined;
             title?: string | undefined;
             description?: string | undefined;
+            lipSyncParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
             htmlImageParams?: {
                 provider: string;
                 model?: string | undefined;
@@ -344,6 +354,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             html: {};
             imageReference: {};
             soundEffect: {};
+            lipSync: {};
         };
     };
     presentationStyle: {
@@ -430,6 +441,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 model?: string | undefined;
             }>;
         };
+        lipSyncParams?: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/context.js

@@ -59,6 +59,7 @@ const initSessionState = () => {
             html: {},
             imageReference: {},
             soundEffect: {},
+            lipSync: {},
         },
     };
 };

package/lib/utils/file.d.ts

@@ -1,4 +1,4 @@
-import type { MulmoScript, MulmoScriptTemplateFile, MulmoStudioContext } from "../types/index.js";
+import type { MulmoScript, MulmoPromptTemplateFile, MulmoStudioContext } from "../types/index.js";
 import { PDFMode } from "../types/index.js";
 import { ZodSchema } from "zod";
 export declare const updateNpmRoot: (_npmRoot: string) => void;
@@ -28,20 +28,20 @@ export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: n
 export declare const getBeatMoviePaths: (context: MulmoStudioContext, index: number) => {
     movieFile: string;
     soundEffectFile: string;
+    lipSyncFile: string;
 };
 export declare const getReferenceImagePath: (context: MulmoStudioContext, key: string, extension: string) => string;
 export declare const getCaptionImagePath: (context: MulmoStudioContext, index: number) => string;
 export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;
-export declare const getTemplateFilePath: (templateName: string) => string;
+export declare const getPromptTemplateFilePath: (promptTemplateName: string) => string;
 export declare const mkdir: (dirPath: string) => void;
 export declare const silent60secPath: () => string;
 export declare const defaultBGMPath: () => string;
 export declare const getHTMLFile: (filename: string) => string;
 export declare const getBaseDirPath: (basedir?: string) => string;
 export declare const getFullPath: (baseDirPath: string | undefined, file: string) => string;
-export declare const readScriptTemplateFile: (scriptName: string) => any;
-export declare const readTemplatePrompt: (templateName: string) => string;
-export declare const getAvailableTemplates: () => MulmoScriptTemplateFile[];
+export declare const getAvailablePromptTemplates: (skipValidation?: boolean) => MulmoPromptTemplateFile[];
+export declare const getAvailableScriptTemplates: () => MulmoScript[];
 export declare const writingMessage: (filePath: string) => void;
 export declare const readAndParseJson: <S extends ZodSchema<any>>(filePath: string, schema: S) => ReturnType<S["parse"]>;
 export declare const generateTimestampedFileName: (prefix: string) => string;

package/lib/utils/file.js

@@ -3,8 +3,10 @@ import path from "path";
 import { parse as yamlParse } from "yaml";
 import { fileURLToPath } from "url";
 import { GraphAILogger } from "graphai";
-import { MulmoScriptTemplateMethods, MulmoStudioContextMethods } from "../methods/index.js";
-import { mulmoScriptTemplateSchema } from "../types/schema.js";
+import { MulmoStudioContextMethods } from "../methods/index.js";
+import { mulmoPromptTemplateSchema } from "../types/schema.js";
+const promptTemplateDirName = "./assets/templates";
+const scriptTemplateDirName = "./scripts/templates";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 let npmRoot = path.resolve(__dirname, "../../");
@@ -91,7 +93,11 @@ export const getBeatMoviePaths = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
     const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
     const filename = beat?.id ? `${beat.id}` : `${index}`;
-    return { movieFile: `${imageProjectDirPath}/${filename}.mov`, soundEffectFile: `${imageProjectDirPath}/${filename}_sound.mov` };
+    return {
+        movieFile: `${imageProjectDirPath}/${filename}.mov`,
+        soundEffectFile: `${imageProjectDirPath}/${filename}_sound.mov`,
+        lipSyncFile: `${imageProjectDirPath}/${filename}_lipsync.mov`,
+    };
 };
 export const getReferenceImagePath = (context, key, extension) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
@@ -108,8 +114,8 @@ export const getOutputPdfFilePath = (outDirPath, fileName, pdfMode, lang) => {
     }
     return path.resolve(outDirPath, `${fileName}_${pdfMode}.pdf`);
 };
-export const getTemplateFilePath = (templateName) => {
-    return path.resolve(npmRoot, "./assets/templates/" + templateName + ".json");
+export const getPromptTemplateFilePath = (promptTemplateName) => {
+    return path.resolve(npmRoot, promptTemplateDirName, promptTemplateName + ".json");
 };
 export const mkdir = (dirPath) => {
     if (!fs.existsSync(dirPath)) {
@@ -144,41 +150,38 @@ export const getFullPath = (baseDirPath, file) => {
     }
     return path.resolve(file);
 };
-export const readScriptTemplateFile = (scriptName) => {
-    const scriptPath = path.resolve(npmRoot, "./scripts/templates", scriptName);
-    const scriptData = fs.readFileSync(scriptPath, "utf-8");
-    // NOTE: We don't want to schema parse the script here to eliminate default values.
-    return JSON.parse(scriptData);
-};
-export const readTemplatePrompt = (templateName) => {
-    const templatePath = getTemplateFilePath(templateName);
-    const templateData = fs.readFileSync(templatePath, "utf-8");
-    // NOTE: We don't want to schema parse the template here to eliminate default values.
-    const template = JSON.parse(templateData);
-    const script = (() => {
-        if (template.scriptName) {
-            const scriptData = readScriptTemplateFile(template.scriptName);
-            return { ...scriptData, ...(template.presentationStyle ?? {}) };
-        }
-        return undefined;
-    })();
-    const prompt = MulmoScriptTemplateMethods.getSystemPrompt(template, script);
-    return prompt;
-};
-export const getAvailableTemplates = () => {
-    const templatesDir = path.resolve(npmRoot, "./assets/templates");
+// templates
+const getPromptTemplates = (dirPath, schema) => {
+    const templatesDir = path.resolve(npmRoot, dirPath);
     if (!fs.existsSync(templatesDir)) {
         return [];
     }
     const files = fs.readdirSync(templatesDir);
     return files.map((file) => {
-        const template = JSON.parse(fs.readFileSync(path.resolve(templatesDir, file), "utf-8"));
-        return {
-            ...mulmoScriptTemplateSchema.parse(template),
-            filename: file.replace(/\.json$/, ""),
-        };
+        try {
+            const promptTemplate = JSON.parse(fs.readFileSync(path.resolve(templatesDir, file), "utf-8"));
+            return {
+                ...(schema ? schema.parse(promptTemplate) : promptTemplate),
+                filename: file.replace(/\.json$/, ""),
+            };
+        }
+        catch (e) {
+            GraphAILogger.info("file: " + file);
+            GraphAILogger.info(e);
+            return {};
+        }
     });
 };
+export const getAvailablePromptTemplates = (skipValidation) => {
+    if (skipValidation) {
+        return getPromptTemplates(promptTemplateDirName, null);
+    }
+    return getPromptTemplates(promptTemplateDirName, mulmoPromptTemplateSchema);
+};
+export const getAvailableScriptTemplates = () => {
+    return getPromptTemplates(scriptTemplateDirName, null);
+};
+// end of template
 export const writingMessage = (filePath) => {
     GraphAILogger.debug(`writing: ${filePath}`);
 };

package/lib/utils/inquirer.js

@@ -1,7 +1,7 @@
 import inquirer from "inquirer";
-import { getAvailableTemplates } from "./file.js";
+import { getAvailablePromptTemplates } from "./file.js";
 export const selectTemplate = async () => {
-    const availableTemplates = getAvailableTemplates();
+    const availableTemplates = getAvailablePromptTemplates();
     const answers = await inquirer.prompt([
         {
             type: "list",

package/lib/utils/preprocess.d.ts

@@ -12,6 +12,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         imageFile?: string | undefined;
         movieFile?: string | undefined;
         soundEffectFile?: string | undefined;
+        lipSyncFile?: string | undefined;
         captionFile?: string | undefined;
     }[];
     script: {
@@ -204,12 +205,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "voice_over";
                 startAt?: number | undefined;
             } | undefined;
-            duration?: number | undefined;
-            speechOptions?: {
-                speed?: number | undefined;
-                instruction?: string | undefined;
-            } | undefined;
-            id?: string | undefined;
             audio?: {
                 type: "audio";
                 source: {
@@ -229,6 +224,12 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "midi";
                 source: string;
             } | undefined;
+            duration?: number | undefined;
+            speechOptions?: {
+                speed?: number | undefined;
+                instruction?: string | undefined;
+            } | undefined;
+            id?: string | undefined;
             imagePrompt?: string | undefined;
             speaker?: string | undefined;
             description?: string | undefined;
@@ -273,6 +274,10 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 provider?: string | undefined;
                 model?: string | undefined;
             } | undefined;
+            lipSyncParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
             htmlImageParams?: {
                 model?: string | undefined;
             } | undefined;
@@ -292,10 +297,15 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 images?: Record<string, any> | undefined;
                 systemPrompt?: string | undefined;
             } | undefined;
+            enableLipSync?: boolean | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
         description?: string | undefined;
+        lipSyncParams?: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/prompt.d.ts

@@ -1,8 +1,7 @@
-import { MulmoBeat, MulmoScript, MulmoScriptTemplate, MulmoStoryboard, MulmoCanvasDimension } from "../types/index.js";
+import { MulmoBeat, MulmoScript, MulmoStoryboard, MulmoCanvasDimension } from "../types/index.js";
 export declare const imagePrompt: (beat: MulmoBeat, style?: string) => string;
 export declare const graphDataScriptFromUrlPrompt: (sourceTextInput: string) => string;
 export declare const graphDataScriptGeneratePrompt: (scene: string) => string;
-export declare const getMulmoScriptTemplateSystemPrompt: (template: MulmoScriptTemplate, script?: MulmoScript) => string;
 export declare const interactiveClarificationPrompt = "If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.";
 export declare const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
 export declare const translateSystemPrompt = "Please translate the given text into the language specified in language (in locale format, like en, ja, fr, ch).";

package/lib/utils/prompt.js

@@ -1,5 +1,3 @@
-import { mulmoScriptSchema } from "../types/schema.js";
-import { zodToJsonSchema } from "zod-to-json-schema";
 export const imagePrompt = (beat, style) => {
     return (beat.imagePrompt || `generate image appropriate for the text. text: ${beat.text}`) + "\n" + (style || "");
 };
@@ -10,18 +8,6 @@ export const graphDataScriptFromUrlPrompt = (sourceTextInput) => {
 export const graphDataScriptGeneratePrompt = (scene) => {
     return `Please generate a script from the following scenes: ${scene}`;
 };
-export const getMulmoScriptTemplateSystemPrompt = (template, script) => {
-    // script is provided, use it as a script template
-    if (script) {
-        return `${template.systemPrompt}\n\`\`\`JSON\n${JSON.stringify(script)}\n\`\`\``;
-    }
-    // script is not provided, use the default schema
-    const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
-        strictUnions: true,
-    });
-    const specificOutputPrompt = `The output should follow the JSON schema specified below. Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`;
-    return `${template.systemPrompt}\n\n${specificOutputPrompt}\n\n\`\`\`JSON\n${JSON.stringify(defaultSchema)}\n\`\`\``;
-};
 export const interactiveClarificationPrompt = `If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.`;
 export const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
 export const translateSystemPrompt = "Please translate the given text into the language specified in language (in locale format, like en, ja, fr, ch).";

package/lib/utils/provider2agent.d.ts

@@ -61,6 +61,18 @@ export declare const provider2SoundEffectAgent: {
         }>;
     };
 };
+export declare const provider2LipSyncAgent: {
+    replicate: {
+        agentName: string;
+        defaultModel: ReplicateModel;
+        models: ReplicateModel[];
+        modelParams: Record<ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+            video: string;
+            audio: string;
+        }>;
+    };
+};
 export declare const provider2LLMAgent: {
     readonly openai: {
         readonly agentName: "openAIAgent";
@@ -90,6 +102,7 @@ export declare const defaultProviders: {
     text2Html: keyof typeof provider2LLMAgent;
     llm: keyof typeof provider2LLMAgent;
     soundEffect: keyof typeof provider2SoundEffectAgent;
+    lipSync: keyof typeof provider2LipSyncAgent;
 };
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;

package/lib/utils/provider2agent.js

@@ -130,6 +130,37 @@ export const provider2SoundEffectAgent = {
         },
     },
 };
+export const provider2LipSyncAgent = {
+    replicate: {
+        agentName: "lipSyncReplicateAgent",
+        defaultModel: "bytedance/latentsync",
+        models: ["bytedance/latentsync", "tmappdev/lipsync"],
+        modelParams: {
+            "bytedance/latentsync": {
+                identifier: "bytedance/latentsync:637ce1919f807ca20da3a448ddc2743535d2853649574cd52a933120e9b9e293",
+                video: "video",
+                audio: "audio",
+            },
+            "tmappdev/lipsync": {
+                identifier: "tmappdev/lipsync:c54ce2fe673ea59b857b91250b3d71a2cd304a78f2370687632805c8405fbf4c",
+                video: "video_input",
+                audio: "audio_input",
+            },
+            /* NOTE: This model does not work with large base64 urls.
+            "sync/lipsync-2": {
+              video: "video",
+              audio: "audio",
+            },
+            */
+            /* NOTE: This model does not work well for some unknown reason.
+            "kwaivgi/kling-lip-sync": {
+              video: "video_url",
+              audio: "audio_file",
+            },
+            */
+        },
+    },
+};
 // : Record<LLM, { agent: string; defaultModel: string; max_tokens: number }>
 export const provider2LLMAgent = {
     openai: {
@@ -160,6 +191,7 @@ export const defaultProviders = {
     text2Html: "openai",
     llm: "openai",
     soundEffect: "replicate",
+    lipSync: "replicate",
 };
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic"];

package/lib/utils/system_prompt.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/utils/system_prompt.js

@@ -0,0 +1 @@
+export {};

package/lib/utils/templates.d.ts

@@ -0,0 +1,3 @@
+import type { MulmoScript } from "../types/type.js";
+export declare const readScriptTemplateFile: (scriptTemplateFileName: string) => MulmoScript;
+export declare const readTemplatePrompt: (promptTemplateFileName: string) => string;

package/lib/utils/templates.js

@@ -0,0 +1,46 @@
+import { mulmoScriptSchema } from "../types/schema.js";
+import { promptTemplates, scriptTemplates } from "../data/index.js";
+import { zodToJsonSchema } from "zod-to-json-schema";
+// script and prompt template
+export const readScriptTemplateFile = (scriptTemplateFileName) => {
+    // NOTE: We don't want to schema parse the script here to eliminate default values.
+    const scriptTemplate = scriptTemplates.find((template) => template.filename === scriptTemplateFileName);
+    if (!scriptTemplate) {
+        throw new Error(`Script template not found: ${scriptTemplateFileName}`);
+    }
+    const { filename: __, ...retValue } = scriptTemplate;
+    return retValue;
+};
+const readPromptTemplateFile = (promptTemplateFileName) => {
+    // NOTE: We don't want to schema parse the template here to eliminate default values.
+    const promptTemplate = promptTemplates.find((template) => template.filename === promptTemplateFileName);
+    if (!promptTemplate) {
+        throw new Error(`Prompt template not found: ${promptTemplateFileName}`);
+    }
+    return promptTemplate;
+};
+const mulmoScriptTemplate2Script = (scriptTemplate) => {
+    if (scriptTemplate.scriptName) {
+        const scriptTemplateData = readScriptTemplateFile(scriptTemplate.scriptName);
+        return { ...scriptTemplateData, ...(scriptTemplate.presentationStyle ?? {}) };
+    }
+    return undefined;
+};
+export const readTemplatePrompt = (promptTemplateFileName) => {
+    const promptTemplate = readPromptTemplateFile(promptTemplateFileName);
+    const script = mulmoScriptTemplate2Script(promptTemplate);
+    const prompt = getMulmoScriptTemplateSystemPrompt(promptTemplate, script);
+    return prompt;
+};
+const getMulmoScriptTemplateSystemPrompt = (template, script) => {
+    // script is provided, use it as a script template
+    if (script) {
+        return `${template.systemPrompt}\n\`\`\`JSON\n${JSON.stringify(script)}\n\`\`\``;
+    }
+    // script is not provided, use the default schema
+    const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
+        strictUnions: true,
+    });
+    const specificOutputPrompt = `The output should follow the JSON schema specified below. Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`;
+    return `${template.systemPrompt}\n\n${specificOutputPrompt}\n\n\`\`\`JSON\n${JSON.stringify(defaultSchema)}\n\`\`\``;
+};

package/lib/utils/utils.js

@@ -76,6 +76,9 @@ export const settings2GraphAIConfig = (settings, env) => {
         soundEffectReplicateAgent: {
             apiKey: getKey("SOUND_EFFECT", "REPLICATE_API_TOKEN"),
         },
+        lipSyncReplicateAgent: {
+            apiKey: getKey("LIPSYNC", "REPLICATE_API_TOKEN"),
+        },
         // TODO
         // browserlessAgent
         // ttsGoogleAgent

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -12,6 +12,9 @@
     "./browser": {
       "types": "./lib/index.browser.d.ts",
       "default": "./lib/index.browser.js"
+    },
+    "./data": {
+      "default": "./lib/data/index.js"
     }
   },
   "bin": {
@@ -48,6 +51,7 @@
     "latest": "yarn upgrade-interactive  --latest",
     "format": "prettier --write '{src,scripts,assets/templates,assets/styles,draft,ideason,scripts_mag2,proto,test,graphai,output,docs/scripts}/**/*.{ts,json,yaml}'",
     "deep_research": "npx tsx ./src/tools/deep_research.ts",
+    "template": "npx tsx batch/template2tsobject.ts && yarn run format",
     "fake_data": "npx tsx test/fake/sample.ts",
     "mcp_server": "npx tsx ./src/mcp/server.ts"
   },
@@ -73,14 +77,14 @@
     "@tavily/core": "^0.5.9",
     "canvas": "^3.1.2",
     "clipboardy": "^4.0.0",
-    "dotenv": "^17.2.0",
+    "dotenv": "^17.2.1",
     "fluent-ffmpeg": "^2.1.3",
     "google-auth-library": "^10.1.0",
     "graphai": "^2.0.13",
     "inquirer": "^12.7.0",
     "marked": "^16.1.1",
     "ora": "^8.2.0",
-    "puppeteer": "^24.14.0",
+    "puppeteer": "^24.15.0",
     "replicate": "^1.0.1",
     "yaml": "^2.8.0",
     "yargs": "^18.0.0",
@@ -93,7 +97,7 @@
     "@receptron/test_utils": "^2.0.0",
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
-    "eslint": "^9.31.0",
+    "eslint": "^9.32.0",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-prettier": "^5.5.3",
     "eslint-plugin-sonarjs": "^3.0.4",

package/scripts/templates/business.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "Sample Title",

package/scripts/templates/children_book.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "桃太郎",

package/scripts/templates/coding.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "Sample Title",