mulmocast 1.1.0 → 1.1.1

package/lib/types/schema.js

@@ -233,6 +233,10 @@ export const mulmoSoundEffectParamsSchema = z.object({
     provider: text2SoundEffectProviderSchema.optional(),
     model: z.string().optional(), // default: provider specific
 });
+export const mulmoLipSyncParamsSchema = z.object({
+    provider: z.string().optional(), // lip sync provider
+    model: z.string().optional(), // default: provider specific
+});
 export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.optional(),
@@ -253,6 +257,7 @@ export const mulmoBeatSchema = z
     })
         .optional(),
     soundEffectParams: mulmoSoundEffectParamsSchema.optional(),
+    lipSyncParams: mulmoLipSyncParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
     speechOptions: speechOptionsSchema.optional(),
     textSlideParams: textSlideParamsSchema.optional(),
@@ -262,6 +267,7 @@ export const mulmoBeatSchema = z
     moviePrompt: z.string().optional(),
     soundEffectPrompt: z.string().optional(),
     htmlPrompt: htmlPromptParamsSchema.optional(),
+    enableLipSync: z.boolean().optional().describe("Enable lip sync generation for this beat"),
 })
     .strict();
 export const mulmoCanvasDimensionSchema = z
@@ -332,6 +338,7 @@ export const mulmoPresentationStyleSchema = z.object({
     soundEffectParams: mulmoSoundEffectParamsSchema.optional().default({
         provider: defaultProviders.soundEffect,
     }),
+    lipSyncParams: mulmoLipSyncParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema
         .extend({
         provider: text2HtmlImageProviderSchema,
@@ -381,6 +388,7 @@ export const mulmoStudioBeatSchema = z
     imageFile: z.string().optional(), // path to the image
     movieFile: z.string().optional(), // path to the movie file
     soundEffectFile: z.string().optional(), // path to the sound effect file
+    lipSyncFile: z.string().optional(), // path to the lip sync file
     captionFile: z.string().optional(), // path to the caption image
 })
     .strict();
@@ -406,6 +414,7 @@ export const mulmoSessionStateSchema = z.object({
         html: z.record(z.number().int(), z.boolean()),
         imageReference: z.record(z.number().int(), z.boolean()),
         soundEffect: z.record(z.number().int(), z.boolean()),
+        lipSync: z.record(z.number().int(), z.boolean()),
     }),
 });
 export const mulmoStudioSchema = z

package/lib/types/type.d.ts

@@ -91,7 +91,7 @@ export type Text2HtmlAgentInfo = {
 export type BeatMediaType = "movie" | "image";
 export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];
 export type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
-export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference" | "soundEffect";
+export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference" | "soundEffect" | "lipSync";
 export type SessionProgressEvent = {
     kind: "session";
     sessionType: SessionType;

package/lib/utils/context.d.ts

@@ -17,6 +17,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             imageFile?: string | undefined;
             movieFile?: string | undefined;
             soundEffectFile?: string | undefined;
+            lipSyncFile?: string | undefined;
             captionFile?: string | undefined;
         }[];
         script: {
@@ -278,6 +279,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     provider?: string | undefined;
                     model?: string | undefined;
                 } | undefined;
+                lipSyncParams?: {
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                } | undefined;
                 htmlImageParams?: {
                     model?: string | undefined;
                 } | undefined;
@@ -297,10 +302,15 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     images?: Record<string, any> | undefined;
                     systemPrompt?: string | undefined;
                 } | undefined;
+                enableLipSync?: boolean | undefined;
             }[];
             lang?: string | undefined;
             title?: string | undefined;
             description?: string | undefined;
+            lipSyncParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
             htmlImageParams?: {
                 provider: string;
                 model?: string | undefined;
@@ -344,6 +354,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             html: {};
             imageReference: {};
             soundEffect: {};
+            lipSync: {};
         };
     };
     presentationStyle: {
@@ -430,6 +441,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 model?: string | undefined;
             }>;
         };
+        lipSyncParams?: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/context.js

@@ -59,6 +59,7 @@ const initSessionState = () => {
             html: {},
             imageReference: {},
             soundEffect: {},
+            lipSync: {},
         },
     };
 };

package/lib/utils/file.d.ts

@@ -1,6 +1,6 @@
 import type { MulmoScript, MulmoScriptTemplateFile, MulmoStudioContext } from "../types/index.js";
 import { PDFMode } from "../types/index.js";
-import { ZodSchema } from "zod";
+import { ZodSchema, ZodType } from "zod";
 export declare const updateNpmRoot: (_npmRoot: string) => void;
 export declare function readMulmoScriptFile<T = MulmoScript>(path: string, errorMessage: string): {
     mulmoData: T;
@@ -28,11 +28,12 @@ export declare const getBeatPngImagePath: (context: MulmoStudioContext, index: n
 export declare const getBeatMoviePaths: (context: MulmoStudioContext, index: number) => {
     movieFile: string;
     soundEffectFile: string;
+    lipSyncFile: string;
 };
 export declare const getReferenceImagePath: (context: MulmoStudioContext, key: string, extension: string) => string;
 export declare const getCaptionImagePath: (context: MulmoStudioContext, index: number) => string;
 export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;
-export declare const getTemplateFilePath: (templateName: string) => string;
+export declare const getPromptTemplateFilePath: (promptTemplateName: string) => string;
 export declare const mkdir: (dirPath: string) => void;
 export declare const silent60secPath: () => string;
 export declare const defaultBGMPath: () => string;
@@ -40,8 +41,12 @@ export declare const getHTMLFile: (filename: string) => string;
 export declare const getBaseDirPath: (basedir?: string) => string;
 export declare const getFullPath: (baseDirPath: string | undefined, file: string) => string;
 export declare const readScriptTemplateFile: (scriptName: string) => any;
-export declare const readTemplatePrompt: (templateName: string) => string;
+export declare const getScriptFromPromptTemplate: (promptTemplateName: string) => any;
+export declare const readTemplatePrompt: (promptTemplateName: string) => string;
 export declare const getAvailableTemplates: () => MulmoScriptTemplateFile[];
+export declare const getAvailablePromptTemplates: () => MulmoScriptTemplateFile[];
+export declare const getAvailableScriptTemplates: () => MulmoScriptTemplateFile[];
+export declare const getPromptTemplates: <T>(dirPath: string, schema: ZodType) => T[];
 export declare const writingMessage: (filePath: string) => void;
 export declare const readAndParseJson: <S extends ZodSchema<any>>(filePath: string, schema: S) => ReturnType<S["parse"]>;
 export declare const generateTimestampedFileName: (prefix: string) => string;

package/lib/utils/file.js

@@ -4,7 +4,9 @@ import { parse as yamlParse } from "yaml";
 import { fileURLToPath } from "url";
 import { GraphAILogger } from "graphai";
 import { MulmoScriptTemplateMethods, MulmoStudioContextMethods } from "../methods/index.js";
-import { mulmoScriptTemplateSchema } from "../types/schema.js";
+import { mulmoScriptTemplateSchema, mulmoPresentationStyleSchema } from "../types/schema.js";
+const promptTemplateDirName = "./assets/templates";
+const scriptTemplateDirName = "./scripts/templates";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 let npmRoot = path.resolve(__dirname, "../../");
@@ -91,7 +93,11 @@ export const getBeatMoviePaths = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
     const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
     const filename = beat?.id ? `${beat.id}` : `${index}`;
-    return { movieFile: `${imageProjectDirPath}/${filename}.mov`, soundEffectFile: `${imageProjectDirPath}/${filename}_sound.mov` };
+    return {
+        movieFile: `${imageProjectDirPath}/${filename}.mov`,
+        soundEffectFile: `${imageProjectDirPath}/${filename}_sound.mov`,
+        lipSyncFile: `${imageProjectDirPath}/${filename}_lipsync.mov`,
+    };
 };
 export const getReferenceImagePath = (context, key, extension) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
@@ -108,8 +114,8 @@ export const getOutputPdfFilePath = (outDirPath, fileName, pdfMode, lang) => {
     }
     return path.resolve(outDirPath, `${fileName}_${pdfMode}.pdf`);
 };
-export const getTemplateFilePath = (templateName) => {
-    return path.resolve(npmRoot, "./assets/templates/" + templateName + ".json");
+export const getPromptTemplateFilePath = (promptTemplateName) => {
+    return path.resolve(npmRoot, promptTemplateDirName, promptTemplateName + ".json");
 };
 export const mkdir = (dirPath) => {
     if (!fs.existsSync(dirPath)) {
@@ -145,36 +151,56 @@ export const getFullPath = (baseDirPath, file) => {
     return path.resolve(file);
 };
 export const readScriptTemplateFile = (scriptName) => {
-    const scriptPath = path.resolve(npmRoot, "./scripts/templates", scriptName);
+    const scriptPath = path.resolve(npmRoot, scriptTemplateDirName, scriptName);
     const scriptData = fs.readFileSync(scriptPath, "utf-8");
     // NOTE: We don't want to schema parse the script here to eliminate default values.
     return JSON.parse(scriptData);
 };
-export const readTemplatePrompt = (templateName) => {
-    const templatePath = getTemplateFilePath(templateName);
-    const templateData = fs.readFileSync(templatePath, "utf-8");
+const readPromptTemplateFile = (promptTemplateName) => {
+    const promptTemplatePath = getPromptTemplateFilePath(promptTemplateName);
+    const promptTemplateData = fs.readFileSync(promptTemplatePath, "utf-8");
     // NOTE: We don't want to schema parse the template here to eliminate default values.
-    const template = JSON.parse(templateData);
-    const script = (() => {
-        if (template.scriptName) {
-            const scriptData = readScriptTemplateFile(template.scriptName);
-            return { ...scriptData, ...(template.presentationStyle ?? {}) };
-        }
-        return undefined;
-    })();
-    const prompt = MulmoScriptTemplateMethods.getSystemPrompt(template, script);
+    const promptTemplate = JSON.parse(promptTemplateData);
+    return promptTemplate;
+};
+const mulmoScriptTemplate2Script = (scriptTemplate) => {
+    if (scriptTemplate.scriptName) {
+        const scriptData = readScriptTemplateFile(scriptTemplate.scriptName);
+        return { ...scriptData, ...(scriptTemplate.presentationStyle ?? {}) };
+    }
+    return undefined;
+};
+export const getScriptFromPromptTemplate = (promptTemplateName) => {
+    const promptTemplate = readPromptTemplateFile(promptTemplateName);
+    return mulmoScriptTemplate2Script(promptTemplate);
+};
+export const readTemplatePrompt = (promptTemplateName) => {
+    const promptTemplate = readPromptTemplateFile(promptTemplateName);
+    const script = mulmoScriptTemplate2Script(promptTemplate);
+    const prompt = MulmoScriptTemplateMethods.getSystemPrompt(promptTemplate, script);
     return prompt;
 };
+// TODO: MulmoScriptTemplateFileは、実際はpromptTempate
+// TODO: remove it after update app
 export const getAvailableTemplates = () => {
-    const templatesDir = path.resolve(npmRoot, "./assets/templates");
+    return getAvailablePromptTemplates();
+};
+export const getAvailablePromptTemplates = () => {
+    return getPromptTemplates(promptTemplateDirName, mulmoScriptTemplateSchema);
+};
+export const getAvailableScriptTemplates = () => {
+    return getPromptTemplates(scriptTemplateDirName, mulmoPresentationStyleSchema);
+};
+export const getPromptTemplates = (dirPath, schema) => {
+    const templatesDir = path.resolve(npmRoot, dirPath);
     if (!fs.existsSync(templatesDir)) {
         return [];
     }
     const files = fs.readdirSync(templatesDir);
     return files.map((file) => {
-        const template = JSON.parse(fs.readFileSync(path.resolve(templatesDir, file), "utf-8"));
+        const promptTemplate = JSON.parse(fs.readFileSync(path.resolve(templatesDir, file), "utf-8"));
         return {
-            ...mulmoScriptTemplateSchema.parse(template),
+            ...schema.parse(promptTemplate),
             filename: file.replace(/\.json$/, ""),
         };
     });

package/lib/utils/preprocess.d.ts

@@ -12,6 +12,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         imageFile?: string | undefined;
         movieFile?: string | undefined;
         soundEffectFile?: string | undefined;
+        lipSyncFile?: string | undefined;
         captionFile?: string | undefined;
     }[];
     script: {
@@ -273,6 +274,10 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 provider?: string | undefined;
                 model?: string | undefined;
             } | undefined;
+            lipSyncParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
             htmlImageParams?: {
                 model?: string | undefined;
             } | undefined;
@@ -292,10 +297,15 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 images?: Record<string, any> | undefined;
                 systemPrompt?: string | undefined;
             } | undefined;
+            enableLipSync?: boolean | undefined;
         }[];
         lang?: string | undefined;
         title?: string | undefined;
         description?: string | undefined;
+        lipSyncParams?: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        } | undefined;
         htmlImageParams?: {
             provider: string;
             model?: string | undefined;

package/lib/utils/provider2agent.d.ts

@@ -61,6 +61,16 @@ export declare const provider2SoundEffectAgent: {
         }>;
     };
 };
+export declare const provider2LipSyncAgent: {
+    replicate: {
+        agentName: string;
+        defaultModel: ReplicateModel;
+        models: ReplicateModel[];
+        modelParams: Record<ReplicateModel, {
+            identifier?: `${string}/${string}:${string}`;
+        }>;
+    };
+};
 export declare const provider2LLMAgent: {
     readonly openai: {
         readonly agentName: "openAIAgent";
@@ -90,6 +100,7 @@ export declare const defaultProviders: {
     text2Html: keyof typeof provider2LLMAgent;
     llm: keyof typeof provider2LLMAgent;
     soundEffect: keyof typeof provider2SoundEffectAgent;
+    lipSync: keyof typeof provider2LipSyncAgent;
 };
 export declare const llm: (keyof typeof provider2LLMAgent)[];
 export type LLM = keyof typeof provider2LLMAgent;

package/lib/utils/provider2agent.js

@@ -130,6 +130,18 @@ export const provider2SoundEffectAgent = {
         },
     },
 };
+export const provider2LipSyncAgent = {
+    replicate: {
+        agentName: "lipSyncReplicateAgent",
+        defaultModel: "bytedance/latentsync",
+        models: ["bytedance/latentsync"],
+        modelParams: {
+            "bytedance/latentsync": {
+                identifier: "bytedance/latentsync:637ce1919f807ca20da3a448ddc2743535d2853649574cd52a933120e9b9e293",
+            },
+        },
+    },
+};
 // : Record<LLM, { agent: string; defaultModel: string; max_tokens: number }>
 export const provider2LLMAgent = {
     openai: {
@@ -160,6 +172,7 @@ export const defaultProviders = {
     text2Html: "openai",
     llm: "openai",
     soundEffect: "replicate",
+    lipSync: "replicate",
 };
 export const llm = Object.keys(provider2LLMAgent);
 export const htmlLLMProvider = ["openai", "anthropic"];

package/lib/utils/utils.js

@@ -76,6 +76,9 @@ export const settings2GraphAIConfig = (settings, env) => {
         soundEffectReplicateAgent: {
             apiKey: getKey("SOUND_EFFECT", "REPLICATE_API_TOKEN"),
         },
+        lipSyncReplicateAgent: {
+            apiKey: getKey("LIPSYNC", "REPLICATE_API_TOKEN"),
+        },
         // TODO
         // browserlessAgent
         // ttsGoogleAgent

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -12,6 +12,9 @@
     "./browser": {
       "types": "./lib/index.browser.d.ts",
       "default": "./lib/index.browser.js"
+    },
+    "./data": {
+      "default": "./lib/data/index.js"
     }
   },
   "bin": {
@@ -48,6 +51,7 @@
     "latest": "yarn upgrade-interactive  --latest",
     "format": "prettier --write '{src,scripts,assets/templates,assets/styles,draft,ideason,scripts_mag2,proto,test,graphai,output,docs/scripts}/**/*.{ts,json,yaml}'",
     "deep_research": "npx tsx ./src/tools/deep_research.ts",
+    "template": "npx tsx batch/template2tsobject.ts",
     "fake_data": "npx tsx test/fake/sample.ts",
     "mcp_server": "npx tsx ./src/mcp/server.ts"
   },
@@ -73,14 +77,14 @@
     "@tavily/core": "^0.5.9",
     "canvas": "^3.1.2",
     "clipboardy": "^4.0.0",
-    "dotenv": "^17.2.0",
+    "dotenv": "^17.2.1",
     "fluent-ffmpeg": "^2.1.3",
     "google-auth-library": "^10.1.0",
     "graphai": "^2.0.13",
     "inquirer": "^12.7.0",
     "marked": "^16.1.1",
     "ora": "^8.2.0",
-    "puppeteer": "^24.14.0",
+    "puppeteer": "^24.15.0",
     "replicate": "^1.0.1",
     "yaml": "^2.8.0",
     "yargs": "^18.0.0",
@@ -93,7 +97,7 @@
     "@receptron/test_utils": "^2.0.0",
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
-    "eslint": "^9.31.0",
+    "eslint": "^9.32.0",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-prettier": "^5.5.3",
     "eslint-plugin-sonarjs": "^3.0.4",

package/scripts/templates/business.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "Sample Title",

package/scripts/templates/children_book.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "桃太郎",

package/scripts/templates/coding.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "Sample Title",

package/scripts/templates/html.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "references": [

package/scripts/templates/image_prompt_only_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/image_prompts_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/image_refs.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0"
+    "version": "1.1"
   },
   "title": "[TITLE OF THE PRESENTAITON OR STORY]",
   "imageParams": {

package/scripts/templates/movie_prompts_no_text_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/movie_prompts_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/presentation.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "htmlImageParams": {

package/scripts/templates/sensei_and_taro.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "韓国の戒厳令とその日本への影響",

package/scripts/templates/shorts_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0"
+    "version": "1.1"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",
   "lang": "en",

package/scripts/templates/text_only_template.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0",
+    "version": "1.1",
     "credit": "closing"
   },
   "title": "[TITLE: Brief, engaging title for the topic]",

package/scripts/templates/voice_over.json

@@ -1,6 +1,6 @@
 {
   "$mulmocast": {
-    "version": "1.0"
+    "version": "1.1"
   },
   "title": "Voice Over Test",
   "captionParams": {