mulmocast 1.1.1 → 1.1.3

package/lib/utils/file.js

@@ -3,8 +3,9 @@ import path from "path";
 import { parse as yamlParse } from "yaml";
 import { fileURLToPath } from "url";
 import { GraphAILogger } from "graphai";
-import { MulmoScriptTemplateMethods, MulmoStudioContextMethods } from "../methods/index.js";
-import { mulmoScriptTemplateSchema, mulmoPresentationStyleSchema } from "../types/schema.js";
+import { MulmoStudioContextMethods } from "../methods/index.js";
+import { mulmoPromptTemplateSchema } from "../types/schema.js";
+import { getMulmoScriptTemplateSystemPrompt } from "./prompt.js";
 const promptTemplateDirName = "./assets/templates";
 const scriptTemplateDirName = "./scripts/templates";
 const __filename = fileURLToPath(import.meta.url);
@@ -150,61 +151,64 @@ export const getFullPath = (baseDirPath, file) => {
     }
     return path.resolve(file);
 };
-export const readScriptTemplateFile = (scriptName) => {
-    const scriptPath = path.resolve(npmRoot, scriptTemplateDirName, scriptName);
-    const scriptData = fs.readFileSync(scriptPath, "utf-8");
+// script and prompt template
+export const readScriptTemplateFile = (scriptTemplateFileName) => {
+    const scriptTemplatePath = path.resolve(npmRoot, scriptTemplateDirName, scriptTemplateFileName);
+    const scriptTemplateData = fs.readFileSync(scriptTemplatePath, "utf-8");
     // NOTE: We don't want to schema parse the script here to eliminate default values.
-    return JSON.parse(scriptData);
+    return JSON.parse(scriptTemplateData);
 };
-const readPromptTemplateFile = (promptTemplateName) => {
-    const promptTemplatePath = getPromptTemplateFilePath(promptTemplateName);
+const readPromptTemplateFile = (promptTemplateFileName) => {
+    const promptTemplatePath = getPromptTemplateFilePath(promptTemplateFileName);
     const promptTemplateData = fs.readFileSync(promptTemplatePath, "utf-8");
     // NOTE: We don't want to schema parse the template here to eliminate default values.
-    const promptTemplate = JSON.parse(promptTemplateData);
-    return promptTemplate;
+    return JSON.parse(promptTemplateData);
 };
 const mulmoScriptTemplate2Script = (scriptTemplate) => {
     if (scriptTemplate.scriptName) {
-        const scriptData = readScriptTemplateFile(scriptTemplate.scriptName);
-        return { ...scriptData, ...(scriptTemplate.presentationStyle ?? {}) };
+        const scriptTemplateData = readScriptTemplateFile(scriptTemplate.scriptName);
+        return { ...scriptTemplateData, ...(scriptTemplate.presentationStyle ?? {}) };
     }
     return undefined;
 };
-export const getScriptFromPromptTemplate = (promptTemplateName) => {
-    const promptTemplate = readPromptTemplateFile(promptTemplateName);
+export const getScriptFromPromptTemplate = (promptTemplateFileName) => {
+    const promptTemplate = readPromptTemplateFile(promptTemplateFileName);
     return mulmoScriptTemplate2Script(promptTemplate);
 };
-export const readTemplatePrompt = (promptTemplateName) => {
-    const promptTemplate = readPromptTemplateFile(promptTemplateName);
+export const readTemplatePrompt = (promptTemplateFileName) => {
+    const promptTemplate = readPromptTemplateFile(promptTemplateFileName);
     const script = mulmoScriptTemplate2Script(promptTemplate);
-    const prompt = MulmoScriptTemplateMethods.getSystemPrompt(promptTemplate, script);
+    const prompt = getMulmoScriptTemplateSystemPrompt(promptTemplate, script);
     return prompt;
 };
-// TODO: MulmoScriptTemplateFileは、実際はpromptTempate
-// TODO: remove it after update app
-export const getAvailableTemplates = () => {
-    return getAvailablePromptTemplates();
-};
-export const getAvailablePromptTemplates = () => {
-    return getPromptTemplates(promptTemplateDirName, mulmoScriptTemplateSchema);
-};
-export const getAvailableScriptTemplates = () => {
-    return getPromptTemplates(scriptTemplateDirName, mulmoPresentationStyleSchema);
-};
-export const getPromptTemplates = (dirPath, schema) => {
+const getPromptTemplates = (dirPath, schema) => {
     const templatesDir = path.resolve(npmRoot, dirPath);
     if (!fs.existsSync(templatesDir)) {
         return [];
     }
     const files = fs.readdirSync(templatesDir);
     return files.map((file) => {
-        const promptTemplate = JSON.parse(fs.readFileSync(path.resolve(templatesDir, file), "utf-8"));
-        return {
-            ...schema.parse(promptTemplate),
-            filename: file.replace(/\.json$/, ""),
-        };
+        try {
+            const promptTemplate = JSON.parse(fs.readFileSync(path.resolve(templatesDir, file), "utf-8"));
+            return {
+                ...(schema ? schema.parse(promptTemplate) : promptTemplate),
+                filename: file.replace(/\.json$/, ""),
+            };
+        }
+        catch (e) {
+            GraphAILogger.info("file: " + file);
+            GraphAILogger.info(e);
+            return {};
+        }
     });
 };
+export const getAvailablePromptTemplates = () => {
+    return getPromptTemplates(promptTemplateDirName, mulmoPromptTemplateSchema);
+};
+export const getAvailableScriptTemplates = () => {
+    return getPromptTemplates(scriptTemplateDirName, null);
+};
+// end of template
 export const writingMessage = (filePath) => {
     GraphAILogger.debug(`writing: ${filePath}`);
 };

package/lib/utils/inquirer.js

@@ -1,7 +1,7 @@
 import inquirer from "inquirer";
-import { getAvailableTemplates } from "./file.js";
+import { getAvailablePromptTemplates } from "./file.js";
 export const selectTemplate = async () => {
-    const availableTemplates = getAvailableTemplates();
+    const availableTemplates = getAvailablePromptTemplates();
     const answers = await inquirer.prompt([
         {
             type: "list",

package/lib/utils/preprocess.d.ts

@@ -205,12 +205,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "voice_over";
                 startAt?: number | undefined;
             } | undefined;
-            duration?: number | undefined;
-            speechOptions?: {
-                speed?: number | undefined;
-                instruction?: string | undefined;
-            } | undefined;
-            id?: string | undefined;
             audio?: {
                 type: "audio";
                 source: {
@@ -230,6 +224,12 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "midi";
                 source: string;
             } | undefined;
+            duration?: number | undefined;
+            speechOptions?: {
+                speed?: number | undefined;
+                instruction?: string | undefined;
+            } | undefined;
+            id?: string | undefined;
             imagePrompt?: string | undefined;
             speaker?: string | undefined;
             description?: string | undefined;

package/lib/utils/prompt.d.ts

@@ -1,8 +1,8 @@
-import { MulmoBeat, MulmoScript, MulmoScriptTemplate, MulmoStoryboard, MulmoCanvasDimension } from "../types/index.js";
+import { MulmoBeat, MulmoScript, MulmoPromptTemplate, MulmoStoryboard, MulmoCanvasDimension } from "../types/index.js";
 export declare const imagePrompt: (beat: MulmoBeat, style?: string) => string;
 export declare const graphDataScriptFromUrlPrompt: (sourceTextInput: string) => string;
 export declare const graphDataScriptGeneratePrompt: (scene: string) => string;
-export declare const getMulmoScriptTemplateSystemPrompt: (template: MulmoScriptTemplate, script?: MulmoScript) => string;
+export declare const getMulmoScriptTemplateSystemPrompt: (template: MulmoPromptTemplate, script?: MulmoScript) => string;
 export declare const interactiveClarificationPrompt = "If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.";
 export declare const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
 export declare const translateSystemPrompt = "Please translate the given text into the language specified in language (in locale format, like en, ja, fr, ch).";

package/lib/utils/provider2agent.d.ts

@@ -68,6 +68,8 @@ export declare const provider2LipSyncAgent: {
         models: ReplicateModel[];
         modelParams: Record<ReplicateModel, {
             identifier?: `${string}/${string}:${string}`;
+            video: string;
+            audio: string;
         }>;
     };
 };

package/lib/utils/provider2agent.js

@@ -134,11 +134,30 @@ export const provider2LipSyncAgent = {
     replicate: {
         agentName: "lipSyncReplicateAgent",
         defaultModel: "bytedance/latentsync",
-        models: ["bytedance/latentsync"],
+        models: ["bytedance/latentsync", "tmappdev/lipsync"],
         modelParams: {
             "bytedance/latentsync": {
                 identifier: "bytedance/latentsync:637ce1919f807ca20da3a448ddc2743535d2853649574cd52a933120e9b9e293",
+                video: "video",
+                audio: "audio",
             },
+            "tmappdev/lipsync": {
+                identifier: "tmappdev/lipsync:c54ce2fe673ea59b857b91250b3d71a2cd304a78f2370687632805c8405fbf4c",
+                video: "video_input",
+                audio: "audio_input",
+            },
+            /* NOTE: This model does not work with large base64 urls.
+            "sync/lipsync-2": {
+              video: "video",
+              audio: "audio",
+            },
+            */
+            /* NOTE: This model does not work well for some unknown reason.
+            "kwaivgi/kling-lip-sync": {
+              video: "video_url",
+              audio: "audio_file",
+            },
+            */
         },
     },
 };

package/lib/utils/system_prompt.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/utils/system_prompt.js

@@ -0,0 +1 @@
+export {};

package/lib/utils/templates.d.ts

@@ -0,0 +1,3 @@
+import type { MulmoScript } from "../types/type.js";
+export declare const readScriptTemplateFile: (scriptTemplateFileName: string) => MulmoScript;
+export declare const readTemplatePrompt: (promptTemplateFileName: string) => string;

package/lib/utils/templates.js

@@ -0,0 +1,46 @@
+import { mulmoScriptSchema } from "../types/schema.js";
+import { promptTemplates, scriptTemplates } from "../data/index.js";
+import { zodToJsonSchema } from "zod-to-json-schema";
+// script and prompt template
+export const readScriptTemplateFile = (scriptTemplateFileName) => {
+    // NOTE: We don't want to schema parse the script here to eliminate default values.
+    const scriptTemplate = scriptTemplates.find((template) => template.filename === scriptTemplateFileName);
+    if (!scriptTemplate) {
+        throw new Error(`Script template not found: ${scriptTemplateFileName}`);
+    }
+    const { filename: __, ...retValue } = scriptTemplate;
+    return retValue;
+};
+const readPromptTemplateFile = (promptTemplateFileName) => {
+    // NOTE: We don't want to schema parse the template here to eliminate default values.
+    const promptTemplate = promptTemplates.find((template) => template.filename === promptTemplateFileName);
+    if (!promptTemplate) {
+        throw new Error(`Prompt template not found: ${promptTemplateFileName}`);
+    }
+    return promptTemplate;
+};
+const mulmoScriptTemplate2Script = (scriptTemplate) => {
+    if (scriptTemplate.scriptName) {
+        const scriptTemplateData = readScriptTemplateFile(scriptTemplate.scriptName);
+        return { ...scriptTemplateData, ...(scriptTemplate.presentationStyle ?? {}) };
+    }
+    return undefined;
+};
+export const readTemplatePrompt = (promptTemplateFileName) => {
+    const promptTemplate = readPromptTemplateFile(promptTemplateFileName);
+    const script = mulmoScriptTemplate2Script(promptTemplate);
+    const prompt = getMulmoScriptTemplateSystemPrompt(promptTemplate, script);
+    return prompt;
+};
+const getMulmoScriptTemplateSystemPrompt = (template, script) => {
+    // script is provided, use it as a script template
+    if (script) {
+        return `${template.systemPrompt}\n\`\`\`JSON\n${JSON.stringify(script)}\n\`\`\``;
+    }
+    // script is not provided, use the default schema
+    const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
+        strictUnions: true,
+    });
+    const specificOutputPrompt = `The output should follow the JSON schema specified below. Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`;
+    return `${template.systemPrompt}\n\n${specificOutputPrompt}\n\n\`\`\`JSON\n${JSON.stringify(defaultSchema)}\n\`\`\``;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",