mulmocast 0.0.5 → 0.0.7

package/lib/utils/preprocess.d.ts

@@ -2,18 +2,27 @@ import { MulmoStudio, MulmoScript } from "../types/index.js";
 export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string) => {
     beats: {
         duration?: number | undefined;
-        multiLingualTexts?: Record<string, {
+        hash?: string | undefined;
+        audioFile?: string | undefined;
+        imageFile?: string | undefined;
+        captionFile?: string | undefined;
+    }[];
+    multiLingual: {
+        multiLingualTexts: Record<string, {
             text: string;
             lang: string;
             texts?: string[] | undefined;
             ttsTexts?: string[] | undefined;
             duration?: number | undefined;
-        }> | undefined;
-        hash?: string | undefined;
-        audioFile?: string | undefined;
-        imageFile?: string | undefined;
+        }>;
     }[];
     script: {
+        audioParams: {
+            padding: number;
+            introPadding: number;
+            closingPadding: number;
+            outroPadding: number;
+        };
         $mulmocast: {
             version: "1.0";
             credit?: "closing" | undefined;
@@ -23,7 +32,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             height: number;
         };
         speechParams: {
-            provider: "openai" | "nijivoice";
+            provider: "openai" | "nijivoice" | "google";
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
@@ -36,6 +45,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         beats: {
             text: string;
             speaker: string;
+            duration?: number | undefined;
             speechOptions?: {
                 speed?: number | undefined;
                 instruction?: string | undefined;
@@ -110,7 +120,8 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "textSlide";
                 slide: {
                     title: string;
-                    bullets: string[];
+                    subtitle?: string | undefined;
+                    bullets?: string[] | undefined;
                 };
             } | {
                 type: "chart";
@@ -133,6 +144,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "mermaid";
                 title: string;
                 appendix?: string[] | undefined;
+            } | {
+                type: "html_tailwind";
+                html: string | string[];
             } | undefined;
             audio?: {
                 type: "audio";
@@ -159,6 +173,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 style?: string | undefined;
                 moderation?: string | undefined;
             } | undefined;
+            audioParams?: {
+                padding?: number | undefined;
+            } | undefined;
             textSlideParams?: {
                 cssStyles: string | string[];
             } | undefined;
@@ -176,9 +193,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         textSlideParams?: {
             cssStyles: string | string[];
         } | undefined;
-        videoParams?: {
-            padding?: number | undefined;
-        } | undefined;
         omitCaptions?: boolean | undefined;
         description?: string | undefined;
         references?: {
@@ -191,4 +205,20 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         __test_invalid__?: boolean | undefined;
     };
     filename: string;
+    state: {
+        inSession: {
+            pdf: boolean;
+            image: boolean;
+            audio: boolean;
+            video: boolean;
+            multiLingual: boolean;
+            caption: boolean;
+        };
+        inBeatSession: {
+            image: Set<number>;
+            audio: Set<number>;
+            multiLingual: Set<number>;
+            caption: Set<number>;
+        };
+    };
 };

package/lib/utils/preprocess.js

@@ -4,11 +4,13 @@ const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
     if (parsed.success) {
         return parsed.data;
     }
-    return {
+    // We need to parse it to fill default values
+    return mulmoStudioSchema.parse({
         script: mulmoScript,
         filename: fileName,
         beats: [...Array(mulmoScript.beats.length)].map(() => ({})),
-    };
+        multiLingual: [...Array(mulmoScript.beats.length)].map(() => ({ multiLingualTexts: {} })),
+    });
 };
 const mulmoCredit = (speaker) => {
     return {
@@ -42,6 +44,9 @@ export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName)
     mulmoScript.beats.forEach((beat, index) => {
         // Filling the default values
         studio.script.beats[index] = mulmoBeatSchema.parse(beat);
+        if (!studio.multiLingual[index]) {
+            studio.multiLingual[index] = { multiLingualTexts: {} };
+        }
     });
     return studio;
 };

package/lib/utils/prompt.d.ts

@@ -0,0 +1,16 @@
+import { MulmoBeat, MulmoScript, MulmoScriptTemplate, MulmoStoryboard } from "../types/index.js";
+export declare const imagePrompt: (beat: MulmoBeat, style?: string) => string;
+export declare const graphDataScriptFromUrlPrompt: (sourceTextInput: string) => string;
+export declare const graphDataScriptGeneratePrompt: (scene: string) => string;
+export declare const getMulmoScriptTemplateSystemPrompt: (template: MulmoScriptTemplate, script?: MulmoScript) => string;
+export declare const interactiveClarificationPrompt = "If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.";
+export declare const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
+export declare const translateSystemPrompt = "Please translate the given text into the language specified in language (in locale format, like en, ja, fr, ch).";
+export declare const translatePrompts: string[];
+export declare const sceneToBeatsPrompt: ({ sampleBeats, beatsPerScene, allScenes, }: {
+    sampleBeats: MulmoScript["beats"];
+    beatsPerScene: number;
+    allScenes: string;
+}) => string;
+export declare const storyToScriptInfoPrompt: (scriptWithoutBeats: Omit<MulmoScript, "beats">, story: MulmoStoryboard) => string;
+export declare const storyToScriptPrompt: (script: MulmoScript, beatsPerScene: number, story: MulmoStoryboard) => string;

package/lib/utils/prompt.js

@@ -0,0 +1,74 @@
+import { mulmoScriptSchema } from "../types/schema.js";
+import { zodToJsonSchema } from "zod-to-json-schema";
+export const imagePrompt = (beat, style) => {
+    return (beat.imagePrompt || `generate image appropriate for the text. text: ${beat.text}`) + "\n" + (style || "");
+};
+// sourceTextInput: ${:sourceText.text}
+export const graphDataScriptFromUrlPrompt = (sourceTextInput) => {
+    return `Please create a script using the information from the following URLs as reference: ${sourceTextInput}`;
+};
+export const graphDataScriptGeneratePrompt = (scene) => {
+    return `Please generate a script from the following scenes: ${scene}`;
+};
+export const getMulmoScriptTemplateSystemPrompt = (template, script) => {
+    // script is provided, use it as a script template
+    if (script) {
+        return `${template.systemPrompt}\n\`\`\`JSON\n${JSON.stringify(script)}\n\`\`\``;
+    }
+    // script is not provided, use the default schema
+    const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
+        strictUnions: true,
+    });
+    const specificOutputPrompt = `The output should follow the JSON schema specified below. Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`;
+    return `${template.systemPrompt}\n\n${specificOutputPrompt}\n\n\`\`\`JSON\n${JSON.stringify(defaultSchema)}\n\`\`\``;
+};
+export const interactiveClarificationPrompt = `If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.`;
+export const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
+export const translateSystemPrompt = "Please translate the given text into the language specified in language (in locale format, like en, ja, fr, ch).";
+export const translatePrompts = ["## Original Language", ":lang", "", "## Language", ":targetLang", "", "## Target", ":beat.text"];
+export const sceneToBeatsPrompt = ({ sampleBeats, beatsPerScene, allScenes, }) => {
+    return `Generate scripts for the given scenes, following the structure of the sample scripts below.
+\`\`\`JSON
+${JSON.stringify(sampleBeats)}
+\`\`\`
+From the content of each scene, generate exactly ${beatsPerScene} scripts (beats).
+The scripts should be created considering the overall content of the scenes.
+The scenes are as follows:
+\`\`\`
+${allScenes}
+\`\`\`
+Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`.trim();
+};
+export const storyToScriptInfoPrompt = (scriptWithoutBeats, story) => {
+    return `Generate script for the given storyboard, following the structure of the sample scripts below.
+Storyboard:
+- title: ${story.title}
+- description: ${story.scenes.map((scene) => scene.description).join("\n")}
+- references: ${story.references?.map((reference) => JSON.stringify(reference)).join("\n")}
+
+Sample script:
+\`\`\`JSON
+${JSON.stringify(scriptWithoutBeats)}
+\`\`\`
+
+Only include keys that exist in the sample script.
+Do not add any keys that are not present in the sample script.
+Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`.trim();
+};
+export const storyToScriptPrompt = (script, beatsPerScene, story) => {
+    return `Generate script for the given storyboard, following the structure of the sample scripts below.
+Storyboard:
+- title: ${story.title}
+- references: ${story.references?.map((reference) => JSON.stringify(reference)).join("\n")}
+- scenes: ${JSON.stringify(story.scenes)}
+
+Sample script:
+\`\`\`JSON
+${JSON.stringify(script)}
+\`\`\`
+
+Generate exactly ${beatsPerScene} scripts (beats) for each scene.
+Only include keys that exist in the sample script.
+Do not add any keys that are not present in the sample script.
+Please provide your response as valid JSON within \`\`\`json code blocks for clarity.`.trim();
+};

package/lib/utils/utils.d.ts

@@ -1,7 +1,12 @@
-export declare const llmAgents: string[];
-type LLMAgent = (typeof llmAgents)[number];
-export declare const defaultOpenAIModel: string;
-export declare const llmPair: (_agent?: LLMAgent, _model?: string) => {
+import { MulmoBeat, MulmoStudioMultiLingualData } from "../types/index.js";
+export declare const llm: readonly ["openAI", "anthropic", "gemini", "groq"];
+export type LLM = (typeof llm)[number];
+export declare const llmConfig: Record<LLM, {
+    agent: string;
+    defaultModel: string;
+    max_tokens: number;
+}>;
+export declare const llmPair: (_llm?: LLM, _model?: string) => {
     agent: string;
     model: string;
     max_tokens: number;
@@ -9,4 +14,4 @@ export declare const llmPair: (_agent?: LLMAgent, _model?: string) => {
 export declare const chunkArray: <T>(array: T[], size?: number) => T[][];
 export declare const isHttp: (fileOrUrl: string) => boolean;
 export declare const text2hash: (input: string) => string;
-export {};
+export declare const localizedText: (beat: MulmoBeat, multiLingualData?: MulmoStudioMultiLingualData, lang?: string) => string;

package/lib/utils/utils.js

@@ -1,22 +1,32 @@
 import * as crypto from "crypto";
-export const llmAgents = ["openAIAgent", "anthropicAgent", "geminiAgent", "groqAgent"];
-const defaultModels = {
-    anthropicAgent: "claude-3-7-sonnet-20250219",
-    geminiAgent: "gemini-1.5-flash",
-    groqAgent: "llama3-8b-8192",
-    openAIAgent: "gpt-4o",
+export const llm = ["openAI", "anthropic", "gemini", "groq"];
+export const llmConfig = {
+    openAI: {
+        agent: "openAIAgent",
+        defaultModel: "gpt-4o",
+        max_tokens: 8192,
+    },
+    anthropic: {
+        agent: "anthropicAgent",
+        defaultModel: "claude-3-7-sonnet-20250219",
+        max_tokens: 8192,
+    },
+    gemini: {
+        agent: "geminiAgent",
+        defaultModel: "gemini-1.5-flash",
+        max_tokens: 8192,
+    },
+    groq: {
+        agent: "groqAgent",
+        defaultModel: "llama3-8b-8192",
+        max_tokens: 4096,
+    },
 };
-const longMaxTokens = {
-    anthropicAgent: 8192,
-    geminiAgent: 8192,
-    groqAgent: 4096,
-    openAIAgent: 8192,
-};
-export const defaultOpenAIModel = defaultModels["openAIAgent"];
-export const llmPair = (_agent, _model) => {
-    const agent = _agent && llmAgents.includes(_agent ?? "") ? _agent : "openAIAgent";
-    const model = _model ?? defaultModels[agent ?? ""];
-    const max_tokens = longMaxTokens[agent];
+export const llmPair = (_llm, _model) => {
+    const llmKey = _llm ?? "openAI";
+    const agent = llmConfig[llmKey]?.agent ?? llmConfig.openAI.agent;
+    const model = _model ?? llmConfig[llmKey]?.defaultModel ?? llmConfig.openAI.defaultModel;
+    const max_tokens = llmConfig[llmKey]?.max_tokens ?? llmConfig.openAI.max_tokens;
     return { agent, model, max_tokens };
 };
 export const chunkArray = (array, size = 3) => {
@@ -32,3 +42,13 @@ export const isHttp = (fileOrUrl) => {
 export const text2hash = (input) => {
     return crypto.createHash("sha256").update(input).digest("hex");
 };
+export const localizedText = (beat, multiLingualData, lang) => {
+    if (lang &&
+        multiLingualData &&
+        multiLingualData?.multiLingualTexts &&
+        multiLingualData?.multiLingualTexts[lang] &&
+        multiLingualData?.multiLingualTexts[lang].text) {
+        return multiLingualData?.multiLingualTexts[lang].text;
+    }
+    return beat.text;
+};

package/package.json

@@ -1,18 +1,18 @@
 {
   "name": "mulmocast",
-  "version": "0.0.5",
+  "version": "0.0.7",
   "description": "",
   "type": "module",
-  "main": "lib/cli/run.js",
+  "main": "lib/cli/bin.js",
   "bin": {
-    "mulmo": "lib/cli/run.js",
-    "mulmo-tool": "lib/cli/tool-cli.js"
+    "mulmo": "lib/cli/bin.js"
   },
   "files": [
     "./lib",
     "./assets/music/StarsBeyondEx.mp3",
     "./assets/audio/silent300.mp3",
     "./assets/audio/silent800.mp3",
+    "./assets/audio/silent60sec.mp3",
     "./assets/html/",
     "./assets/font/",
     "./assets/templates/"
@@ -23,20 +23,22 @@
     "test": "tests"
   },
   "scripts": {
-    "audio": "npx tsx ./src/cli/run.ts audio",
-    "translate": "npx tsx ./src/cli/run.ts translate",
-    "movie": "npx tsx ./src/cli/run.ts movie",
-    "images": "npx tsx ./src/cli/run.ts images",
-    "preprocess": "npx tsx ./src/cli/run.ts preprocess",
+    "audio": "npx tsx ./src/cli/bin.ts audio",
+    "translate": "npx tsx ./src/cli/bin.ts translate",
+    "movie": "npx tsx ./src/cli/bin.ts movie",
+    "images": "npx tsx ./src/cli/bin.ts images",
+    "preprocess": "npx tsx ./src/cli/bin.ts preprocess",
+    "pdf": "npx tsx ./src/cli/bin.ts pdf",
     "test": "rm -f scratchpad/test*.* && npx tsx ./src/audio.ts scripts/test/test.json && npx tsx ./src/images.ts scripts/test/test.json && npx tsx ./src/movie.ts scripts/test/test.json",
     "ci_test": "tsx --test ./test/*/test_*.ts",
     "lint": "eslint src test",
     "build": "tsc",
     "build_test": "tsc && git checkout -- lib/*",
-    "prompt": "npx tsx ./src/tools/prompt.ts",
-    "cli": "npx tsx ./src/cli/run.ts",
-    "scripting": "npx tsx ./src/cli/tool-cli.ts scripting",
-    "dump_prompt": "npx tsx ./src/cli/tool-cli.ts prompt",
+    "cli": "npx tsx ./src/cli/bin.ts",
+    "scripting": "npx tsx ./src/cli/bin.ts tool scripting",
+    "prompt": "npx tsx ./src/cli/bin.ts tool prompt",
+    "schema": "npx tsx ./src/cli/bin.ts tool schema",
+    "story_to_script": "npx tsx ./src/cli/bin.ts tool story_to_script",
     "latest": "yarn upgrade-interactive  --latest",
     "format": "prettier --write '{src,scripts,assets/templates,draft,ideason,scripts_mag2,proto,test,graphai,output,docs/scripts}/**/*.{ts,json,yaml}'"
   },
@@ -48,43 +50,45 @@
   },
   "homepage": "https://github.com/receptron/mulmocast-cli#readme",
   "dependencies": {
-    "@graphai/agent_filters": "^1.0.1",
+    "@google-cloud/text-to-speech": "^6.1.0",
     "@graphai/anthropic_agent": "^2.0.0",
     "@graphai/browserless_agent": "^2.0.0",
     "@graphai/gemini_agent": "^1.0.1",
     "@graphai/groq_agent": "^1.0.1",
     "@graphai/input_agents": "^1.0.1",
-    "@graphai/openai_agent": "^1.0.5",
-    "@graphai/vanilla": "^2.0.1",
+    "@graphai/openai_agent": "^2.0.0",
+    "@graphai/stream_agent_filter": "^2.0.1",
+    "@graphai/vanilla": "^2.0.2",
     "@graphai/vanilla_node_agents": "^2.0.0",
     "@pdf-lib/fontkit": "^1.1.1",
-    "@types/fluent-ffmpeg": "^2.1.26",
     "canvas": "^3.1.0",
+    "clipboardy": "^4.0.0",
     "dotenv": "^16.4.7",
     "fluent-ffmpeg": "^2.1.3",
     "google-auth-library": "^9.15.1",
-    "graphai": "^1.0.13",
-    "inquirer": "^12.6.0",
+    "graphai": "^2.0.3",
+    "inquirer": "^12.6.1",
     "marked": "^15.0.11",
     "ora": "^8.2.0",
     "pdf-lib": "^1.17.1",
     "puppeteer": "^24.8.1",
-    "yaml": "^2.7.1",
+    "yaml": "^2.8.0",
     "yargs": "^17.7.2",
     "zod": "^3.24.4",
     "zod-to-json-schema": "^3.24.5"
   },
   "devDependencies": {
     "@receptron/test_utils": "^2.0.0",
+    "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
-    "eslint": "^9.26.0",
-    "eslint-config-prettier": "^10.0.1",
+    "eslint": "^9.27.0",
+    "eslint-config-prettier": "^10.1.5",
     "eslint-plugin-prettier": "^5.4.0",
     "prettier": "^3.3.3",
     "ts-node": "^10.9.2",
     "tsx": "^4.19.4",
     "typescript": "^5.7.3",
-    "typescript-eslint": "^8.32.0"
+    "typescript-eslint": "^8.32.1"
   },
   "engines": {
     "node": ">=16.0.0"