mulmocast 0.0.25 → 0.0.27

package/lib/utils/context.d.ts

@@ -0,0 +1,414 @@
+import type { MulmoScript, MulmoPresentationStyle, MulmoStudioMultiLingual } from "../types/type.js";
+import { FileObject } from "../types/index.js";
+export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<MulmoScript | null>;
+export declare const getMultiLingual: (multilingualFilePath: string, studioBeatsLength: number) => MulmoStudioMultiLingual;
+export declare const getPresentationStyle: (presentationStylePath: string | undefined) => MulmoPresentationStyle | null;
+export declare const initializeContextFromFiles: (files: FileObject, raiseError: boolean, force?: boolean, caption?: string, lang?: string) => Promise<{
+    studio: {
+        beats: {
+            duration?: number | undefined;
+            hash?: string | undefined;
+            audioFile?: string | undefined;
+            imageFile?: string | undefined;
+            movieFile?: string | undefined;
+            captionFile?: string | undefined;
+        }[];
+        script: {
+            audioParams: {
+                padding: number;
+                introPadding: number;
+                closingPadding: number;
+                outroPadding: number;
+                bgmVolume: number;
+                audioVolume: number;
+                suppressSpeech: boolean;
+                bgm?: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                } | undefined;
+            };
+            $mulmocast: {
+                version: "1.0";
+                credit?: "closing" | undefined;
+            };
+            canvasSize: {
+                width: number;
+                height: number;
+            };
+            speechParams: {
+                provider: "openai" | "nijivoice" | "google" | "elevenlabs";
+                speakers: Record<string, {
+                    voiceId: string;
+                    displayName?: Record<string, string> | undefined;
+                    speechOptions?: {
+                        speed?: number | undefined;
+                        instruction?: string | undefined;
+                    } | undefined;
+                    provider?: "openai" | "nijivoice" | "google" | "elevenlabs" | undefined;
+                }>;
+            };
+            beats: {
+                text: string;
+                speaker: string;
+                duration?: number | undefined;
+                speechOptions?: {
+                    speed?: number | undefined;
+                    instruction?: string | undefined;
+                } | undefined;
+                image?: {
+                    type: "markdown";
+                    markdown: string | string[];
+                } | {
+                    type: "web";
+                    url: string;
+                } | {
+                    type: "pdf";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                } | {
+                    type: "image";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                } | {
+                    type: "svg";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                } | {
+                    type: "textSlide";
+                    slide: {
+                        title: string;
+                        subtitle?: string | undefined;
+                        bullets?: string[] | undefined;
+                    };
+                } | {
+                    type: "chart";
+                    title: string;
+                    chartData: Record<string, any>;
+                } | {
+                    code: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                    type: "mermaid";
+                    title: string;
+                    appendix?: string[] | undefined;
+                } | {
+                    type: "html_tailwind";
+                    html: string | string[];
+                } | {
+                    type: "beat";
+                    id?: string | undefined;
+                } | {
+                    type: "movie";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                    mixAudio: number;
+                } | undefined;
+                id?: string | undefined;
+                audio?: {
+                    type: "audio";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                } | {
+                    type: "midi";
+                    source: string;
+                } | undefined;
+                description?: string | undefined;
+                imageParams?: {
+                    style?: string | undefined;
+                    model?: string | undefined;
+                    moderation?: string | undefined;
+                    images?: Record<string, {
+                        type: "image";
+                        source: {
+                            url: string;
+                            kind: "url";
+                        } | {
+                            kind: "base64";
+                            data: string;
+                        } | {
+                            text: string;
+                            kind: "text";
+                        } | {
+                            path: string;
+                            kind: "path";
+                        };
+                    }> | undefined;
+                } | undefined;
+                audioParams?: {
+                    padding?: number | undefined;
+                } | undefined;
+                movieParams?: {
+                    fillOption: {
+                        style: "aspectFit" | "aspectFill";
+                    };
+                } | undefined;
+                htmlImageParams?: {
+                    model?: string | undefined;
+                } | undefined;
+                textSlideParams?: {
+                    cssStyles: string | string[];
+                } | undefined;
+                captionParams?: {
+                    styles: string[];
+                    lang?: string | undefined;
+                } | undefined;
+                imageNames?: string[] | undefined;
+                imagePrompt?: string | undefined;
+                moviePrompt?: string | undefined;
+                htmlPrompt?: {
+                    prompt: string;
+                    data?: any;
+                    images?: Record<string, any> | undefined;
+                    systemPrompt?: string | undefined;
+                } | undefined;
+            }[];
+            lang?: string | undefined;
+            title?: string | undefined;
+            description?: string | undefined;
+            imageParams?: {
+                provider: "openai" | "google";
+                style?: string | undefined;
+                model?: string | undefined;
+                moderation?: string | undefined;
+                images?: Record<string, {
+                    type: "image";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                }> | undefined;
+            } | undefined;
+            movieParams?: {
+                provider?: "openai" | "google" | "replicate" | undefined;
+                model?: string | undefined;
+                fillOption?: {
+                    style: "aspectFit" | "aspectFill";
+                } | undefined;
+                transition?: {
+                    type: "fade" | "slideout_left";
+                    duration: number;
+                } | undefined;
+            } | undefined;
+            htmlImageParams?: {
+                provider: "openai" | "anthropic";
+                model?: string | undefined;
+            } | undefined;
+            textSlideParams?: {
+                cssStyles: string | string[];
+            } | undefined;
+            captionParams?: {
+                styles: string[];
+                lang?: string | undefined;
+            } | undefined;
+            references?: {
+                type: "image" | "audio" | "article" | "paper" | "video";
+                url: string;
+                title?: string | undefined;
+                description?: string | undefined;
+            }[] | undefined;
+            imagePath?: string | undefined;
+            __test_invalid__?: boolean | undefined;
+        };
+        filename: string;
+    };
+    fileDirs: FileObject;
+    force: boolean;
+    lang: string | undefined;
+    sessionState: {
+        inSession: {
+            audio: boolean;
+            image: boolean;
+            video: boolean;
+            multiLingual: boolean;
+            caption: boolean;
+            pdf: boolean;
+        };
+        inBeatSession: {
+            audio: {};
+            image: {};
+            movie: {};
+            multiLingual: {};
+            caption: {};
+        };
+    };
+    presentationStyle: {
+        audioParams: {
+            padding: number;
+            introPadding: number;
+            closingPadding: number;
+            outroPadding: number;
+            bgmVolume: number;
+            audioVolume: number;
+            suppressSpeech: boolean;
+            bgm?: {
+                url: string;
+                kind: "url";
+            } | {
+                kind: "base64";
+                data: string;
+            } | {
+                text: string;
+                kind: "text";
+            } | {
+                path: string;
+                kind: "path";
+            } | undefined;
+        };
+        $mulmocast: {
+            version: "1.0";
+            credit?: "closing" | undefined;
+        };
+        canvasSize: {
+            width: number;
+            height: number;
+        };
+        speechParams: {
+            provider: "openai" | "nijivoice" | "google" | "elevenlabs";
+            speakers: Record<string, {
+                voiceId: string;
+                displayName?: Record<string, string> | undefined;
+                speechOptions?: {
+                    speed?: number | undefined;
+                    instruction?: string | undefined;
+                } | undefined;
+                provider?: "openai" | "nijivoice" | "google" | "elevenlabs" | undefined;
+            }>;
+        };
+        imageParams?: {
+            provider: "openai" | "google";
+            style?: string | undefined;
+            model?: string | undefined;
+            moderation?: string | undefined;
+            images?: Record<string, {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            }> | undefined;
+        } | undefined;
+        movieParams?: {
+            provider?: "openai" | "google" | "replicate" | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        } | undefined;
+        htmlImageParams?: {
+            provider: "openai" | "anthropic";
+            model?: string | undefined;
+        } | undefined;
+        textSlideParams?: {
+            cssStyles: string | string[];
+        } | undefined;
+        captionParams?: {
+            styles: string[];
+            lang?: string | undefined;
+        } | undefined;
+    };
+    multiLingual: {
+        multiLingualTexts: Record<string, {
+            text: string;
+            lang: string;
+            texts?: string[] | undefined;
+            ttsTexts?: string[] | undefined;
+            duration?: number | undefined;
+        }>;
+    }[];
+} | null>;

package/lib/utils/context.js

@@ -0,0 +1,96 @@
+import { GraphAILogger } from "graphai";
+import fs from "fs";
+import { readMulmoScriptFile, fetchMulmoScriptFile } from "./file.js";
+import { createOrUpdateStudioData } from "./preprocess.js";
+import { mulmoPresentationStyleSchema, mulmoStudioMultiLingualSchema } from "../types/index.js";
+export const fetchScript = async (isHttpPath, mulmoFilePath, fileOrUrl) => {
+    if (isHttpPath) {
+        const res = await fetchMulmoScriptFile(fileOrUrl);
+        if (!res.result || !res.script) {
+            GraphAILogger.info(`ERROR: HTTP error! ${res.status} ${fileOrUrl}`);
+            return null;
+        }
+        return res.script;
+    }
+    if (!fs.existsSync(mulmoFilePath)) {
+        GraphAILogger.info(`ERROR: File not exists ${mulmoFilePath}`);
+        return null;
+    }
+    return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath)?.mulmoData ?? null;
+};
+export const getMultiLingual = (multilingualFilePath, studioBeatsLength) => {
+    if (fs.existsSync(multilingualFilePath)) {
+        const jsonData = readMulmoScriptFile(multilingualFilePath, "ERROR: File does not exist " + multilingualFilePath)?.mulmoData ?? null;
+        const dataSet = mulmoStudioMultiLingualSchema.parse(jsonData);
+        while (dataSet.length < studioBeatsLength) {
+            dataSet.push({ multiLingualTexts: {} });
+        }
+        dataSet.length = studioBeatsLength;
+        return dataSet;
+    }
+    return [...Array(studioBeatsLength)].map(() => ({ multiLingualTexts: {} }));
+};
+export const getPresentationStyle = (presentationStylePath) => {
+    if (presentationStylePath) {
+        if (!fs.existsSync(presentationStylePath)) {
+            throw new Error(`ERROR: File not exists ${presentationStylePath}`);
+        }
+        const jsonData = readMulmoScriptFile(presentationStylePath, "ERROR: File does not exist " + presentationStylePath)?.mulmoData ?? null;
+        return mulmoPresentationStyleSchema.parse(jsonData);
+    }
+    return null;
+};
+const initSessionState = () => {
+    return {
+        inSession: {
+            audio: false,
+            image: false,
+            video: false,
+            multiLingual: false,
+            caption: false,
+            pdf: false,
+        },
+        inBeatSession: {
+            audio: {},
+            image: {},
+            movie: {},
+            multiLingual: {},
+            caption: {},
+        },
+    };
+};
+const buildContext = (studio, files, presentationStyle, multiLingual, force, lang) => {
+    return {
+        studio,
+        fileDirs: files,
+        force: Boolean(force),
+        lang,
+        sessionState: initSessionState(),
+        presentationStyle: presentationStyle ?? studio.script,
+        multiLingual,
+    };
+};
+export const initializeContextFromFiles = async (files, raiseError, force, caption, lang) => {
+    const { fileName, isHttpPath, fileOrUrl, mulmoFilePath, outputStudioFilePath, presentationStylePath, outputMultilingualFilePath } = files;
+    // read mulmoScript, presentationStyle, currentStudio from files
+    const mulmoScript = await fetchScript(isHttpPath, mulmoFilePath, fileOrUrl);
+    if (!mulmoScript) {
+        return null;
+    }
+    const presentationStyle = getPresentationStyle(presentationStylePath);
+    // Create or update MulmoStudio file with MulmoScript
+    const currentStudio = readMulmoScriptFile(outputStudioFilePath);
+    try {
+        // validate mulmoStudioSchema. skip if __test_invalid__ is true
+        const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName, caption);
+        const multiLingual = getMultiLingual(outputMultilingualFilePath, studio.beats.length);
+        return buildContext(studio, files, presentationStyle, multiLingual, force, lang);
+    }
+    catch (error) {
+        GraphAILogger.info(`Error: invalid MulmoScript Schema: ${isHttpPath ? fileOrUrl : mulmoFilePath} \n ${error}`);
+        if (raiseError) {
+            throw error;
+        }
+        return null;
+    }
+};

package/lib/utils/ffmpeg_utils.d.ts

@@ -4,6 +4,7 @@ export type FfmpegContext = {
     inputCount: number;
     filterComplex: string[];
 };
+export declare const setFfmpegPath: (ffmpegPath: string) => void;
 export declare const FfmpegContextInit: () => FfmpegContext;
 export declare const FfmpegContextAddInput: (context: FfmpegContext, input: string, inputOptions?: string[]) => number;
 export declare const FfmpegContextPushFormattedAudio: (context: FfmpegContext, sourceId: string, outputId: string, duration?: number | undefined) => void;

package/lib/utils/ffmpeg_utils.js

@@ -1,5 +1,8 @@
 import ffmpeg from "fluent-ffmpeg";
 import { GraphAILogger } from "graphai";
+export const setFfmpegPath = (ffmpegPath) => {
+    ffmpeg.setFfmpegPath(ffmpegPath);
+};
 export const FfmpegContextInit = () => {
     return {
         command: ffmpeg(),

package/lib/utils/file.d.ts

@@ -41,3 +41,4 @@ export declare const readTemplatePrompt: (templateName: string) => string;
 export declare const getAvailableTemplates: () => MulmoScriptTemplateFile[];
 export declare const writingMessage: (filePath: string) => void;
 export declare const readAndParseJson: <S extends ZodSchema<any>>(filePath: string, schema: S) => ReturnType<S["parse"]>;
+export declare const generateTimestampedFileName: (prefix: string) => string;

package/lib/utils/file.js

@@ -182,3 +182,8 @@ export const readAndParseJson = (filePath, schema) => {
     const json = JSON.parse(fileContent);
     return schema.parse(json);
 };
+export const generateTimestampedFileName = (prefix) => {
+    const now = new Date();
+    const pad = (n) => n.toString().padStart(2, "0");
+    return `${prefix}_${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
+};

package/lib/utils/preprocess.d.ts

@@ -1,5 +1,5 @@
 import { MulmoStudio, MulmoScript } from "../types/index.js";
-export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string) => {
+export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string) => {
     beats: {
         duration?: number | undefined;
         hash?: string | undefined;
@@ -16,6 +16,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             outroPadding: number;
             bgmVolume: number;
             audioVolume: number;
+            suppressSpeech: boolean;
             bgm?: {
                 url: string;
                 kind: "url";

package/lib/utils/preprocess.js

@@ -1,5 +1,5 @@
 import { GraphAILogger } from "graphai";
-import { mulmoScriptSchema, mulmoStudioSchema } from "../types/index.js";
+import { mulmoScriptSchema, mulmoStudioSchema, mulmoCaptionParamsSchema } from "../types/index.js";
 const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
     const isTest = process.env.NODE_ENV === "test";
     const parsed = isTest && currentStudio ? { data: mulmoStudioSchema.parse(currentStudio), success: true, error: null } : mulmoStudioSchema.safeParse(currentStudio);
@@ -36,7 +36,7 @@ const mulmoCredit = (speaker) => {
         },
     };
 };
-export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName) => {
+export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaption) => {
     const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : mulmoScriptSchema.parse(_mulmoScript); // validate and insert default value
     const studio = rebuildStudio(currentStudio, mulmoScript, fileName);
     // TODO: Move this code out of this function later
@@ -46,5 +46,11 @@ export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName)
     }
     studio.script = mulmoScriptSchema.parse(mulmoScript); // update the script
     studio.beats = studio.script.beats.map((_, index) => studio.beats[index] ?? {});
+    if (videoCaption) {
+        studio.script.captionParams = mulmoCaptionParamsSchema.parse({
+            ...(studio.script.captionParams ?? {}),
+            lang: videoCaption,
+        });
+    }
     return studio;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.0.25",
+  "version": "0.0.27",
   "description": "",
   "type": "module",
   "main": "lib/index.js",
@@ -48,7 +48,8 @@
     "latest": "yarn upgrade-interactive  --latest",
     "format": "prettier --write '{src,scripts,assets/templates,assets/styles,draft,ideason,scripts_mag2,proto,test,graphai,output,docs/scripts}/**/*.{ts,json,yaml}'",
     "deep_research": "npx tsx ./src/tools/deep_research.ts",
-    "fake_data": "npx tsx test/fake/sample.ts"
+    "fake_data": "npx tsx test/fake/sample.ts",
+    "mcp_server": "npx tsx ./src/mcp/server.ts"
   },
   "repository": "git+ssh://git@github.com/receptron/mulmocast-cli.git",
   "author": "snakajima",
@@ -68,6 +69,7 @@
     "@graphai/stream_agent_filter": "^2.0.2",
     "@graphai/vanilla": "^2.0.4",
     "@graphai/vanilla_node_agents": "^2.0.1",
+    "@modelcontextprotocol/sdk": "^1.13.1",
     "@tavily/core": "^0.5.8",
     "canvas": "^3.1.2",
     "clipboardy": "^4.0.0",
@@ -76,12 +78,12 @@
     "google-auth-library": "^9.15.1",
     "graphai": "^2.0.9",
     "inquirer": "^12.6.3",
-    "marked": "^15.0.12",
+    "marked": "^16.0.0",
     "ora": "^8.2.0",
-    "puppeteer": "^24.11.0",
+    "puppeteer": "^24.11.1",
     "replicate": "^1.0.1",
     "yaml": "^2.8.0",
-    "yargs": "^17.7.2",
+    "yargs": "^18.0.0",
     "zod": "^3.25.67",
     "zod-to-json-schema": "^3.24.6"
   },