mulmocast 1.1.5 → 1.1.6

package/lib/utils/context.d.ts

@@ -1,9 +1,351 @@
 import type { MulmoScript, MulmoPresentationStyle, MulmoStudioMultiLingual } from "../types/type.js";
 import { FileObject } from "../types/index.js";
+export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: string, videoCaptionLang?: string, presentationStyle?: MulmoPresentationStyle | null) => {
+    beats: {
+        duration?: number | undefined;
+        startAt?: number | undefined;
+        hash?: string | undefined;
+        audioDuration?: number | undefined;
+        movieDuration?: number | undefined;
+        silenceDuration?: number | undefined;
+        hasMovieAudio?: boolean | undefined;
+        audioFile?: string | undefined;
+        imageFile?: string | undefined;
+        movieFile?: string | undefined;
+        soundEffectFile?: string | undefined;
+        lipSyncFile?: string | undefined;
+        captionFile?: string | undefined;
+    }[];
+    script: {
+        lang: string;
+        imageParams: {
+            provider: string;
+            model?: string | undefined;
+            style?: string | undefined;
+            quality?: string | undefined;
+            moderation?: string | undefined;
+            images?: Record<string, {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "imagePrompt";
+                prompt: string;
+            }> | undefined;
+        };
+        audioParams: {
+            padding: number;
+            introPadding: number;
+            closingPadding: number;
+            outroPadding: number;
+            bgmVolume: number;
+            audioVolume: number;
+            suppressSpeech: boolean;
+            bgm?: {
+                url: string;
+                kind: "url";
+            } | {
+                kind: "base64";
+                data: string;
+            } | {
+                text: string;
+                kind: "text";
+            } | {
+                path: string;
+                kind: "path";
+            } | undefined;
+        };
+        movieParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
+        soundEffectParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        };
+        $mulmocast: {
+            version: "1.1";
+            credit?: "closing" | undefined;
+        };
+        canvasSize: {
+            width: number;
+            height: number;
+        };
+        speechParams: {
+            speakers: Record<string, {
+                voiceId: string;
+                lang?: Record<string, {
+                    voiceId: string;
+                    displayName?: Record<string, string> | undefined;
+                    isDefault?: boolean | undefined;
+                    speechOptions?: {
+                        speed?: number | undefined;
+                        instruction?: string | undefined;
+                    } | undefined;
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                }> | undefined;
+                displayName?: Record<string, string> | undefined;
+                isDefault?: boolean | undefined;
+                speechOptions?: {
+                    speed?: number | undefined;
+                    instruction?: string | undefined;
+                } | undefined;
+                provider?: string | undefined;
+                model?: string | undefined;
+            }>;
+        };
+        beats: {
+            text: string;
+            image?: {
+                type: "markdown";
+                markdown: string | string[];
+            } | {
+                type: "web";
+                url: string;
+            } | {
+                type: "pdf";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "svg";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "movie";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "textSlide";
+                slide: {
+                    title: string;
+                    subtitle?: string | undefined;
+                    bullets?: string[] | undefined;
+                };
+            } | {
+                type: "chart";
+                title: string;
+                chartData: Record<string, any>;
+            } | {
+                code: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+                type: "mermaid";
+                title: string;
+                appendix?: string[] | undefined;
+            } | {
+                type: "html_tailwind";
+                html: string | string[];
+            } | {
+                type: "beat";
+                id?: string | undefined;
+            } | {
+                type: "voice_over";
+                startAt?: number | undefined;
+            } | undefined;
+            audio?: {
+                type: "audio";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "midi";
+                source: string;
+            } | undefined;
+            duration?: number | undefined;
+            speechOptions?: {
+                speed?: number | undefined;
+                instruction?: string | undefined;
+            } | undefined;
+            id?: string | undefined;
+            imagePrompt?: string | undefined;
+            speaker?: string | undefined;
+            description?: string | undefined;
+            imageParams?: {
+                provider: string;
+                model?: string | undefined;
+                style?: string | undefined;
+                quality?: string | undefined;
+                moderation?: string | undefined;
+                images?: Record<string, {
+                    type: "image";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                } | {
+                    type: "imagePrompt";
+                    prompt: string;
+                }> | undefined;
+            } | undefined;
+            audioParams?: {
+                movieVolume: number;
+                padding?: number | undefined;
+            } | undefined;
+            movieParams?: {
+                speed?: number | undefined;
+                provider?: string | undefined;
+                model?: string | undefined;
+                fillOption?: {
+                    style: "aspectFit" | "aspectFill";
+                } | undefined;
+            } | undefined;
+            soundEffectParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
+            lipSyncParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
+            htmlImageParams?: {
+                model?: string | undefined;
+            } | undefined;
+            textSlideParams?: {
+                cssStyles: string | string[];
+            } | undefined;
+            captionParams?: {
+                styles: string[];
+                lang?: string | undefined;
+            } | undefined;
+            imageNames?: string[] | undefined;
+            moviePrompt?: string | undefined;
+            soundEffectPrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
+            enableLipSync?: boolean | undefined;
+        }[];
+        title?: string | undefined;
+        description?: string | undefined;
+        lipSyncParams?: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        } | undefined;
+        htmlImageParams?: {
+            provider: string;
+            model?: string | undefined;
+        } | undefined;
+        textSlideParams?: {
+            cssStyles: string | string[];
+        } | undefined;
+        captionParams?: {
+            styles: string[];
+            lang?: string | undefined;
+        } | undefined;
+        references?: {
+            type: string;
+            url: string;
+            title?: string | undefined;
+            description?: string | undefined;
+        }[] | undefined;
+        imagePath?: string | undefined;
+        __test_invalid__?: boolean | undefined;
+    };
+    filename: string;
+};
 export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<MulmoScript | null>;
 export declare const getMultiLingual: (multilingualFilePath: string, studioBeatsLength: number) => MulmoStudioMultiLingual;
 export declare const getPresentationStyle: (presentationStylePath: string | undefined) => MulmoPresentationStyle | null;
-export declare const initializeContextFromFiles: (files: FileObject, raiseError: boolean, force?: boolean, caption?: string, lang?: string) => Promise<{
+export declare const initializeContextFromFiles: (files: FileObject, raiseError: boolean, force?: boolean, captionLang?: string, targetLang?: string) => Promise<{
     studio: {
         beats: {
             duration?: number | undefined;
@@ -21,6 +363,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             captionFile?: string | undefined;
         }[];
         script: {
+            lang: string;
             imageParams: {
                 provider: string;
                 model?: string | undefined;
@@ -317,7 +660,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
                 enableLipSync?: boolean | undefined;
             }[];
-            lang?: string | undefined;
             title?: string | undefined;
             description?: string | undefined;
             lipSyncParams?: {
@@ -346,30 +688,16 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
         };
         filename: string;
     };
+    multiLingual: {
+        multiLingualTexts: Record<string, {
+            text: string;
+            lang: string;
+            texts?: string[] | undefined;
+            ttsTexts?: string[] | undefined;
+            duration?: number | undefined;
+        }>;
+    }[];
     fileDirs: FileObject;
-    force: boolean;
-    lang: string | undefined;
-    sessionState: {
-        inSession: {
-            audio: boolean;
-            image: boolean;
-            video: boolean;
-            multiLingual: boolean;
-            caption: boolean;
-            pdf: boolean;
-        };
-        inBeatSession: {
-            audio: {};
-            image: {};
-            movie: {};
-            multiLingual: {};
-            caption: {};
-            html: {};
-            imageReference: {};
-            soundEffect: {};
-            lipSync: {};
-        };
-    };
     presentationStyle: {
         imageParams: {
             provider: string;
@@ -482,13 +810,27 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             lang?: string | undefined;
         } | undefined;
     };
-    multiLingual: {
-        multiLingualTexts: Record<string, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }>;
-    }[];
+    sessionState: {
+        inSession: {
+            audio: boolean;
+            image: boolean;
+            video: boolean;
+            multiLingual: boolean;
+            caption: boolean;
+            pdf: boolean;
+        };
+        inBeatSession: {
+            audio: {};
+            image: {};
+            movie: {};
+            multiLingual: {};
+            caption: {};
+            html: {};
+            imageReference: {};
+            soundEffect: {};
+            lipSync: {};
+        };
+    };
+    force: boolean;
+    lang: string;
 } | null>;

package/lib/utils/context.js

@@ -1,44 +1,28 @@
 import { GraphAILogger } from "graphai";
 import fs from "fs";
 import { readMulmoScriptFile, fetchMulmoScriptFile } from "./file.js";
-import { createOrUpdateStudioData } from "./preprocess.js";
+import { mulmoStudioSchema, mulmoCaptionParamsSchema } from "../types/index.js";
+import { MulmoPresentationStyleMethods, MulmoScriptMethods } from "../methods/index.js";
 import { mulmoPresentationStyleSchema, mulmoStudioMultiLingualSchema } from "../types/index.js";
-export const fetchScript = async (isHttpPath, mulmoFilePath, fileOrUrl) => {
-    if (isHttpPath) {
-        const res = await fetchMulmoScriptFile(fileOrUrl);
-        if (!res.result || !res.script) {
-            GraphAILogger.info(`ERROR: HTTP error! ${res.status} ${fileOrUrl}`);
-            return null;
-        }
-        return res.script;
-    }
-    if (!fs.existsSync(mulmoFilePath)) {
-        GraphAILogger.info(`ERROR: File not exists ${mulmoFilePath}`);
-        return null;
-    }
-    return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath)?.mulmoData ?? null;
-};
-export const getMultiLingual = (multilingualFilePath, studioBeatsLength) => {
-    if (fs.existsSync(multilingualFilePath)) {
-        const jsonData = readMulmoScriptFile(multilingualFilePath, "ERROR: File does not exist " + multilingualFilePath)?.mulmoData ?? null;
-        const dataSet = mulmoStudioMultiLingualSchema.parse(jsonData);
-        while (dataSet.length < studioBeatsLength) {
-            dataSet.push({ multiLingualTexts: {} });
-        }
-        dataSet.length = studioBeatsLength;
-        return dataSet;
-    }
-    return [...Array(studioBeatsLength)].map(() => ({ multiLingualTexts: {} }));
-};
-export const getPresentationStyle = (presentationStylePath) => {
-    if (presentationStylePath) {
-        if (!fs.existsSync(presentationStylePath)) {
-            throw new Error(`ERROR: File not exists ${presentationStylePath}`);
-        }
-        const jsonData = readMulmoScriptFile(presentationStylePath, "ERROR: File does not exist " + presentationStylePath)?.mulmoData ?? null;
-        return mulmoPresentationStyleSchema.parse(jsonData);
-    }
-    return null;
+const mulmoCredit = (speaker) => {
+    return {
+        speaker,
+        text: "",
+        image: {
+            type: "image",
+            source: {
+                kind: "url",
+                url: "https://github.com/receptron/mulmocast-cli/raw/refs/heads/main/assets/images/mulmocast_credit.png",
+            },
+        },
+        audio: {
+            type: "audio",
+            source: {
+                kind: "url",
+                url: "https://github.com/receptron/mulmocast-cli/raw/refs/heads/main/assets/audio/silent300.mp3",
+            },
+        },
+    };
 };
 const initSessionState = () => {
     return {
@@ -63,32 +47,87 @@ const initSessionState = () => {
         },
     };
 };
-const buildContext = (studio, files, presentationStyle, multiLingual, force, lang) => {
-    return {
-        studio,
-        fileDirs: files,
-        force: Boolean(force),
-        lang,
-        sessionState: initSessionState(),
-        presentationStyle: presentationStyle ?? studio.script,
-        multiLingual,
-    };
+export const createStudioData = (_mulmoScript, fileName, videoCaptionLang, presentationStyle) => {
+    // validate and insert default value
+    const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : MulmoScriptMethods.validate(_mulmoScript);
+    // We need to parse it to fill default values
+    const studio = mulmoStudioSchema.parse({
+        script: mulmoScript,
+        filename: fileName,
+        beats: [...Array(mulmoScript.beats.length)].map(() => ({})),
+    });
+    // TODO: Move this code out of this function later
+    // Addition cloing credit
+    if (mulmoScript.$mulmocast.credit === "closing") {
+        const defaultSpeaker = MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle ?? studio.script);
+        mulmoScript.beats.push(mulmoCredit(mulmoScript.beats[0].speaker ?? defaultSpeaker)); // First speaker
+    }
+    studio.script = MulmoScriptMethods.validate(mulmoScript); // update the script
+    studio.beats = studio.script.beats.map((_, index) => studio.beats[index] ?? {});
+    if (videoCaptionLang) {
+        studio.script.captionParams = mulmoCaptionParamsSchema.parse({
+            ...(studio.script.captionParams ?? {}),
+            lang: videoCaptionLang,
+        });
+    }
+    return studio;
+};
+export const fetchScript = async (isHttpPath, mulmoFilePath, fileOrUrl) => {
+    if (isHttpPath) {
+        const res = await fetchMulmoScriptFile(fileOrUrl);
+        if (!res.result || !res.script) {
+            GraphAILogger.info(`ERROR: HTTP error! ${res.status} ${fileOrUrl}`);
+            return null;
+        }
+        return res.script;
+    }
+    if (!fs.existsSync(mulmoFilePath)) {
+        GraphAILogger.info(`ERROR: File not exists ${mulmoFilePath}`);
+        return null;
+    }
+    return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath)?.mulmoData ?? null;
+};
+export const getMultiLingual = (multilingualFilePath, studioBeatsLength) => {
+    if (!fs.existsSync(multilingualFilePath)) {
+        return [...Array(studioBeatsLength)].map(() => ({ multiLingualTexts: {} }));
+    }
+    const jsonData = readMulmoScriptFile(multilingualFilePath, "ERROR: File does not exist " + multilingualFilePath)?.mulmoData ?? null;
+    const dataSet = mulmoStudioMultiLingualSchema.parse(jsonData);
+    while (dataSet.length < studioBeatsLength) {
+        dataSet.push({ multiLingualTexts: {} });
+    }
+    dataSet.length = studioBeatsLength;
+    return dataSet;
+};
+export const getPresentationStyle = (presentationStylePath) => {
+    if (!presentationStylePath) {
+        return null;
+    }
+    if (!fs.existsSync(presentationStylePath)) {
+        throw new Error(`ERROR: File not exists ${presentationStylePath}`);
+    }
+    const jsonData = readMulmoScriptFile(presentationStylePath, "ERROR: File does not exist " + presentationStylePath)?.mulmoData ?? null;
+    return mulmoPresentationStyleSchema.parse(jsonData);
 };
-export const initializeContextFromFiles = async (files, raiseError, force, caption, lang) => {
-    const { fileName, isHttpPath, fileOrUrl, mulmoFilePath, outputStudioFilePath, presentationStylePath, outputMultilingualFilePath } = files;
-    // read mulmoScript, presentationStyle, currentStudio from files
+export const initializeContextFromFiles = async (files, raiseError, force, captionLang, targetLang) => {
+    const { fileName, isHttpPath, fileOrUrl, mulmoFilePath, presentationStylePath, outputMultilingualFilePath } = files;
     const mulmoScript = await fetchScript(isHttpPath, mulmoFilePath, fileOrUrl);
     if (!mulmoScript) {
         return null;
     }
-    const presentationStyle = getPresentationStyle(presentationStylePath);
-    // Create or update MulmoStudio file with MulmoScript
-    const currentStudio = readMulmoScriptFile(outputStudioFilePath);
     try {
-        // validate mulmoStudioSchema. skip if __test_invalid__ is true
-        const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName, caption, presentationStyle);
+        const presentationStyle = getPresentationStyle(presentationStylePath);
+        const studio = createStudioData(mulmoScript, fileName, captionLang, presentationStyle);
         const multiLingual = getMultiLingual(outputMultilingualFilePath, studio.beats.length);
-        return buildContext(studio, files, presentationStyle, multiLingual, force, lang);
+        return {
+            studio,
+            multiLingual,
+            fileDirs: files,
+            presentationStyle: presentationStyle ?? studio.script,
+            sessionState: initSessionState(),
+            force: Boolean(force),
+            lang: targetLang ?? studio.script.lang, // This lang is target Language. studio.lang is default Language
+        };
     }
     catch (error) {
         GraphAILogger.info(`Error: invalid MulmoScript Schema: ${isHttpPath ? fileOrUrl : mulmoFilePath} \n ${error}`);

package/lib/utils/filters.d.ts

@@ -1,4 +1,5 @@
 import "dotenv/config";
 import type { AgentFilterFunction } from "graphai";
+export declare const nijovoiceTextAgentFilter: AgentFilterFunction;
 export declare const fileCacheAgentFilter: AgentFilterFunction;
 export declare const browserlessCacheGenerator: (cacheDir: string) => AgentFilterFunction;

package/lib/utils/filters.js

@@ -6,6 +6,14 @@ import { GraphAILogger } from "graphai";
 import { writingMessage } from "./file.js";
 import { text2hash } from "./utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
+import { replacementsJa, replacePairsJa } from "../utils/string.js";
+export const nijovoiceTextAgentFilter = async (context, next) => {
+    const { text, provider, lang } = context.namedInputs;
+    if (provider === "nijivoice" && lang === "ja") {
+        context.namedInputs.text = replacePairsJa(replacementsJa)(text);
+    }
+    return next(context);
+};
 export const fileCacheAgentFilter = async (context, next) => {
     const { force, file, index, mulmoContext, sessionType } = context.namedInputs.cache;
     const shouldUseCache = async () => {

package/lib/utils/preprocess.d.ts

@@ -1,5 +1,5 @@
 import { MulmoStudio, MulmoScript, MulmoPresentationStyle } from "../types/index.js";
-export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string, presentationStyle?: MulmoPresentationStyle | null) => {
+export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaptionLang?: string, presentationStyle?: MulmoPresentationStyle | null) => {
     beats: {
         duration?: number | undefined;
         startAt?: number | undefined;
@@ -16,6 +16,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         captionFile?: string | undefined;
     }[];
     script: {
+        lang: string;
         imageParams: {
             provider: string;
             model?: string | undefined;
@@ -312,7 +313,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
             enableLipSync?: boolean | undefined;
         }[];
-        lang?: string | undefined;
         title?: string | undefined;
         description?: string | undefined;
         lipSyncParams?: {

package/lib/utils/preprocess.js

@@ -37,7 +37,7 @@ const mulmoCredit = (speaker) => {
         },
     };
 };
-export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaption, presentationStyle) => {
+export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaptionLang, presentationStyle) => {
     const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : MulmoScriptMethods.validate(_mulmoScript); // validate and insert default value
     const studio = rebuildStudio(currentStudio, mulmoScript, fileName);
     // TODO: Move this code out of this function later
@@ -48,10 +48,10 @@ export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName,
     }
     studio.script = MulmoScriptMethods.validate(mulmoScript); // update the script
     studio.beats = studio.script.beats.map((_, index) => studio.beats[index] ?? {});
-    if (videoCaption) {
+    if (videoCaptionLang) {
         studio.script.captionParams = mulmoCaptionParamsSchema.parse({
             ...(studio.script.captionParams ?? {}),
-            lang: videoCaption,
+            lang: videoCaptionLang,
         });
     }
     return studio;