mulmocast 2.9.0 → 2.9.2

package/lib/utils/image_preprocess.js

@@ -0,0 +1,52 @@
+import { MulmoStudioContextMethods } from "../methods/index.js";
+import { getHTMLFile } from "./file.js";
+import { renderMarkdownToImage, renderHTMLToImage, interpolate } from "./markdown.js";
+export const processTextSlide = async (params) => {
+    const { beat, imagePath, textSlideStyle, canvasSize } = params;
+    if (!beat.image || beat.image.type !== "textSlide")
+        return;
+    const slide = beat.image.slide;
+    const markdown = `# ${slide.title}\n` + slide.bullets.map((text) => `- ${text}`).join("\n");
+    await renderMarkdownToImage(markdown, textSlideStyle, imagePath, canvasSize.width, canvasSize.height);
+};
+export const processMarkdown = async (params) => {
+    const { beat, imagePath, textSlideStyle, canvasSize } = params;
+    if (!beat.image || beat.image.type !== "markdown")
+        return;
+    const markdown = Array.isArray(beat.image.markdown) ? beat.image.markdown.join("\n") : beat.image.markdown;
+    await renderMarkdownToImage(markdown, textSlideStyle, imagePath, canvasSize.width, canvasSize.height);
+};
+export const processImage = (params) => {
+    const { beat, context } = params;
+    if (!beat.image || beat.image.type !== "image")
+        return;
+    if (beat.image.source.kind === "url") {
+        return beat.image.source.url;
+    }
+    else if (beat.image.source.kind === "path") {
+        return MulmoStudioContextMethods.resolveAssetPath(context, beat.image.source.path);
+    }
+};
+export const processChart = async (params) => {
+    const { beat, imagePath, canvasSize } = params;
+    if (!beat.image || beat.image.type !== "chart")
+        return;
+    const template = getHTMLFile("chart");
+    const htmlData = interpolate(template, {
+        title: beat.image.title,
+        width: Math.round(canvasSize.width * 0.625).toString(),
+        chart_data: JSON.stringify(beat.image.chartData),
+    });
+    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+};
+export const processMermaid = async (params) => {
+    const { beat, imagePath, canvasSize } = params;
+    if (!beat.image || beat.image.type !== "mermaid")
+        return;
+    const template = getHTMLFile("mermaid");
+    const htmlData = interpolate(template, {
+        title: beat.image.title,
+        diagram_code: beat.image.code,
+    });
+    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+};

package/lib/utils/markdown.d.ts

@@ -0,0 +1,3 @@
+export declare const renderHTMLToImage: (html: string, outputPath: string, width: number, height: number, isMermaid?: boolean, omitBackground?: boolean) => Promise<void>;
+export declare const renderMarkdownToImage: (markdown: string, style: string, outputPath: string, width: number, height: number) => Promise<void>;
+export declare const interpolate: (template: string, data: Record<string, string>) => string;

package/lib/utils/markdown.js

@@ -0,0 +1,42 @@
+import { marked } from "marked";
+import puppeteer from "puppeteer";
+const isCI = process.env.CI === "true";
+export const renderHTMLToImage = async (html, outputPath, width, height, isMermaid = false, omitBackground = false) => {
+    // Use Puppeteer to render HTML to an image
+    const browser = await puppeteer.launch({
+        args: isCI ? ["--no-sandbox"] : [],
+    });
+    const page = await browser.newPage();
+    // Set the page content to the HTML generated from the Markdown
+    await page.setContent(html);
+    // Adjust page settings if needed (like width, height, etc.)
+    await page.setViewport({ width, height });
+    await page.addStyleTag({ content: "html,body{margin:0;padding:0;overflow:hidden}" });
+    if (isMermaid) {
+        await page.waitForFunction(() => {
+            const el = document.querySelector(".mermaid");
+            return el && el.dataset.ready === "true";
+        }, { timeout: 20000 });
+    }
+    // Measure the size of the page and scale the page to the width and height
+    await page.evaluate(({ vw, vh }) => {
+        const de = document.documentElement;
+        const sw = Math.max(de.scrollWidth, document.body.scrollWidth || 0);
+        const sh = Math.max(de.scrollHeight, document.body.scrollHeight || 0);
+        const scale = Math.min(vw / (sw || vw), vh / (sh || vh), 1); // <=1 で縮小のみ
+        de.style.overflow = "hidden";
+        document.body.style.zoom = String(scale);
+    }, { vw: width, vh: height });
+    // Step 3: Capture screenshot of the page (which contains the Markdown-rendered HTML)
+    await page.screenshot({ path: outputPath, omitBackground });
+    await browser.close();
+};
+export const renderMarkdownToImage = async (markdown, style, outputPath, width, height) => {
+    const header = `<head><style>${style}</style></head>`;
+    const body = await marked(markdown);
+    const html = `<html>${header}<body>${body}</body></html>`;
+    await renderHTMLToImage(html, outputPath, width, height);
+};
+export const interpolate = (template, data) => {
+    return template.replace(/\$\{(.*?)\}/g, (_, key) => data[key.trim()] ?? "");
+};

package/lib/utils/pdf.d.ts

@@ -0,0 +1,9 @@
+import { PDFFont } from "pdf-lib";
+export declare const fontSize = 12;
+export declare const textMargin = 6;
+export declare const drawSize: (fitWidth: boolean, expectWidth: number, expectHeight: number, origWidth: number, origHeight: number) => {
+    containerWidth: number;
+    drawWidth: number;
+    drawHeight: number;
+};
+export declare const wrapText: (text: string, font: PDFFont, fontSize: number, maxWidth: number) => string[];

package/lib/utils/pdf.js

@@ -0,0 +1,77 @@
+export const fontSize = 12;
+export const textMargin = 6;
+export const drawSize = (fitWidth, expectWidth, expectHeight, origWidth, origHeight) => {
+    if (fitWidth) {
+        const drawWidth = expectWidth;
+        const scale = drawWidth / origWidth;
+        const drawHeight = origHeight * scale;
+        return {
+            containerWidth: expectWidth,
+            drawWidth,
+            drawHeight,
+        };
+    }
+    const drawHeight = expectHeight;
+    const scale = drawHeight / origHeight;
+    const drawWidth = origWidth * scale;
+    return {
+        containerWidth: expectWidth,
+        drawWidth,
+        drawHeight,
+    };
+};
+const isFullwidth = (char) => {
+    const code = char.charCodeAt(0);
+    return ((code >= 0x3000 && code <= 0x9fff) || // CJK, punctuation, kana
+        (code >= 0xff00 && code <= 0xffef) // Fullwidth ASCII
+    );
+};
+export const wrapText = (text, font, fontSize, maxWidth) => {
+    const lines = [];
+    for (const rawLine of text.split("\n")) {
+        let line = "";
+        let buffer = "";
+        for (let i = 0; i < rawLine.length; i++) {
+            const char = rawLine[i];
+            buffer += char;
+            const width = font.widthOfTextAtSize(line + buffer, fontSize);
+            const nextChar = rawLine[i + 1];
+            const currentIsFull = isFullwidth(char);
+            const nextIsFull = nextChar && isFullwidth(nextChar);
+            const isBreakable = currentIsFull || (!currentIsFull && (char === " " || nextChar === " " || nextIsFull));
+            if (width > maxWidth && line) {
+                lines.push(line);
+                line = buffer;
+                buffer = "";
+            }
+            if (isBreakable || i === rawLine.length - 1) {
+                line += buffer;
+                buffer = "";
+            }
+        }
+        if (line)
+            lines.push(line);
+    }
+    return lines;
+};
+/*
+export const wrapText = (text: string, font: PDFFont, fontSize: number, maxWidth: number) => {
+  const words = text.split(" ");
+  const lines = [];
+  let currentLine = "";
+
+  for (const word of words) {
+    const nextLine = currentLine ? `${currentLine} ${word}` : word;
+    // console.log(nextLine);
+    const width = font.widthOfTextAtSize(nextLine ?? "", fontSize);
+    if (width <= maxWidth) {
+      currentLine = nextLine;
+    } else {
+      lines.push(currentLine);
+      currentLine = word;
+    }
+  }
+  if (currentLine) lines.push(currentLine);
+  return lines;
+};
+*/

package/lib/utils/preprocess.d.ts

@@ -0,0 +1,343 @@
+import { MulmoStudio, MulmoScript, MulmoPresentationStyle } from "../types/index.js";
+export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaptionLang?: string, presentationStyle?: MulmoPresentationStyle | null) => {
+    beats: {
+        duration?: number | undefined;
+        startAt?: number | undefined;
+        hash?: string | undefined;
+        audioDuration?: number | undefined;
+        movieDuration?: number | undefined;
+        silenceDuration?: number | undefined;
+        hasMovieAudio?: boolean | undefined;
+        audioFile?: string | undefined;
+        imageFile?: string | undefined;
+        movieFile?: string | undefined;
+        soundEffectFile?: string | undefined;
+        lipSyncFile?: string | undefined;
+        captionFile?: string | undefined;
+    }[];
+    script: {
+        lang: string;
+        imageParams: {
+            provider: string;
+            model?: string | undefined;
+            style?: string | undefined;
+            quality?: string | undefined;
+            moderation?: string | undefined;
+            images?: Record<string, {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "imagePrompt";
+                prompt: string;
+            }> | undefined;
+        };
+        audioParams: {
+            padding: number;
+            introPadding: number;
+            closingPadding: number;
+            outroPadding: number;
+            bgmVolume: number;
+            audioVolume: number;
+            suppressSpeech: boolean;
+            bgm?: {
+                url: string;
+                kind: "url";
+            } | {
+                kind: "base64";
+                data: string;
+            } | {
+                text: string;
+                kind: "text";
+            } | {
+                path: string;
+                kind: "path";
+            } | undefined;
+        };
+        movieParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+            fillOption?: {
+                style: "aspectFit" | "aspectFill";
+            } | undefined;
+            transition?: {
+                type: "fade" | "slideout_left";
+                duration: number;
+            } | undefined;
+        };
+        soundEffectParams: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        };
+        $mulmocast: {
+            version: "1.1";
+            credit?: "closing" | undefined;
+        };
+        canvasSize: {
+            width: number;
+            height: number;
+        };
+        speechParams: {
+            speakers: Record<string, {
+                voiceId: string;
+                lang?: Record<string, {
+                    voiceId: string;
+                    displayName?: Record<string, string> | undefined;
+                    isDefault?: boolean | undefined;
+                    speechOptions?: {
+                        speed?: number | undefined;
+                        instruction?: string | undefined;
+                    } | undefined;
+                    provider?: string | undefined;
+                    model?: string | undefined;
+                }> | undefined;
+                displayName?: Record<string, string> | undefined;
+                isDefault?: boolean | undefined;
+                speechOptions?: {
+                    speed?: number | undefined;
+                    instruction?: string | undefined;
+                } | undefined;
+                provider?: string | undefined;
+                model?: string | undefined;
+            }>;
+        };
+        beats: {
+            text: string;
+            image?: {
+                type: "markdown";
+                markdown: string | string[];
+            } | {
+                type: "web";
+                url: string;
+            } | {
+                type: "pdf";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "image";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "svg";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "movie";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "textSlide";
+                slide: {
+                    title: string;
+                    subtitle?: string | undefined;
+                    bullets?: string[] | undefined;
+                };
+            } | {
+                type: "chart";
+                title: string;
+                chartData: Record<string, any>;
+            } | {
+                code: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+                type: "mermaid";
+                title: string;
+                appendix?: string[] | undefined;
+            } | {
+                type: "html_tailwind";
+                html: string | string[];
+            } | {
+                type: "beat";
+                id?: string | undefined;
+            } | {
+                type: "voice_over";
+                startAt?: number | undefined;
+            } | undefined;
+            audio?: {
+                type: "audio";
+                source: {
+                    url: string;
+                    kind: "url";
+                } | {
+                    kind: "base64";
+                    data: string;
+                } | {
+                    text: string;
+                    kind: "text";
+                } | {
+                    path: string;
+                    kind: "path";
+                };
+            } | {
+                type: "midi";
+                source: string;
+            } | undefined;
+            duration?: number | undefined;
+            speechOptions?: {
+                speed?: number | undefined;
+                instruction?: string | undefined;
+            } | undefined;
+            id?: string | undefined;
+            imagePrompt?: string | undefined;
+            speaker?: string | undefined;
+            description?: string | undefined;
+            imageParams?: {
+                provider: string;
+                model?: string | undefined;
+                style?: string | undefined;
+                quality?: string | undefined;
+                moderation?: string | undefined;
+                images?: Record<string, {
+                    type: "image";
+                    source: {
+                        url: string;
+                        kind: "url";
+                    } | {
+                        kind: "base64";
+                        data: string;
+                    } | {
+                        text: string;
+                        kind: "text";
+                    } | {
+                        path: string;
+                        kind: "path";
+                    };
+                } | {
+                    type: "imagePrompt";
+                    prompt: string;
+                }> | undefined;
+            } | undefined;
+            audioParams?: {
+                movieVolume: number;
+                padding?: number | undefined;
+            } | undefined;
+            movieParams?: {
+                speed?: number | undefined;
+                provider?: string | undefined;
+                model?: string | undefined;
+                fillOption?: {
+                    style: "aspectFit" | "aspectFill";
+                } | undefined;
+            } | undefined;
+            soundEffectParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
+            lipSyncParams?: {
+                provider?: string | undefined;
+                model?: string | undefined;
+            } | undefined;
+            htmlImageParams?: {
+                model?: string | undefined;
+            } | undefined;
+            textSlideParams?: {
+                cssStyles: string | string[];
+            } | undefined;
+            captionParams?: {
+                styles: string[];
+                lang?: string | undefined;
+            } | undefined;
+            imageNames?: string[] | undefined;
+            moviePrompt?: string | undefined;
+            soundEffectPrompt?: string | undefined;
+            htmlPrompt?: {
+                prompt: string;
+                data?: any;
+                images?: Record<string, any> | undefined;
+                systemPrompt?: string | undefined;
+            } | undefined;
+            enableLipSync?: boolean | undefined;
+        }[];
+        title?: string | undefined;
+        description?: string | undefined;
+        lipSyncParams?: {
+            provider?: string | undefined;
+            model?: string | undefined;
+        } | undefined;
+        htmlImageParams?: {
+            provider: string;
+            model?: string | undefined;
+        } | undefined;
+        textSlideParams?: {
+            cssStyles: string | string[];
+        } | undefined;
+        captionParams?: {
+            styles: string[];
+            lang?: string | undefined;
+        } | undefined;
+        references?: {
+            type: string;
+            url: string;
+            title?: string | undefined;
+            description?: string | undefined;
+        }[] | undefined;
+        imagePath?: string | undefined;
+        __test_invalid__?: boolean | undefined;
+    };
+    filename: string;
+};

package/lib/utils/preprocess.js

@@ -0,0 +1,58 @@
+import { GraphAILogger } from "graphai";
+import { mulmoStudioSchema, mulmoCaptionParamsSchema } from "../types/index.js";
+import { MulmoPresentationStyleMethods, MulmoScriptMethods } from "../methods/index.js";
+const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
+    const isTest = process.env.NODE_ENV === "test";
+    const parsed = isTest && currentStudio ? { data: mulmoStudioSchema.parse(currentStudio), success: true, error: null } : mulmoStudioSchema.safeParse(currentStudio);
+    if (parsed.success) {
+        return parsed.data;
+    }
+    if (currentStudio) {
+        GraphAILogger.info("currentStudio is invalid", parsed.error);
+    }
+    // We need to parse it to fill default values
+    return mulmoStudioSchema.parse({
+        script: mulmoScript,
+        filename: fileName,
+        beats: [...Array(mulmoScript.beats.length)].map(() => ({})),
+    });
+};
+const mulmoCredit = (speaker) => {
+    return {
+        speaker,
+        text: "",
+        image: {
+            type: "image",
+            source: {
+                kind: "url",
+                url: "https://github.com/receptron/mulmocast-cli/raw/refs/heads/main/assets/images/mulmocast_credit.png",
+            },
+        },
+        audio: {
+            type: "audio",
+            source: {
+                kind: "url",
+                url: "https://github.com/receptron/mulmocast-cli/raw/refs/heads/main/assets/audio/silent300.mp3",
+            },
+        },
+    };
+};
+export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName, videoCaptionLang, presentationStyle) => {
+    const mulmoScript = _mulmoScript.__test_invalid__ ? _mulmoScript : MulmoScriptMethods.validate(_mulmoScript); // validate and insert default value
+    const studio = rebuildStudio(currentStudio, mulmoScript, fileName);
+    // TODO: Move this code out of this function later
+    // Addition cloing credit
+    if (mulmoScript.$mulmocast.credit === "closing") {
+        const defaultSpeaker = MulmoPresentationStyleMethods.getDefaultSpeaker(presentationStyle ?? studio.script);
+        mulmoScript.beats.push(mulmoCredit(mulmoScript.beats[0].speaker ?? defaultSpeaker)); // First speaker
+    }
+    studio.script = MulmoScriptMethods.validate(mulmoScript); // update the script
+    studio.beats = studio.script.beats.map((_, index) => studio.beats[index] ?? {});
+    if (videoCaptionLang) {
+        studio.script.captionParams = mulmoCaptionParamsSchema.parse({
+            ...(studio.script.captionParams ?? {}),
+            lang: videoCaptionLang,
+        });
+    }
+    return studio;
+};