mulmocast 0.0.4 → 0.0.6

package/lib/types/schema.js

@@ -4,7 +4,7 @@ const URLStringSchema = z.string().url();
 export const localizedTextSchema = z
     .object({
     text: z.string(),
-    lang: z.string(),
+    lang: langSchema,
     // caption: z.string(),
     texts: z.array(z.string()).optional(),
     ttsTexts: z.array(z.string()).optional(),
@@ -77,7 +77,8 @@ export const mulmoTextSlideMediaSchema = z
     type: z.literal("textSlide"),
     slide: z.object({
         title: z.string(),
-        bullets: z.array(z.string()),
+        subtitle: z.string().optional(),
+        bullets: z.array(z.string()).optional(),
     }),
 })
     .strict();
@@ -96,6 +97,12 @@ export const mulmoMermaidMediaSchema = z
     appendix: z.array(z.string()).optional().describe("The appendix of the mermaid diagram; typically, style information."),
 })
     .strict();
+export const mulmoHtmlTailwindMediaSchema = z
+    .object({
+    type: z.literal("html_tailwind"),
+    html: stringOrStringArray,
+})
+    .strict();
 export const mulmoImageAssetSchema = z.union([
     mulmoMarkdownMediaSchema,
     mulmoWebMediaSchema,
@@ -106,6 +113,7 @@ export const mulmoImageAssetSchema = z.union([
     mulmoTextSlideMediaSchema,
     mulmoChartMediaSchema,
     mulmoMermaidMediaSchema,
+    mulmoHtmlTailwindMediaSchema,
 ]);
 const mulmoAudioMediaSchema = z
     .object({
@@ -133,17 +141,28 @@ export const textSlideParamsSchema = z
     cssStyles: stringOrStringArray,
 })
     .strict();
+/* TODO: Add something later
 export const videoParamsSchema = z
-    .object({
+  .object({
     padding: z.number().optional(), // msec
+  })
+  .strict();
+*/
+export const audioParamsSchema = z
+    .object({
+    introPadding: z.number().describe("Padding at the beginning of the audio"), // seconds
+    padding: z.number().describe("Padding between beats"), // seconds
+    closingPadding: z.number().describe("Padding before the last beat"), // seconds
+    outroPadding: z.number().describe("Padding at the end of the audio"), // seconds
 })
     .strict();
 export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.default("Presenter"),
-    text: z.string(),
+    text: z.string().describe("Text to be spoken. If empty, the audio is not generated."),
     image: mulmoImageAssetSchema.optional(),
     audio: mulmoAudioAssetSchema.optional(),
+    duration: z.number().optional().describe("Duration of the beat. Used only when the text is empty"),
     imageParams: mulmoImageParamsSchema.optional(), // beat specific parameters
     speechOptions: speechOptionsSchema.optional(),
     textSlideParams: textSlideParamsSchema.optional(),
@@ -163,7 +182,7 @@ export const mulmoCastCreditSchema = z
     credit: z.literal("closing").optional(),
 })
     .strict();
-export const text2SpeechProviderSchema = z.union([z.literal("openai"), z.literal("nijivoice")]).default("openai");
+export const text2SpeechProviderSchema = z.union([z.literal("openai"), z.literal("nijivoice"), z.literal("google")]).default("openai");
 export const mulmoSpeechParamsSchema = z
     .object({
     provider: text2SpeechProviderSchema, // has default value
@@ -191,7 +210,13 @@ export const mulmoPresentationStyleSchema = z.object({
         .optional(),
     // for textSlides
     textSlideParams: textSlideParamsSchema.optional(),
-    videoParams: videoParamsSchema.optional(),
+    // videoParams: videoParamsSchema.optional(),
+    audioParams: audioParamsSchema.default({
+        introPadding: 1.0,
+        padding: 0.3,
+        closingPadding: 0.8,
+        outroPadding: 1.0,
+    }),
     // TODO: Switch to showCaptions later
     omitCaptions: z.boolean().optional(), // default is false
 });
@@ -216,18 +241,23 @@ export const mulmoScriptSchema = mulmoPresentationStyleSchema
     .strict();
 export const mulmoStudioBeatSchema = z
     .object({
-    multiLingualTexts: multiLingualTextsSchema.optional(),
     hash: z.string().optional(),
     duration: z.number().optional(),
     audioFile: z.string().optional(),
     imageFile: z.string().optional(), // path to the image
+    captionFile: z.string().optional(), // path to the caption image
 })
     .strict();
+export const mulmoStudioMultiLingualDataSchema = z.object({
+    multiLingualTexts: multiLingualTextsSchema,
+});
+export const mulmoStudioMultiLingualSchema = z.array(mulmoStudioMultiLingualDataSchema).min(1);
 export const mulmoStudioSchema = z
     .object({
     script: mulmoScriptSchema,
     filename: z.string(),
     beats: z.array(mulmoStudioBeatSchema).min(1),
+    multiLingual: mulmoStudioMultiLingualSchema,
 })
     .strict();
 export const mulmoScriptTemplateSchema = z

package/lib/types/type.d.ts

@@ -1,4 +1,4 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, text2ImageProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, text2ImageProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes } from "../utils/const.js";
 import { z } from "zod";
 export type LANG = z.infer<typeof langSchema>;
@@ -20,6 +20,9 @@ export type MulmoStudioBeat = z.infer<typeof mulmoStudioBeatSchema>;
 export type MulmoMediaSource = z.infer<typeof mediaSourceSchema>;
 export type MulmoStudio = z.infer<typeof mulmoStudioSchema>;
 export type MulmoScriptTemplate = z.infer<typeof mulmoScriptTemplateSchema>;
+export type MulmoStudioMultiLingual = z.infer<typeof mulmoStudioMultiLingualSchema>;
+export type MulmoStudioMultiLingualData = z.infer<typeof mulmoStudioMultiLingualDataSchema>;
+export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
 export type MulmoTextSlideMedia = z.infer<typeof mulmoTextSlideMediaSchema>;
 export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
@@ -36,7 +39,9 @@ export type FileDirs = {
 export type MulmoStudioContext = {
     fileDirs: FileDirs;
     studio: MulmoStudio;
+    lang?: string;
     force: boolean;
+    caption?: string;
 };
 export type ScriptingParams = {
     urls: string[];

package/lib/utils/const.d.ts

@@ -4,3 +4,4 @@ export declare const imageDirName = "images";
 export declare const cacheDirName = "cache";
 export declare const pdf_modes: string[];
 export declare const pdf_sizes: string[];
+export declare const languages: string[];

package/lib/utils/const.js

@@ -4,3 +4,4 @@ export const imageDirName = "images";
 export const cacheDirName = "cache";
 export const pdf_modes = ["slide", "talk", "handout"];
 export const pdf_sizes = ["letter", "a4"];
+export const languages = ["en", "ja"];

package/lib/utils/ffmpeg_utils.d.ts

@@ -0,0 +1,12 @@
+import ffmpeg from "fluent-ffmpeg";
+export type FfmpegContext = {
+    command: ffmpeg.FfmpegCommand;
+    inputCount: number;
+    filterComplex: string[];
+};
+export declare const FfmpegContextInit: () => FfmpegContext;
+export declare const FfmpegContextAddInput: (context: FfmpegContext, input: string) => number;
+export declare const FfmpegContextPushFormattedAudio: (context: FfmpegContext, sourceId: string, outputId: string, duration?: number | undefined) => void;
+export declare const FfmpegContextInputFormattedAudio: (context: FfmpegContext, input: string, duration?: number | undefined) => string;
+export declare const FfmpegContextGenerateOutput: (context: FfmpegContext, output: string, options?: string[]) => Promise<number>;
+export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<number>;

package/lib/utils/ffmpeg_utils.js

@@ -0,0 +1,63 @@
+import ffmpeg from "fluent-ffmpeg";
+import { GraphAILogger } from "graphai";
+export const FfmpegContextInit = () => {
+    return {
+        command: ffmpeg(),
+        inputCount: 0,
+        filterComplex: [],
+    };
+};
+export const FfmpegContextAddInput = (context, input) => {
+    context.command.input(input);
+    context.inputCount++;
+    return context.inputCount - 1; // returned the index of the input
+};
+export const FfmpegContextPushFormattedAudio = (context, sourceId, outputId, duration = undefined) => {
+    if (duration !== undefined) {
+        context.filterComplex.push(`${sourceId}atrim=duration=${duration},aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo${outputId}`);
+    }
+    else {
+        context.filterComplex.push(`${sourceId}aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo${outputId}`);
+    }
+};
+export const FfmpegContextInputFormattedAudio = (context, input, duration = undefined) => {
+    const index = FfmpegContextAddInput(context, input);
+    const audioId = `[a${index}]`;
+    FfmpegContextPushFormattedAudio(context, `[${index}:a]`, audioId, duration);
+    return audioId;
+};
+export const FfmpegContextGenerateOutput = (context, output, options = []) => {
+    return new Promise((resolve, reject) => {
+        context.command
+            .complexFilter(context.filterComplex)
+            .outputOptions(options)
+            .output(output)
+            .on("start", (__cmdLine) => {
+            GraphAILogger.log("Started FFmpeg ..."); // with command:', cmdLine);
+        })
+            .on("error", (err, stdout, stderr) => {
+            GraphAILogger.error("Error occurred:", err);
+            GraphAILogger.error("FFmpeg stdout:", stdout);
+            GraphAILogger.error("FFmpeg stderr:", stderr);
+            GraphAILogger.info("Video/Audio creation failed. An unexpected error occurred.");
+            reject();
+        })
+            .on("end", () => {
+            resolve(0);
+        })
+            .run();
+    });
+};
+export const ffmpegGetMediaDuration = (filePath) => {
+    return new Promise((resolve, reject) => {
+        ffmpeg.ffprobe(filePath, (err, metadata) => {
+            if (err) {
+                GraphAILogger.info("Error while getting metadata:", err);
+                reject(err);
+            }
+            else {
+                resolve(metadata.format.duration);
+            }
+        });
+    });
+};

package/lib/utils/file.d.ts

@@ -1,5 +1,6 @@
 import { MulmoScript, MulmoScriptTemplate, MulmoMediaSource, MulmoStudioContext } from "../types/index.js";
 import { PDFMode } from "../types/index.js";
+import { ZodSchema } from "zod";
 export declare function readMulmoScriptFile<T = MulmoScript>(path: string, errorMessage: string): {
     mulmoData: T;
     mulmoDataPath: string;
@@ -24,16 +25,18 @@ export declare const fetchMulmoScriptFile: (url: string) => Promise<{
     script?: undefined;
 }>;
 export declare const getOutputStudioFilePath: (outDirPath: string, fileName: string) => string;
+export declare const resolveDirPath: (dirPath: string, studioFileName: string) => string;
 export declare const getAudioSegmentDirPath: (audioDirPath: string, studioFileName: string) => string;
 export declare const getAudioSegmentFilePath: (audioDirPath: string, studioFileName: string, fileName: string) => string;
-export declare const getAudioCombinedFilePath: (audioDirPath: string, fileName: string) => string;
+export declare const getAudioCombinedFilePath: (audioDirPath: string, fileName: string, lang?: string) => string;
 export declare const getAudioArtifactFilePath: (outDirPath: string, fileName: string) => string;
-export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string) => string;
-export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode) => string;
+export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string, lang?: string, caption?: string) => string;
+export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;
 export declare const getTemplateFilePath: (templateName: string) => string;
 export declare const mkdir: (dirPath: string) => void;
 export declare const silentPath: string;
 export declare const silentLastPath: string;
+export declare const silent60secPath: string;
 export declare const defaultBGMPath: string;
 export declare const getHTMLFile: (filename: string) => string;
 export declare const getBaseDirPath: (basedir?: string) => string;
@@ -44,3 +47,4 @@ export declare const getAvailableTemplates: () => (MulmoScriptTemplate & {
 })[];
 export declare const writingMessage: (filePath: string) => void;
 export declare const resolveMediaSource: (source: MulmoMediaSource, context: MulmoStudioContext) => string | null;
+export declare const readAndParseJson: <S extends ZodSchema<any>>(filePath: string, schema: S) => ReturnType<S["parse"]>;

package/lib/utils/file.js

@@ -45,23 +45,35 @@ export const fetchMulmoScriptFile = async (url) => {
 export const getOutputStudioFilePath = (outDirPath, fileName) => {
     return path.resolve(outDirPath, fileName + "_studio.json");
 };
+export const resolveDirPath = (dirPath, studioFileName) => {
+    return path.resolve(dirPath, studioFileName);
+};
+// TODO: probably better to just use resolveDirPath instead.
 export const getAudioSegmentDirPath = (audioDirPath, studioFileName) => {
     return path.resolve(audioDirPath, studioFileName);
 };
 export const getAudioSegmentFilePath = (audioDirPath, studioFileName, fileName) => {
     return path.resolve(getAudioSegmentDirPath(audioDirPath, studioFileName), fileName + ".mp3");
 };
-export const getAudioCombinedFilePath = (audioDirPath, fileName) => {
+export const getAudioCombinedFilePath = (audioDirPath, fileName, lang) => {
+    if (lang) {
+        return path.resolve(audioDirPath, fileName, `${fileName}_${lang}.mp3`);
+    }
     return path.resolve(audioDirPath, fileName, fileName + ".mp3");
 };
 export const getAudioArtifactFilePath = (outDirPath, fileName) => {
     return path.resolve(outDirPath, fileName + ".mp3");
 };
-export const getOutputVideoFilePath = (outDirPath, fileName) => {
-    return path.resolve(outDirPath, fileName + ".mp4");
+export const getOutputVideoFilePath = (outDirPath, fileName, lang, caption) => {
+    const suffix = lang ? `_${lang}` : "";
+    const suffix2 = caption ? `__${caption}` : "";
+    return path.resolve(outDirPath, `${fileName}${suffix}${suffix2}.mp4`);
 };
-export const getOutputPdfFilePath = (outDirPath, fileName, pdfMode) => {
-    return path.resolve(outDirPath, fileName + "_" + pdfMode + ".pdf");
+export const getOutputPdfFilePath = (outDirPath, fileName, pdfMode, lang) => {
+    if (lang) {
+        return path.resolve(outDirPath, `${fileName}_${pdfMode}_${lang}.pdf`);
+    }
+    return path.resolve(outDirPath, `${fileName}_${pdfMode}.pdf`);
 };
 export const getTemplateFilePath = (templateName) => {
     return path.resolve(__dirname, "../../assets/templates/" + templateName + ".json");
@@ -74,6 +86,7 @@ export const mkdir = (dirPath) => {
 };
 export const silentPath = path.resolve(__dirname, "../../assets/audio/silent300.mp3");
 export const silentLastPath = path.resolve(__dirname, "../../assets/audio/silent800.mp3");
+export const silent60secPath = path.resolve(__dirname, "../../assets/audio/silent60sec.mp3");
 export const defaultBGMPath = path.resolve(__dirname, "../../assets/music/StarsBeyondEx.mp3");
 export const getHTMLFile = (filename) => {
     const htmlPath = path.resolve(__dirname, `../../assets/html/${filename}.html`);
@@ -131,3 +144,9 @@ export const resolveMediaSource = (source, context) => {
     }
     return null;
 };
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const readAndParseJson = (filePath, schema) => {
+    const fileContent = fs.readFileSync(filePath, "utf-8");
+    const json = JSON.parse(fileContent);
+    return schema.parse(json);
+};

package/lib/utils/image_plugins/chart.js

@@ -5,11 +5,16 @@ const processChart = async (params) => {
     const { beat, imagePath, canvasSize, textSlideStyle } = params;
     if (!beat.image || beat.image.type !== imageType)
         return;
+    const isCircular = beat.image.chartData.type === "pie" ||
+        beat.image.chartData.type === "doughnut" ||
+        beat.image.chartData.type === "polarArea" ||
+        beat.image.chartData.type === "radar";
+    const chart_width = isCircular ? Math.min(canvasSize.width, canvasSize.height) * 0.75 : canvasSize.width * 0.75;
     const template = getHTMLFile("chart");
     const htmlData = interpolate(template, {
         title: beat.image.title,
         style: textSlideStyle,
-        width: Math.round(canvasSize.width * 0.625).toString(),
+        chart_width: chart_width.toString(),
         chart_data: JSON.stringify(beat.image.chartData),
     });
     await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);

package/lib/utils/image_plugins/html_tailwind.d.ts

@@ -0,0 +1,3 @@
+import { ImageProcessorParams } from "../../types/index.js";
+export declare const imageType = "html_tailwind";
+export declare const process: (params: ImageProcessorParams) => Promise<string | undefined>;

package/lib/utils/image_plugins/html_tailwind.js

@@ -0,0 +1,18 @@
+import { getHTMLFile } from "../file.js";
+import { renderHTMLToImage, interpolate } from "../markdown.js";
+export const imageType = "html_tailwind";
+const processHtmlTailwind = async (params) => {
+    const { beat, imagePath, canvasSize } = params;
+    if (!beat.image || beat.image.type !== imageType)
+        return;
+    const html = Array.isArray(beat.image.html) ? beat.image.html.join("\n") : beat.image.html;
+    const template = getHTMLFile("tailwind");
+    const htmlData = interpolate(template, {
+        // style: textSlideStyle,
+        // width: Math.round(canvasSize.width * 0.625).toString(),
+        html_body: html,
+    });
+    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+    return imagePath;
+};
+export const process = processHtmlTailwind;

package/lib/utils/image_plugins/index.d.ts

@@ -4,4 +4,5 @@ import * as pluginImage from "./image.js";
 import * as pluginChart from "./chart.js";
 import * as pluginMermaid from "./mermaid.js";
 import * as pluginMovie from "./movie.js";
-export declare const imagePlugins: (typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginImage | typeof pluginChart | typeof pluginMermaid | typeof pluginMovie)[];
+import * as pluginHtmlTailwind from "./html_tailwind.js";
+export declare const imagePlugins: (typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginImage | typeof pluginChart | typeof pluginMermaid | typeof pluginMovie | typeof pluginHtmlTailwind)[];

package/lib/utils/image_plugins/index.js

@@ -4,4 +4,5 @@ import * as pluginImage from "./image.js";
 import * as pluginChart from "./chart.js";
 import * as pluginMermaid from "./mermaid.js";
 import * as pluginMovie from "./movie.js";
-export const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie];
+import * as pluginHtmlTailwind from "./html_tailwind.js";
+export const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie, pluginHtmlTailwind];

package/lib/utils/image_plugins/mermaid.js

@@ -14,7 +14,7 @@ const processMermaid = async (params) => {
             style: textSlideStyle,
             diagram_code: `${diagram_code}\n${beat.image.appendix?.join("\n") ?? ""}`,
         });
-        await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+        await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, true);
     }
     return imagePath;
 };

package/lib/utils/image_plugins/tailwind.d.ts

@@ -0,0 +1,3 @@
+import { ImageProcessorParams } from "../../types/index.js";
+export declare const imageType = "tailwind";
+export declare const process: (params: ImageProcessorParams) => Promise<string | undefined>;

package/lib/utils/image_plugins/tailwind.js

@@ -0,0 +1,18 @@
+import { getHTMLFile } from "../file.js";
+import { renderHTMLToImage, interpolate } from "../markdown.js";
+export const imageType = "tailwind";
+const processTailwind = async (params) => {
+    const { beat, imagePath, canvasSize } = params;
+    if (!beat.image || beat.image.type !== imageType)
+        return;
+    const html = Array.isArray(beat.image.html) ? beat.image.html.join("\n") : beat.image.html;
+    const template = getHTMLFile("tailwind");
+    const htmlData = interpolate(template, {
+        // style: textSlideStyle,
+        // width: Math.round(canvasSize.width * 0.625).toString(),
+        html_body: html,
+    });
+    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+    return imagePath;
+};
+export const process = processTailwind;

package/lib/utils/image_plugins/text_slide.js

@@ -5,8 +5,15 @@ const processTextSlide = async (params) => {
     if (!beat.image || beat.image.type !== imageType)
         return;
     const slide = beat.image.slide;
-    const markdown = `# ${slide.title}\n` + slide.bullets.map((text) => `- ${text}`).join("\n");
-    await renderMarkdownToImage(markdown, textSlideStyle, imagePath, canvasSize.width, canvasSize.height);
+    const markdown = `# ${slide.title}\n` + (slide.subtitle ? `## ${slide.subtitle}\n` : "") + (slide.bullets ?? []).map((text) => `- ${text}`).join("\n");
+    const topMargin = (() => {
+        if (slide.bullets?.length && slide.bullets.length > 0) {
+            return "";
+        }
+        const marginTop = slide.subtitle ? canvasSize.height * 0.4 : canvasSize.height * 0.45;
+        return `body {margin-top: ${marginTop}px;}`;
+    })();
+    await renderMarkdownToImage(markdown, textSlideStyle + topMargin, imagePath, canvasSize.width, canvasSize.height);
     return imagePath;
 };
 export const process = processTextSlide;

package/lib/utils/markdown.d.ts

@@ -1,3 +1,3 @@
-export declare const renderHTMLToImage: (html: string, outputPath: string, width: number, height: number) => Promise<void>;
+export declare const renderHTMLToImage: (html: string, outputPath: string, width: number, height: number, isMermaid?: boolean, omitBackground?: boolean) => Promise<void>;
 export declare const renderMarkdownToImage: (markdown: string, style: string, outputPath: string, width: number, height: number) => Promise<void>;
 export declare const interpolate: (template: string, data: Record<string, string>) => string;

package/lib/utils/markdown.js

@@ -2,7 +2,7 @@ import { GraphAILogger } from "graphai";
 import { marked } from "marked";
 import puppeteer from "puppeteer";
 const isCI = process.env.CI === "true";
-export const renderHTMLToImage = async (html, outputPath, width, height) => {
+export const renderHTMLToImage = async (html, outputPath, width, height, isMermaid = false, omitBackground = false) => {
     // Use Puppeteer to render HTML to an image
     const browser = await puppeteer.launch({
         args: isCI ? ["--no-sandbox"] : [],
@@ -12,8 +12,14 @@ export const renderHTMLToImage = async (html, outputPath, width, height) => {
     await page.setContent(html);
     // Adjust page settings if needed (like width, height, etc.)
     await page.setViewport({ width, height });
+    if (isMermaid) {
+        await page.waitForFunction(() => {
+            const el = document.querySelector(".mermaid");
+            return el && el.dataset.ready === "true";
+        }, { timeout: 20000 });
+    }
     // Step 3: Capture screenshot of the page (which contains the Markdown-rendered HTML)
-    await page.screenshot({ path: outputPath });
+    await page.screenshot({ path: outputPath, omitBackground: omitBackground });
     await browser.close();
     GraphAILogger.info(`HTML image rendered to ${outputPath}`);
 };

package/lib/utils/preprocess.d.ts

@@ -2,16 +2,10 @@ import { MulmoStudio, MulmoScript } from "../types/index.js";
 export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string) => {
     beats: {
         duration?: number | undefined;
-        multiLingualTexts?: Record<string, {
-            text: string;
-            lang: string;
-            texts?: string[] | undefined;
-            ttsTexts?: string[] | undefined;
-            duration?: number | undefined;
-        }> | undefined;
         hash?: string | undefined;
         audioFile?: string | undefined;
         imageFile?: string | undefined;
+        captionFile?: string | undefined;
     }[];
     script: {
         $mulmocast: {
@@ -23,7 +17,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             height: number;
         };
         speechParams: {
-            provider: "openai" | "nijivoice";
+            provider: "openai" | "nijivoice" | "google";
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
@@ -33,9 +27,16 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 } | undefined;
             }>;
         };
+        audioParams: {
+            introPadding: number;
+            padding: number;
+            closingPadding: number;
+            outroPadding: number;
+        };
         beats: {
             text: string;
             speaker: string;
+            duration?: number | undefined;
             speechOptions?: {
                 speed?: number | undefined;
                 instruction?: string | undefined;
@@ -110,7 +111,8 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "textSlide";
                 slide: {
                     title: string;
-                    bullets: string[];
+                    subtitle?: string | undefined;
+                    bullets?: string[] | undefined;
                 };
             } | {
                 type: "chart";
@@ -133,6 +135,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "mermaid";
                 title: string;
                 appendix?: string[] | undefined;
+            } | {
+                type: "html_tailwind";
+                html: string | string[];
             } | undefined;
             audio?: {
                 type: "audio";
@@ -176,9 +181,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         textSlideParams?: {
             cssStyles: string | string[];
         } | undefined;
-        videoParams?: {
-            padding?: number | undefined;
-        } | undefined;
         omitCaptions?: boolean | undefined;
         description?: string | undefined;
         references?: {
@@ -191,4 +193,13 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         __test_invalid__?: boolean | undefined;
     };
     filename: string;
+    multiLingual: {
+        multiLingualTexts: Record<string, {
+            text: string;
+            lang: string;
+            texts?: string[] | undefined;
+            ttsTexts?: string[] | undefined;
+            duration?: number | undefined;
+        }>;
+    }[];
 };

package/lib/utils/preprocess.js

@@ -8,6 +8,7 @@ const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
         script: mulmoScript,
         filename: fileName,
         beats: [...Array(mulmoScript.beats.length)].map(() => ({})),
+        multiLingual: [...Array(mulmoScript.beats.length)].map(() => ({ multiLingualTexts: {} })),
     };
 };
 const mulmoCredit = (speaker) => {
@@ -42,6 +43,9 @@ export const createOrUpdateStudioData = (_mulmoScript, currentStudio, fileName)
     mulmoScript.beats.forEach((beat, index) => {
         // Filling the default values
         studio.script.beats[index] = mulmoBeatSchema.parse(beat);
+        if (!studio.multiLingual[index]) {
+            studio.multiLingual[index] = { multiLingualTexts: {} };
+        }
     });
     return studio;
 };

package/lib/utils/prompt.d.ts

@@ -0,0 +1,15 @@
+import { MulmoBeat, MulmoScript, MulmoScriptTemplate, MulmoStoryboard } from "../types/index.js";
+export declare const imagePrompt: (beat: MulmoBeat, style?: string) => string;
+export declare const graphDataScriptFromUrlPrompt: (sourceTextInput: string) => string;
+export declare const graphDataScriptGeneratePrompt: (scene: string) => string;
+export declare const getMulmoScriptTemplateSystemPrompt: (template: MulmoScriptTemplate) => string;
+export declare const interactiveClarificationPrompt = "If there are any unclear points, be sure to ask the user questions and clarify them before generating the script.";
+export declare const prefixPrompt = "Here is the web content that can be used as reference material for the script:";
+export declare const translateSystemPrompt = "Please translate the given text into the language specified in language (in locale format, like en, ja, fr, ch).";
+export declare const translatePrompts: string[];
+export declare const sceneToBeatsPrompt: ({ sampleBeats, beatsPerScene, allScenes, }: {
+    sampleBeats: MulmoScript["beats"];
+    beatsPerScene: number;
+    allScenes: string;
+}) => string;
+export declare const storyToScriptInfoPrompt: (scriptWithoutBeats: Omit<MulmoScript, "beats">, story: MulmoStoryboard) => string;