mulmocast 0.0.5 → 0.0.7

package/lib/types/schema.js

@@ -4,7 +4,7 @@ const URLStringSchema = z.string().url();
 export const localizedTextSchema = z
     .object({
     text: z.string(),
-    lang: z.string(),
+    lang: langSchema,
     // caption: z.string(),
     texts: z.array(z.string()).optional(),
     ttsTexts: z.array(z.string()).optional(),
@@ -77,7 +77,8 @@ export const mulmoTextSlideMediaSchema = z
     type: z.literal("textSlide"),
     slide: z.object({
         title: z.string(),
-        bullets: z.array(z.string()),
+        subtitle: z.string().optional(),
+        bullets: z.array(z.string()).optional(),
     }),
 })
     .strict();
@@ -96,6 +97,12 @@ export const mulmoMermaidMediaSchema = z
     appendix: z.array(z.string()).optional().describe("The appendix of the mermaid diagram; typically, style information."),
 })
     .strict();
+export const mulmoHtmlTailwindMediaSchema = z
+    .object({
+    type: z.literal("html_tailwind"),
+    html: stringOrStringArray,
+})
+    .strict();
 export const mulmoImageAssetSchema = z.union([
     mulmoMarkdownMediaSchema,
     mulmoWebMediaSchema,
@@ -106,6 +113,7 @@ export const mulmoImageAssetSchema = z.union([
     mulmoTextSlideMediaSchema,
     mulmoChartMediaSchema,
     mulmoMermaidMediaSchema,
+    mulmoHtmlTailwindMediaSchema,
 ]);
 const mulmoAudioMediaSchema = z
     .object({
@@ -133,18 +141,36 @@ export const textSlideParamsSchema = z
     cssStyles: stringOrStringArray,
 })
     .strict();
+/* TODO: Add something later
 export const videoParamsSchema = z
-    .object({
+  .object({
     padding: z.number().optional(), // msec
+  })
+  .strict();
+*/
+export const beatAudioParamsSchema = z
+    .object({
+    padding: z.number().optional().describe("Padding between beats"), // seconds
+})
+    .strict();
+// Note: we can't extend beatAudioParamsSchema because it has padding as optional
+export const audioParamsSchema = z
+    .object({
+    padding: z.number().describe("Padding between beats"), // seconds
+    introPadding: z.number().describe("Padding at the beginning of the audio"), // seconds
+    closingPadding: z.number().describe("Padding before the last beat"), // seconds
+    outroPadding: z.number().describe("Padding at the end of the audio"), // seconds
 })
     .strict();
 export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.default("Presenter"),
-    text: z.string(),
+    text: z.string().describe("Text to be spoken. If empty, the audio is not generated."),
     image: mulmoImageAssetSchema.optional(),
     audio: mulmoAudioAssetSchema.optional(),
+    duration: z.number().optional().describe("Duration of the beat. Used only when the text is empty"),
     imageParams: mulmoImageParamsSchema.optional(), // beat specific parameters
+    audioParams: beatAudioParamsSchema.optional(), // beat specific parameters
     speechOptions: speechOptionsSchema.optional(),
     textSlideParams: textSlideParamsSchema.optional(),
     imagePrompt: z.string().optional(), // specified or inserted by preprocessor
@@ -163,7 +189,7 @@ export const mulmoCastCreditSchema = z
     credit: z.literal("closing").optional(),
 })
     .strict();
-export const text2SpeechProviderSchema = z.union([z.literal("openai"), z.literal("nijivoice")]).default("openai");
+export const text2SpeechProviderSchema = z.union([z.literal("openai"), z.literal("nijivoice"), z.literal("google")]).default("openai");
 export const mulmoSpeechParamsSchema = z
     .object({
     provider: text2SpeechProviderSchema, // has default value
@@ -191,7 +217,13 @@ export const mulmoPresentationStyleSchema = z.object({
         .optional(),
     // for textSlides
     textSlideParams: textSlideParamsSchema.optional(),
-    videoParams: videoParamsSchema.optional(),
+    // videoParams: videoParamsSchema.optional(),
+    audioParams: audioParamsSchema.default({
+        introPadding: 1.0,
+        padding: 0.3,
+        closingPadding: 0.8,
+        outroPadding: 1.0,
+    }),
     // TODO: Switch to showCaptions later
     omitCaptions: z.boolean().optional(), // default is false
 });
@@ -216,18 +248,55 @@ export const mulmoScriptSchema = mulmoPresentationStyleSchema
     .strict();
 export const mulmoStudioBeatSchema = z
     .object({
-    multiLingualTexts: multiLingualTextsSchema.optional(),
     hash: z.string().optional(),
     duration: z.number().optional(),
     audioFile: z.string().optional(),
     imageFile: z.string().optional(), // path to the image
+    captionFile: z.string().optional(), // path to the caption image
 })
     .strict();
+export const mulmoStudioMultiLingualDataSchema = z.object({
+    multiLingualTexts: multiLingualTextsSchema,
+});
+export const mulmoStudioMultiLingualSchema = z.array(mulmoStudioMultiLingualDataSchema).min(1);
+export const mulmoSessionStateSchema = z.object({
+    inSession: z.object({
+        audio: z.boolean(),
+        image: z.boolean(),
+        video: z.boolean(),
+        multiLingual: z.boolean(),
+        caption: z.boolean(),
+        pdf: z.boolean(),
+    }),
+    inBeatSession: z.object({
+        audio: z.set(z.number()),
+        image: z.set(z.number()),
+        multiLingual: z.set(z.number()),
+        caption: z.set(z.number()),
+    }),
+});
 export const mulmoStudioSchema = z
     .object({
     script: mulmoScriptSchema,
     filename: z.string(),
     beats: z.array(mulmoStudioBeatSchema).min(1),
+    multiLingual: mulmoStudioMultiLingualSchema,
+    state: mulmoSessionStateSchema.default({
+        inSession: {
+            audio: false,
+            image: false,
+            video: false,
+            multiLingual: false,
+            caption: false,
+            pdf: false,
+        },
+        inBeatSession: {
+            audio: new Set(),
+            image: new Set(),
+            multiLingual: new Set(),
+            caption: new Set(),
+        },
+    }),
 })
     .strict();
 export const mulmoScriptTemplateSchema = z
@@ -235,7 +304,7 @@ export const mulmoScriptTemplateSchema = z
     title: z.string(),
     description: z.string(),
     systemPrompt: z.string(),
-    script: mulmoScriptSchema.optional(),
+    scriptName: z.string().optional(),
 })
     .strict();
 export const mulmoStoryboardSceneSchema = z

package/lib/types/type.d.ts

@@ -1,5 +1,6 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, text2ImageProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema } from "./schema.js";
-import { pdf_modes, pdf_sizes } from "../utils/const.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, text2ImageProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema } from "./schema.js";
+import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
+import { LLM } from "../utils/utils.js";
 import { z } from "zod";
 export type LANG = z.infer<typeof langSchema>;
 export type MulmoBeat = z.infer<typeof mulmoBeatSchema>;
@@ -20,6 +21,9 @@ export type MulmoStudioBeat = z.infer<typeof mulmoStudioBeatSchema>;
 export type MulmoMediaSource = z.infer<typeof mediaSourceSchema>;
 export type MulmoStudio = z.infer<typeof mulmoStudioSchema>;
 export type MulmoScriptTemplate = z.infer<typeof mulmoScriptTemplateSchema>;
+export type MulmoStudioMultiLingual = z.infer<typeof mulmoStudioMultiLingualSchema>;
+export type MulmoStudioMultiLingualData = z.infer<typeof mulmoStudioMultiLingualDataSchema>;
+export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
 export type MulmoTextSlideMedia = z.infer<typeof mulmoTextSlideMediaSchema>;
 export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
@@ -36,7 +40,9 @@ export type FileDirs = {
 export type MulmoStudioContext = {
     fileDirs: FileDirs;
     studio: MulmoStudio;
+    lang?: string;
     force: boolean;
+    caption?: string;
 };
 export type ScriptingParams = {
     urls: string[];
@@ -45,7 +51,7 @@ export type ScriptingParams = {
     templateName: string;
     filename: string;
     llm_model?: string;
-    llm_agent?: string;
+    llm?: LLM;
 };
 export type ImageProcessorParams = {
     beat: MulmoBeat;
@@ -62,3 +68,4 @@ export type Text2ImageAgentInfo = {
     imageParams: MulmoImageParams;
 };
 export type BeatMediaType = "movie" | "image";
+export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];

package/lib/utils/const.d.ts

@@ -4,3 +4,8 @@ export declare const imageDirName = "images";
 export declare const cacheDirName = "cache";
 export declare const pdf_modes: string[];
 export declare const pdf_sizes: string[];
+export declare const languages: string[];
+export declare const storyToScriptGenerateMode: {
+    stepWise: string;
+    oneStep: string;
+};

package/lib/utils/const.js

@@ -4,3 +4,8 @@ export const imageDirName = "images";
 export const cacheDirName = "cache";
 export const pdf_modes = ["slide", "talk", "handout"];
 export const pdf_sizes = ["letter", "a4"];
+export const languages = ["en", "ja"];
+export const storyToScriptGenerateMode = {
+    stepWise: "step_wise",
+    oneStep: "one_step",
+};

package/lib/utils/ffmpeg_utils.d.ts

@@ -0,0 +1,12 @@
+import ffmpeg from "fluent-ffmpeg";
+export type FfmpegContext = {
+    command: ffmpeg.FfmpegCommand;
+    inputCount: number;
+    filterComplex: string[];
+};
+export declare const FfmpegContextInit: () => FfmpegContext;
+export declare const FfmpegContextAddInput: (context: FfmpegContext, input: string) => number;
+export declare const FfmpegContextPushFormattedAudio: (context: FfmpegContext, sourceId: string, outputId: string, duration?: number | undefined) => void;
+export declare const FfmpegContextInputFormattedAudio: (context: FfmpegContext, input: string, duration?: number | undefined) => string;
+export declare const FfmpegContextGenerateOutput: (context: FfmpegContext, output: string, options?: string[]) => Promise<number>;
+export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<number>;

package/lib/utils/ffmpeg_utils.js

@@ -0,0 +1,63 @@
+import ffmpeg from "fluent-ffmpeg";
+import { GraphAILogger } from "graphai";
+export const FfmpegContextInit = () => {
+    return {
+        command: ffmpeg(),
+        inputCount: 0,
+        filterComplex: [],
+    };
+};
+export const FfmpegContextAddInput = (context, input) => {
+    context.command.input(input);
+    context.inputCount++;
+    return context.inputCount - 1; // returned the index of the input
+};
+export const FfmpegContextPushFormattedAudio = (context, sourceId, outputId, duration = undefined) => {
+    if (duration !== undefined) {
+        context.filterComplex.push(`${sourceId}atrim=duration=${duration},aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo${outputId}`);
+    }
+    else {
+        context.filterComplex.push(`${sourceId}aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo${outputId}`);
+    }
+};
+export const FfmpegContextInputFormattedAudio = (context, input, duration = undefined) => {
+    const index = FfmpegContextAddInput(context, input);
+    const audioId = `[a${index}]`;
+    FfmpegContextPushFormattedAudio(context, `[${index}:a]`, audioId, duration);
+    return audioId;
+};
+export const FfmpegContextGenerateOutput = (context, output, options = []) => {
+    return new Promise((resolve, reject) => {
+        context.command
+            .complexFilter(context.filterComplex)
+            .outputOptions(options)
+            .output(output)
+            .on("start", (__cmdLine) => {
+            GraphAILogger.log("Started FFmpeg ..."); // with command:', cmdLine);
+        })
+            .on("error", (err, stdout, stderr) => {
+            GraphAILogger.error("Error occurred:", err);
+            GraphAILogger.error("FFmpeg stdout:", stdout);
+            GraphAILogger.error("FFmpeg stderr:", stderr);
+            GraphAILogger.info("Video/Audio creation failed. An unexpected error occurred.");
+            reject();
+        })
+            .on("end", () => {
+            resolve(0);
+        })
+            .run();
+    });
+};
+export const ffmpegGetMediaDuration = (filePath) => {
+    return new Promise((resolve, reject) => {
+        ffmpeg.ffprobe(filePath, (err, metadata) => {
+            if (err) {
+                GraphAILogger.info("Error while getting metadata:", err);
+                reject(err);
+            }
+            else {
+                resolve(metadata.format.duration);
+            }
+        });
+    });
+};

package/lib/utils/file.d.ts

@@ -1,5 +1,6 @@
 import { MulmoScript, MulmoScriptTemplate, MulmoMediaSource, MulmoStudioContext } from "../types/index.js";
 import { PDFMode } from "../types/index.js";
+import { ZodSchema } from "zod";
 export declare function readMulmoScriptFile<T = MulmoScript>(path: string, errorMessage: string): {
     mulmoData: T;
     mulmoDataPath: string;
@@ -24,23 +25,27 @@ export declare const fetchMulmoScriptFile: (url: string) => Promise<{
     script?: undefined;
 }>;
 export declare const getOutputStudioFilePath: (outDirPath: string, fileName: string) => string;
+export declare const resolveDirPath: (dirPath: string, studioFileName: string) => string;
 export declare const getAudioSegmentDirPath: (audioDirPath: string, studioFileName: string) => string;
 export declare const getAudioSegmentFilePath: (audioDirPath: string, studioFileName: string, fileName: string) => string;
-export declare const getAudioCombinedFilePath: (audioDirPath: string, fileName: string) => string;
+export declare const getAudioCombinedFilePath: (audioDirPath: string, fileName: string, lang?: string) => string;
 export declare const getAudioArtifactFilePath: (outDirPath: string, fileName: string) => string;
-export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string) => string;
-export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode) => string;
+export declare const getOutputVideoFilePath: (outDirPath: string, fileName: string, lang?: string, caption?: string) => string;
+export declare const getOutputPdfFilePath: (outDirPath: string, fileName: string, pdfMode: PDFMode, lang?: string) => string;
 export declare const getTemplateFilePath: (templateName: string) => string;
 export declare const mkdir: (dirPath: string) => void;
 export declare const silentPath: string;
 export declare const silentLastPath: string;
+export declare const silent60secPath: string;
 export declare const defaultBGMPath: string;
 export declare const getHTMLFile: (filename: string) => string;
 export declare const getBaseDirPath: (basedir?: string) => string;
 export declare const getFullPath: (baseDirPath: string | undefined, file: string) => string;
+export declare const readScriptTemplateFile: (scriptName: string) => any;
 export declare const readTemplatePrompt: (templateName: string) => string;
 export declare const getAvailableTemplates: () => (MulmoScriptTemplate & {
     filename: string;
 })[];
 export declare const writingMessage: (filePath: string) => void;
 export declare const resolveMediaSource: (source: MulmoMediaSource, context: MulmoStudioContext) => string | null;
+export declare const readAndParseJson: <S extends ZodSchema<any>>(filePath: string, schema: S) => ReturnType<S["parse"]>;

package/lib/utils/file.js

@@ -45,23 +45,35 @@ export const fetchMulmoScriptFile = async (url) => {
 export const getOutputStudioFilePath = (outDirPath, fileName) => {
     return path.resolve(outDirPath, fileName + "_studio.json");
 };
+export const resolveDirPath = (dirPath, studioFileName) => {
+    return path.resolve(dirPath, studioFileName);
+};
+// TODO: probably better to just use resolveDirPath instead.
 export const getAudioSegmentDirPath = (audioDirPath, studioFileName) => {
     return path.resolve(audioDirPath, studioFileName);
 };
 export const getAudioSegmentFilePath = (audioDirPath, studioFileName, fileName) => {
     return path.resolve(getAudioSegmentDirPath(audioDirPath, studioFileName), fileName + ".mp3");
 };
-export const getAudioCombinedFilePath = (audioDirPath, fileName) => {
+export const getAudioCombinedFilePath = (audioDirPath, fileName, lang) => {
+    if (lang) {
+        return path.resolve(audioDirPath, fileName, `${fileName}_${lang}.mp3`);
+    }
     return path.resolve(audioDirPath, fileName, fileName + ".mp3");
 };
 export const getAudioArtifactFilePath = (outDirPath, fileName) => {
     return path.resolve(outDirPath, fileName + ".mp3");
 };
-export const getOutputVideoFilePath = (outDirPath, fileName) => {
-    return path.resolve(outDirPath, fileName + ".mp4");
+export const getOutputVideoFilePath = (outDirPath, fileName, lang, caption) => {
+    const suffix = lang ? `_${lang}` : "";
+    const suffix2 = caption ? `__${caption}` : "";
+    return path.resolve(outDirPath, `${fileName}${suffix}${suffix2}.mp4`);
 };
-export const getOutputPdfFilePath = (outDirPath, fileName, pdfMode) => {
-    return path.resolve(outDirPath, fileName + "_" + pdfMode + ".pdf");
+export const getOutputPdfFilePath = (outDirPath, fileName, pdfMode, lang) => {
+    if (lang) {
+        return path.resolve(outDirPath, `${fileName}_${pdfMode}_${lang}.pdf`);
+    }
+    return path.resolve(outDirPath, `${fileName}_${pdfMode}.pdf`);
 };
 export const getTemplateFilePath = (templateName) => {
     return path.resolve(__dirname, "../../assets/templates/" + templateName + ".json");
@@ -74,6 +86,7 @@ export const mkdir = (dirPath) => {
 };
 export const silentPath = path.resolve(__dirname, "../../assets/audio/silent300.mp3");
 export const silentLastPath = path.resolve(__dirname, "../../assets/audio/silent800.mp3");
+export const silent60secPath = path.resolve(__dirname, "../../assets/audio/silent60sec.mp3");
 export const defaultBGMPath = path.resolve(__dirname, "../../assets/music/StarsBeyondEx.mp3");
 export const getHTMLFile = (filename) => {
     const htmlPath = path.resolve(__dirname, `../../assets/html/${filename}.html`);
@@ -98,11 +111,23 @@ export const getFullPath = (baseDirPath, file) => {
     }
     return path.resolve(file);
 };
+export const readScriptTemplateFile = (scriptName) => {
+    const scriptPath = path.resolve(__dirname, "../../scripts/templates", scriptName);
+    const scriptData = fs.readFileSync(scriptPath, "utf-8");
+    // NOTE: We don't want to schema parse the script here to eliminate default values.
+    return JSON.parse(scriptData);
+};
 export const readTemplatePrompt = (templateName) => {
     const templatePath = getTemplateFilePath(templateName);
-    const scriptData = fs.readFileSync(templatePath, "utf-8");
-    const template = mulmoScriptTemplateSchema.parse(JSON.parse(scriptData));
-    const prompt = MulmoScriptTemplateMethods.getSystemPrompt(template);
+    const templateData = fs.readFileSync(templatePath, "utf-8");
+    const template = mulmoScriptTemplateSchema.parse(JSON.parse(templateData));
+    const script = (() => {
+        if (template.scriptName) {
+            return readScriptTemplateFile(template.scriptName);
+        }
+        return undefined;
+    })();
+    const prompt = MulmoScriptTemplateMethods.getSystemPrompt(template, script);
     return prompt;
 };
 export const getAvailableTemplates = () => {
@@ -120,7 +145,7 @@ export const getAvailableTemplates = () => {
     });
 };
 export const writingMessage = (filePath) => {
-    GraphAILogger.info(`writing: ${filePath}`);
+    GraphAILogger.debug(`writing: ${filePath}`);
 };
 export const resolveMediaSource = (source, context) => {
     if (source.kind === "path") {
@@ -131,3 +156,9 @@ export const resolveMediaSource = (source, context) => {
     }
     return null;
 };
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const readAndParseJson = (filePath, schema) => {
+    const fileContent = fs.readFileSync(filePath, "utf-8");
+    const json = JSON.parse(fileContent);
+    return schema.parse(json);
+};

package/lib/utils/filters.js

@@ -5,9 +5,10 @@ import fsPromise from "fs/promises";
 import { GraphAILogger } from "graphai";
 import { writingMessage } from "./file.js";
 import { text2hash } from "./utils.js";
+import { MulmoStudioMethods } from "../methods/mulmo_studio.js";
 export const fileCacheAgentFilter = async (context, next) => {
     const { namedInputs } = context;
-    const { file, text, force } = namedInputs;
+    const { file, force, studio, index, sessionType } = namedInputs;
     const shouldUseCache = async () => {
         if (force) {
             return false;
@@ -21,19 +22,23 @@ export const fileCacheAgentFilter = async (context, next) => {
         }
     };
     if (await shouldUseCache()) {
-        const elements = file.split("/");
-        GraphAILogger.info("cache hit: " + elements[elements.length - 1], text.slice(0, 10));
         return true;
     }
-    const output = (await next(context));
-    const buffer = output ? output["buffer"] : undefined;
-    if (buffer) {
-        writingMessage(file);
-        await fsPromise.writeFile(file, buffer);
-        return true;
+    try {
+        MulmoStudioMethods.setBeatSessionState(studio, sessionType, index, true);
+        const output = (await next(context));
+        const buffer = output ? output["buffer"] : undefined;
+        if (buffer) {
+            writingMessage(file);
+            await fsPromise.writeFile(file, buffer);
+            return true;
+        }
+        GraphAILogger.log("no cache, no buffer: " + file);
+        return false;
+    }
+    finally {
+        MulmoStudioMethods.setBeatSessionState(studio, sessionType, index, false);
     }
-    GraphAILogger.log("no cache, no buffer: " + file);
-    return false;
 };
 export const browserlessCacheGenerator = (cacheDir) => {
     const browserlessCache = async (context, next) => {

package/lib/utils/image_plugins/chart.js

@@ -5,11 +5,16 @@ const processChart = async (params) => {
     const { beat, imagePath, canvasSize, textSlideStyle } = params;
     if (!beat.image || beat.image.type !== imageType)
         return;
+    const isCircular = beat.image.chartData.type === "pie" ||
+        beat.image.chartData.type === "doughnut" ||
+        beat.image.chartData.type === "polarArea" ||
+        beat.image.chartData.type === "radar";
+    const chart_width = isCircular ? Math.min(canvasSize.width, canvasSize.height) * 0.75 : canvasSize.width * 0.75;
     const template = getHTMLFile("chart");
     const htmlData = interpolate(template, {
         title: beat.image.title,
         style: textSlideStyle,
-        width: Math.round(canvasSize.width * 0.625).toString(),
+        chart_width: chart_width.toString(),
         chart_data: JSON.stringify(beat.image.chartData),
     });
     await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);

package/lib/utils/image_plugins/html_tailwind.d.ts

@@ -0,0 +1,3 @@
+import { ImageProcessorParams } from "../../types/index.js";
+export declare const imageType = "html_tailwind";
+export declare const process: (params: ImageProcessorParams) => Promise<string | undefined>;

package/lib/utils/image_plugins/html_tailwind.js

@@ -0,0 +1,18 @@
+import { getHTMLFile } from "../file.js";
+import { renderHTMLToImage, interpolate } from "../markdown.js";
+export const imageType = "html_tailwind";
+const processHtmlTailwind = async (params) => {
+    const { beat, imagePath, canvasSize } = params;
+    if (!beat.image || beat.image.type !== imageType)
+        return;
+    const html = Array.isArray(beat.image.html) ? beat.image.html.join("\n") : beat.image.html;
+    const template = getHTMLFile("tailwind");
+    const htmlData = interpolate(template, {
+        // style: textSlideStyle,
+        // width: Math.round(canvasSize.width * 0.625).toString(),
+        html_body: html,
+    });
+    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+    return imagePath;
+};
+export const process = processHtmlTailwind;

package/lib/utils/image_plugins/index.d.ts

@@ -4,4 +4,5 @@ import * as pluginImage from "./image.js";
 import * as pluginChart from "./chart.js";
 import * as pluginMermaid from "./mermaid.js";
 import * as pluginMovie from "./movie.js";
-export declare const imagePlugins: (typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginImage | typeof pluginChart | typeof pluginMermaid | typeof pluginMovie)[];
+import * as pluginHtmlTailwind from "./html_tailwind.js";
+export declare const imagePlugins: (typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginImage | typeof pluginChart | typeof pluginMermaid | typeof pluginMovie | typeof pluginHtmlTailwind)[];

package/lib/utils/image_plugins/index.js

@@ -4,4 +4,5 @@ import * as pluginImage from "./image.js";
 import * as pluginChart from "./chart.js";
 import * as pluginMermaid from "./mermaid.js";
 import * as pluginMovie from "./movie.js";
-export const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie];
+import * as pluginHtmlTailwind from "./html_tailwind.js";
+export const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie, pluginHtmlTailwind];

package/lib/utils/image_plugins/mermaid.js

@@ -14,7 +14,7 @@ const processMermaid = async (params) => {
             style: textSlideStyle,
             diagram_code: `${diagram_code}\n${beat.image.appendix?.join("\n") ?? ""}`,
         });
-        await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+        await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, true);
     }
     return imagePath;
 };

package/lib/utils/image_plugins/tailwind.d.ts

@@ -0,0 +1,3 @@
+import { ImageProcessorParams } from "../../types/index.js";
+export declare const imageType = "tailwind";
+export declare const process: (params: ImageProcessorParams) => Promise<string | undefined>;

package/lib/utils/image_plugins/tailwind.js

@@ -0,0 +1,18 @@
+import { getHTMLFile } from "../file.js";
+import { renderHTMLToImage, interpolate } from "../markdown.js";
+export const imageType = "tailwind";
+const processTailwind = async (params) => {
+    const { beat, imagePath, canvasSize } = params;
+    if (!beat.image || beat.image.type !== imageType)
+        return;
+    const html = Array.isArray(beat.image.html) ? beat.image.html.join("\n") : beat.image.html;
+    const template = getHTMLFile("tailwind");
+    const htmlData = interpolate(template, {
+        // style: textSlideStyle,
+        // width: Math.round(canvasSize.width * 0.625).toString(),
+        html_body: html,
+    });
+    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+    return imagePath;
+};
+export const process = processTailwind;

package/lib/utils/image_plugins/text_slide.js

@@ -5,8 +5,15 @@ const processTextSlide = async (params) => {
     if (!beat.image || beat.image.type !== imageType)
         return;
     const slide = beat.image.slide;
-    const markdown = `# ${slide.title}\n` + slide.bullets.map((text) => `- ${text}`).join("\n");
-    await renderMarkdownToImage(markdown, textSlideStyle, imagePath, canvasSize.width, canvasSize.height);
+    const markdown = `# ${slide.title}\n` + (slide.subtitle ? `## ${slide.subtitle}\n` : "") + (slide.bullets ?? []).map((text) => `- ${text}`).join("\n");
+    const topMargin = (() => {
+        if (slide.bullets?.length && slide.bullets.length > 0) {
+            return "";
+        }
+        const marginTop = slide.subtitle ? canvasSize.height * 0.4 : canvasSize.height * 0.45;
+        return `body {margin-top: ${marginTop}px;}`;
+    })();
+    await renderMarkdownToImage(markdown, textSlideStyle + topMargin, imagePath, canvasSize.width, canvasSize.height);
     return imagePath;
 };
 export const process = processTextSlide;

package/lib/utils/markdown.d.ts

@@ -1,3 +1,3 @@
-export declare const renderHTMLToImage: (html: string, outputPath: string, width: number, height: number) => Promise<void>;
+export declare const renderHTMLToImage: (html: string, outputPath: string, width: number, height: number, isMermaid?: boolean, omitBackground?: boolean) => Promise<void>;
 export declare const renderMarkdownToImage: (markdown: string, style: string, outputPath: string, width: number, height: number) => Promise<void>;
 export declare const interpolate: (template: string, data: Record<string, string>) => string;

package/lib/utils/markdown.js

@@ -1,8 +1,7 @@
-import { GraphAILogger } from "graphai";
 import { marked } from "marked";
 import puppeteer from "puppeteer";
 const isCI = process.env.CI === "true";
-export const renderHTMLToImage = async (html, outputPath, width, height) => {
+export const renderHTMLToImage = async (html, outputPath, width, height, isMermaid = false, omitBackground = false) => {
     // Use Puppeteer to render HTML to an image
     const browser = await puppeteer.launch({
         args: isCI ? ["--no-sandbox"] : [],
@@ -12,10 +11,15 @@ export const renderHTMLToImage = async (html, outputPath, width, height) => {
     await page.setContent(html);
     // Adjust page settings if needed (like width, height, etc.)
     await page.setViewport({ width, height });
+    if (isMermaid) {
+        await page.waitForFunction(() => {
+            const el = document.querySelector(".mermaid");
+            return el && el.dataset.ready === "true";
+        }, { timeout: 20000 });
+    }
     // Step 3: Capture screenshot of the page (which contains the Markdown-rendered HTML)
-    await page.screenshot({ path: outputPath });
+    await page.screenshot({ path: outputPath, omitBackground: omitBackground });
     await browser.close();
-    GraphAILogger.info(`HTML image rendered to ${outputPath}`);
 };
 export const renderMarkdownToImage = async (markdown, style, outputPath, width, height) => {
     const header = `<head><style>${style}</style></head>`;