mulmocast 0.1.2 → 0.1.3

package/lib/types/schema.js

@@ -153,7 +153,13 @@ const mulmoMidiMediaSchema = z
     .strict();
 export const mulmoAudioAssetSchema = z.union([mulmoAudioMediaSchema, mulmoMidiMediaSchema]);
 const imageIdSchema = z.string();
-export const mulmoImageParamsImagesSchema = z.record(imageIdSchema, mulmoImageMediaSchema);
+export const mulmoImagePromptMediaSchema = z
+    .object({
+    type: z.literal("imagePrompt"),
+    prompt: z.string(),
+})
+    .strict();
+export const mulmoImageParamsImagesSchema = z.record(imageIdSchema, z.union([mulmoImageMediaSchema, mulmoImagePromptMediaSchema]));
 export const mulmoFillOptionSchema = z
     .object({
     style: z.enum(["aspectFit", "aspectFill"]).default("aspectFit"),
@@ -329,7 +335,7 @@ export const mulmoReferenceSchema = z.object({
     url: URLStringSchema,
     title: z.string().optional(),
     description: z.string().optional(),
-    type: z.enum(["article", "paper", "image", "video", "audio"]).default("article"),
+    type: z.union([z.enum(["article", "paper", "image", "video", "audio"]), z.string()]).default("article"),
 });
 export const mulmoScriptSchema = mulmoPresentationStyleSchema
     .extend({
@@ -378,6 +384,7 @@ export const mulmoSessionStateSchema = z.object({
         multiLingual: z.record(z.number().int(), z.boolean()),
         caption: z.record(z.number().int(), z.boolean()),
         html: z.record(z.number().int(), z.boolean()),
+        imageReference: z.record(z.number().int(), z.boolean()),
     }),
 });
 export const mulmoStudioSchema = z

package/lib/types/type.d.ts

@@ -1,4 +1,4 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import { LLM } from "../utils/utils.js";
 import { z } from "zod";
@@ -35,6 +35,7 @@ export type MulmoOpenAIImageModel = z.infer<typeof mulmoOpenAIImageModelSchema>;
 export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
 export type MulmoGoogleMovieModel = z.infer<typeof mulmoGoogleMovieModelSchema>;
 export type MulmoReplicateMovieModel = z.infer<typeof mulmoReplicateMovieModelSchema>;
+export type MulmoImagePromptMedia = z.infer<typeof mulmoImagePromptMediaSchema>;
 export type MulmoTextSlideMedia = z.infer<typeof mulmoTextSlideMediaSchema>;
 export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
@@ -90,7 +91,7 @@ export type Text2HtmlAgentInfo = {
 export type BeatMediaType = "movie" | "image";
 export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];
 export type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
-export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html";
+export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html" | "imageReference";
 export type SessionProgressEvent = {
     kind: "session";
     sessionType: SessionType;

package/lib/utils/context.d.ts

@@ -193,6 +193,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     type: "midi";
                     source: string;
                 } | undefined;
+                imagePrompt?: string | undefined;
                 description?: string | undefined;
                 imageParams?: {
                     provider: "openai" | "google";
@@ -214,6 +215,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                             path: string;
                             kind: "path";
                         };
+                    } | {
+                        type: "imagePrompt";
+                        prompt: string;
                     }> | undefined;
                 } | undefined;
                 audioParams?: {
@@ -236,7 +240,6 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     lang?: string | undefined;
                 } | undefined;
                 imageNames?: string[] | undefined;
-                imagePrompt?: string | undefined;
                 moviePrompt?: string | undefined;
                 htmlPrompt?: {
                     prompt: string;
@@ -268,6 +271,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         path: string;
                         kind: "path";
                     };
+                } | {
+                    type: "imagePrompt";
+                    prompt: string;
                 }> | undefined;
             } | undefined;
             movieParams?: {
@@ -293,7 +299,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 lang?: string | undefined;
             } | undefined;
             references?: {
-                type: "image" | "audio" | "article" | "paper" | "video";
+                type: string;
                 url: string;
                 title?: string | undefined;
                 description?: string | undefined;
@@ -322,6 +328,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             multiLingual: {};
             caption: {};
             html: {};
+            imageReference: {};
         };
     };
     presentationStyle: {
@@ -387,6 +394,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     path: string;
                     kind: "path";
                 };
+            } | {
+                type: "imagePrompt";
+                prompt: string;
             }> | undefined;
         } | undefined;
         movieParams?: {

package/lib/utils/context.js

@@ -57,6 +57,7 @@ const initSessionState = () => {
             multiLingual: {},
             caption: {},
             html: {},
+            imageReference: {},
         },
     };
 };

package/lib/utils/ffmpeg_utils.d.ts

@@ -12,4 +12,4 @@ export declare const FfmpegContextPushFormattedAudio: (context: FfmpegContext, s
 export declare const FfmpegContextInputFormattedAudio: (context: FfmpegContext, input: string, duration?: number | undefined, inputOptions?: string[]) => string;
 export declare const FfmpegContextGenerateOutput: (context: FfmpegContext, output: string, options?: string[]) => Promise<number>;
 export declare const ffmpegGetMediaDuration: (filePath: string) => Promise<number>;
-export declare const extractImageFromMovie: (movieFile: string, imagePath: string) => Promise<void>;
+export declare const extractImageFromMovie: (movieFile: string, imagePath: string) => Promise<object>;

package/lib/utils/ffmpeg_utils.js

@@ -77,7 +77,7 @@ export const extractImageFromMovie = (movieFile, imagePath) => {
         ffmpeg(movieFile)
             .outputOptions(["-frames:v 1"])
             .output(imagePath)
-            .on("end", () => resolve())
+            .on("end", () => resolve({}))
             .on("error", (err) => reject(err))
             .run();
     });

package/lib/utils/file.js

@@ -29,9 +29,9 @@ export function readMulmoScriptFile(arg2, errorMessage) {
             fileName: parsedPath.name,
         };
     }
-    catch (__error) {
+    catch (error) {
         if (errorMessage) {
-            GraphAILogger.info("read file format is broken.");
+            GraphAILogger.info("read file format is broken.", error);
         }
         return null;
     }
@@ -159,8 +159,8 @@ export const readTemplatePrompt = (templateName) => {
     const template = JSON.parse(templateData);
     const script = (() => {
         if (template.scriptName) {
-            const script = readScriptTemplateFile(template.scriptName);
-            return { ...script, ...(template.presentationStyle ?? {}) };
+            const scriptData = readScriptTemplateFile(template.scriptName);
+            return { ...scriptData, ...(template.presentationStyle ?? {}) };
         }
         return undefined;
     })();

package/lib/utils/filters.js

@@ -7,17 +7,16 @@ import { writingMessage } from "./file.js";
 import { text2hash } from "./utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 export const fileCacheAgentFilter = async (context, next) => {
-    const { namedInputs } = context;
-    const { file, force, mulmoContext, index, sessionType } = namedInputs;
+    const { force, file, index, mulmoContext, sessionType } = context.namedInputs.cache;
     const shouldUseCache = async () => {
-        if (force) {
+        if (force && force.some((element) => element)) {
             return false;
         }
         try {
             await fsPromise.access(file);
             return true;
         }
-        catch (__e) {
+        catch {
             return false;
         }
     };

package/lib/utils/markdown.js

@@ -18,7 +18,7 @@ export const renderHTMLToImage = async (html, outputPath, width, height, isMerma
         }, { timeout: 20000 });
     }
     // Step 3: Capture screenshot of the page (which contains the Markdown-rendered HTML)
-    await page.screenshot({ path: outputPath, omitBackground: omitBackground });
+    await page.screenshot({ path: outputPath, omitBackground });
     await browser.close();
 };
 export const renderMarkdownToImage = async (markdown, style, outputPath, width, height) => {

package/lib/utils/preprocess.d.ts

@@ -188,6 +188,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 type: "midi";
                 source: string;
             } | undefined;
+            imagePrompt?: string | undefined;
             description?: string | undefined;
             imageParams?: {
                 provider: "openai" | "google";
@@ -209,6 +210,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                         path: string;
                         kind: "path";
                     };
+                } | {
+                    type: "imagePrompt";
+                    prompt: string;
                 }> | undefined;
             } | undefined;
             audioParams?: {
@@ -231,7 +235,6 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 lang?: string | undefined;
             } | undefined;
             imageNames?: string[] | undefined;
-            imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
             htmlPrompt?: {
                 prompt: string;
@@ -263,6 +266,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                     path: string;
                     kind: "path";
                 };
+            } | {
+                type: "imagePrompt";
+                prompt: string;
             }> | undefined;
         } | undefined;
         movieParams?: {
@@ -288,7 +294,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             lang?: string | undefined;
         } | undefined;
         references?: {
-            type: "image" | "audio" | "article" | "paper" | "video";
+            type: string;
             url: string;
             title?: string | undefined;
             description?: string | undefined;

package/lib/utils/string.js

@@ -20,18 +20,18 @@ export const recursiveSplitJa = (text) => {
     const delimiters = ["。", "？", "！", "、"];
     return delimiters
         .reduce((textData, delimiter) => {
-        return textData.map((text) => splitIntoSentencesJa(text, delimiter, 7)).flat(1);
+        return textData.map((textInner) => splitIntoSentencesJa(textInner, delimiter, 7)).flat(1);
     }, [text])
         .flat(1);
 };
 export function replacePairsJa(str, replacements) {
-    replacements.forEach(({ from, to }) => {
+    return replacements.reduce((tmp, current) => {
+        const { from, to } = current;
         // Escape any special regex characters in the 'from' string.
         const escapedFrom = from.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
         const regex = new RegExp(escapedFrom, "g");
-        str = str.replace(regex, to);
-    });
-    return str;
+        return tmp.replace(regex, to);
+    }, str);
 }
 export const replacementsJa = [
     { from: "Anthropic", to: "アンスロピック" },

package/lib/utils/utils.d.ts

@@ -18,5 +18,12 @@ export declare const text2hash: (input: string) => string;
 export declare const localizedText: (beat: MulmoBeat, multiLingualData?: MulmoStudioMultiLingualData, lang?: string) => string;
 export declare const sleep: (milliseconds: number) => Promise<unknown>;
 export declare function userAssert(condition: boolean, message: string): asserts condition;
-export declare const settings2GraphAIConfig: (settings?: Record<string, string>) => ConfigDataDictionary<DefaultConfigData>;
+export declare const settings2GraphAIConfig: (settings?: Record<string, string>, env?: Record<string, string | undefined>) => ConfigDataDictionary<DefaultConfigData>;
 export declare const getExtention: (contentType: string | null, url: string) => string;
+type Primitive = string | number | boolean | symbol | bigint;
+type CleanableValue = Primitive | null | undefined | CleanableObject | CleanableValue[];
+type CleanableObject = {
+    [key: string]: CleanableValue;
+};
+export declare const deepClean: <T extends CleanableValue>(input: T) => T | undefined;
+export {};

package/lib/utils/utils.js

@@ -60,47 +60,42 @@ export function userAssert(condition, message) {
         throw new Error(message);
     }
 }
-export const settings2GraphAIConfig = (settings) => {
-    const config = {};
-    if (settings) {
-        if (settings.OPENAI_API_KEY) {
-            config.openAIAgent = {
-                apiKey: settings.OPENAI_API_KEY,
-            };
-            config.ttsOpenaiAgent = {
-                apiKey: settings.OPENAI_API_KEY,
-            };
-            config.imageOpenaiAgent = {
-                apiKey: settings.OPENAI_API_KEY,
-            };
-        }
-        if (settings.ANTHROPIC_API_TOKEN) {
-            config.anthropicAgent = {
-                apiKey: settings.ANTHROPIC_API_TOKEN,
-            };
-        }
-        if (settings.REPLICATE_API_TOKEN) {
-            config.movieReplicateAgent = {
-                apiKey: settings.REPLICATE_API_TOKEN,
-            };
-        }
-        if (settings.NIJIVOICE_API_KEY) {
-            config.ttsNijivoiceAgent = {
-                apiKey: settings.NIJIVOICE_API_KEY,
-            };
-        }
-        if (settings.ELEVENLABS_API_KEY) {
-            config.ttsElevenlabsAgent = {
-                apiKey: settings.ELEVENLABS_API_KEY,
-            };
-        }
+export const settings2GraphAIConfig = (settings, env) => {
+    const getKey = (prefix, key) => {
+        return settings?.[`${prefix}_${key}`] ?? settings?.[key] ?? env?.[`${prefix}_${key}`] ?? env?.[key];
+    };
+    const config = {
+        openAIAgent: {
+            apiKey: getKey("LLM", "OPENAI_API_KEY"),
+            baseURL: getKey("LLM", "OPENAI_BASE_URL"),
+        },
+        ttsOpenaiAgent: {
+            apiKey: getKey("TTS", "OPENAI_API_KEY"),
+            baseURL: getKey("TTS", "OPENAI_BASE_URL"),
+        },
+        imageOpenaiAgent: {
+            apiKey: getKey("IMAGE", "OPENAI_API_KEY"),
+            baseURL: getKey("IMAGE", "OPENAI_BASE_URL"),
+        },
+        anthropicAgent: {
+            apiKey: getKey("LLM", "ANTHROPIC_API_TOKEN"),
+        },
+        movieReplicateAgent: {
+            apiKey: getKey("MOVIE", "REPLICATE_API_TOKEN"),
+        },
+        ttsNijivoiceAgent: {
+            apiKey: getKey("TTS", "NIJIVOICE_API_KEY"),
+        },
+        ttsElevenlabsAgent: {
+            apiKey: getKey("TTS", "ELEVENLABS_API_KEY"),
+        },
         // TODO
         // browserlessAgent
         // ttsGoogleAgent
         // geminiAgent, groqAgent for tool
         // TAVILY_API_KEY ( for deep research)
-    }
-    return config;
+    };
+    return deepClean(config) ?? {};
 };
 export const getExtention = (contentType, url) => {
     if (contentType?.includes("jpeg") || contentType?.includes("jpg")) {
@@ -116,3 +111,23 @@ export const getExtention = (contentType, url) => {
     }
     return "png"; // default
 };
+export const deepClean = (input) => {
+    if (input === null || input === undefined || input === "") {
+        return undefined;
+    }
+    if (Array.isArray(input)) {
+        const cleanedArray = input.map(deepClean).filter((v) => v !== undefined);
+        return cleanedArray.length > 0 ? cleanedArray : undefined;
+    }
+    if (typeof input === "object") {
+        const result = {};
+        for (const [key, value] of Object.entries(input)) {
+            const cleaned = deepClean(value);
+            if (cleaned !== undefined) {
+                result[key] = cleaned;
+            }
+        }
+        return Object.keys(result).length > 0 ? result : undefined;
+    }
+    return input;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "",
   "type": "module",
   "main": "lib/index.js",
@@ -59,7 +59,7 @@
   },
   "homepage": "https://github.com/receptron/mulmocast-cli#readme",
   "dependencies": {
-    "@google-cloud/text-to-speech": "^6.1.0",
+    "@google-cloud/text-to-speech": "^6.2.0",
     "@graphai/anthropic_agent": "^2.0.5",
     "@graphai/browserless_agent": "^2.0.1",
     "@graphai/gemini_agent": "^2.0.0",
@@ -69,18 +69,18 @@
     "@graphai/stream_agent_filter": "^2.0.2",
     "@graphai/vanilla": "^2.0.5",
     "@graphai/vanilla_node_agents": "^2.0.1",
-    "@modelcontextprotocol/sdk": "^1.13.1",
+    "@modelcontextprotocol/sdk": "^1.15.1",
     "@tavily/core": "^0.5.9",
     "canvas": "^3.1.2",
     "clipboardy": "^4.0.0",
-    "dotenv": "^17.1.0",
+    "dotenv": "^17.2.0",
     "fluent-ffmpeg": "^2.1.3",
     "google-auth-library": "^9.15.1",
     "graphai": "^2.0.12",
     "inquirer": "^12.7.0",
     "marked": "^16.0.0",
     "ora": "^8.2.0",
-    "puppeteer": "^24.12.0",
+    "puppeteer": "^24.12.1",
     "replicate": "^1.0.1",
     "yaml": "^2.8.0",
     "yargs": "^18.0.0",
@@ -93,9 +93,10 @@
     "@receptron/test_utils": "^2.0.0",
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
-    "eslint": "^9.30.1",
+    "eslint": "^9.31.0",
     "eslint-config-prettier": "^10.1.5",
     "eslint-plugin-prettier": "^5.5.1",
+    "eslint-plugin-sonarjs": "^3.0.4",
     "prettier": "^3.6.2",
     "ts-node": "^10.9.2",
     "tsx": "^4.20.3",

package/scripts/templates/html.json

@@ -0,0 +1,42 @@
+{
+  "$mulmocast": {
+    "version": "1.0",
+    "credit": "closing"
+  },
+  "references": [
+    {
+      "url": "https://www.somegreatwebsite.com/article/123",
+      "title": "Title of the article we are referencing",
+      "type": "[TYPE OF ARTICLE: article, paper, image, video, audio]"
+    }
+  ],
+  "title": "[TITLE: Brief, engaging title for the topic]",
+  "htmlImageParams": {
+    "provider": "anthropic",
+    "model": "claude-3-7-sonnet-20250219"
+  },
+  "lang": "en",
+  "beats": [
+    {
+      "text": "[NARRATION: Narration for the beat.]",
+      "htmlPrompt": {
+        "prompt": "[PROMPT to create appropriate HTML page for the beat.]"
+      }
+    },
+    {
+      "text": "[NARRATION: Narration for the beat.]",
+      "htmlPrompt": {
+        "prompt": "[PROMPT to create appropriate HTML page for the beat with the data.]",
+        "data": {
+          "description": "DATA TO BE PRESENTED IN THIS BEAT (in any format)]",
+          "net_income": {
+            "Q2 FY2024": 320,
+            "Q3 FY2024": 333,
+            "Q4 FY2024": 350
+          },
+          "unit": "USD (Million)"
+        }
+      }
+    }
+  ]
+}

package/scripts/templates/image_refs.json

@@ -0,0 +1,35 @@
+{
+  "$mulmocast": {
+    "version": "1.0"
+  },
+  "title": "[TITLE OF THE PRESENTAITON OR STORY]",
+  "imageParams": {
+    "images": {
+      "[CHARACTER_ID_1]": {
+        "type": "imagePrompt",
+        "prompt": "[IMAGE PROMPT FOR THIS CHARACTER]"
+      },
+      "[CHARACTER_ID_2]": {
+        "type": "imagePrompt",
+        "prompt": "[IMAGE PROMPT FOR THIS CHARACTER]"
+      }
+    }
+  },
+  "beats": [
+    {
+      "text": "[NARRATION FOR THIS BEAT]",
+      "imagePrompt": "[IMAGE PROMPT FOR THIS BEAT (with both characters)]",
+      "imageNames": ["[CHARACTER_ID_1]", "[CHARACTER_ID_2]"]
+    },
+    {
+      "text": "[NARRATION FOR THIS BEAT]",
+      "imagePrompt": "[IMAGE PROMPT FOR THIS BEAT (only character 1)]",
+      "imageNames": ["[CHARACTER_ID_1]"]
+    },
+    {
+      "text": "[NARRATION FOR THIS BEAT]",
+      "imagePrompt": "[IMAGE PROMPT FOR THIS BEAT (no character)]",
+      "imageNames": []
+    }
+  ]
+}