mulmocast 0.1.0 → 0.1.2

package/lib/types/schema.js

@@ -20,7 +20,7 @@ export const speechOptionsSchema = z
 })
     .strict();
 const speakerIdSchema = z.string();
-export const text2SpeechProviderSchema = z.union([z.literal("openai"), z.literal("nijivoice"), z.literal("google"), z.literal("elevenlabs")]).default("openai");
+export const text2SpeechProviderSchema = z.enum(["openai", "nijivoice", "google", "elevenlabs"]).default("openai");
 export const speakerDataSchema = z
     .object({
     displayName: z.record(langSchema, z.string()).optional(),
@@ -159,19 +159,19 @@ export const mulmoFillOptionSchema = z
     style: z.enum(["aspectFit", "aspectFill"]).default("aspectFit"),
 })
     .describe("How to handle aspect ratio differences between image and canvas");
-export const text2ImageProviderSchema = z.union([z.literal("openai"), z.literal("google")]).default("openai");
+export const text2ImageProviderSchema = z.enum(["openai", "google"]).default("openai");
 // NOTE: This is for UI only. (until we figure out how to use it in mulmoImageParamsSchema)
 export const mulmoOpenAIImageModelSchema = z
     .object({
     provider: z.literal("openai"),
-    model: z.union([z.literal("dall-e-3"), z.literal("gpt-image-1")]).optional(),
+    model: z.enum(["dall-e-3", "gpt-image-1"]).optional(),
 })
     .strict();
 // NOTE: This is for UI only. (until we figure out how to use it in mulmoImageParamsSchema)
 export const mulmoGoogleImageModelSchema = z
     .object({
     provider: z.literal("google"),
-    model: z.union([z.literal("imagen-3.0-fast-generate-001"), z.literal("imagen-3.0-generate-002"), z.literal("imagen-3.0-capability-001")]).optional(),
+    model: z.enum(["imagen-3.0-fast-generate-001", "imagen-3.0-generate-002", "imagen-3.0-capability-001"]).optional(),
 })
     .strict();
 export const mulmoImageParamsSchema = z
@@ -265,8 +265,22 @@ export const mulmoSpeechParamsSchema = z
     speakers: speakerDictionarySchema,
 })
     .strict();
-export const text2HtmlImageProviderSchema = z.union([z.literal("openai"), z.literal("anthropic")]).default("openai");
-export const text2MovieProviderSchema = z.union([z.literal("openai"), z.literal("google"), z.literal("replicate")]).default("google");
+export const text2HtmlImageProviderSchema = z.enum(["openai", "anthropic"]).default("openai");
+export const text2MovieProviderSchema = z.enum(["google", "replicate"]).default("google");
+// NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
+export const mulmoGoogleMovieModelSchema = z
+    .object({
+    provider: z.literal("google"),
+    model: z.enum(["veo-2.0-generate-001"]).optional(),
+})
+    .strict();
+// NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
+export const mulmoReplicateMovieModelSchema = z
+    .object({
+    provider: z.literal("replicate"),
+    model: z.enum(["bytedance/seedance-1-lite", "kwaivgi/kling-v2.1", "google/veo-3"]).optional(),
+})
+    .strict();
 export const mulmoTransitionSchema = z.object({
     type: z.enum(["fade", "slideout_left"]),
     duration: z.number().min(0).max(2).default(0.3), // transition duration in seconds
@@ -315,7 +329,7 @@ export const mulmoReferenceSchema = z.object({
     url: URLStringSchema,
     title: z.string().optional(),
     description: z.string().optional(),
-    type: z.union([z.literal("article"), z.literal("paper"), z.literal("image"), z.literal("video"), z.literal("audio")]).default("article"),
+    type: z.enum(["article", "paper", "image", "video", "audio"]).default("article"),
 });
 export const mulmoScriptSchema = mulmoPresentationStyleSchema
     .extend({
@@ -363,6 +377,7 @@ export const mulmoSessionStateSchema = z.object({
         movie: z.record(z.number().int(), z.boolean()),
         multiLingual: z.record(z.number().int(), z.boolean()),
         caption: z.record(z.number().int(), z.boolean()),
+        html: z.record(z.number().int(), z.boolean()),
     }),
 });
 export const mulmoStudioSchema = z

package/lib/types/type.d.ts

@@ -1,4 +1,4 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import { LLM } from "../utils/utils.js";
 import { z } from "zod";
@@ -33,6 +33,8 @@ export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
 export type MulmoMovieParams = z.infer<typeof mulmoMovieParamsSchema>;
 export type MulmoOpenAIImageModel = z.infer<typeof mulmoOpenAIImageModelSchema>;
 export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
+export type MulmoGoogleMovieModel = z.infer<typeof mulmoGoogleMovieModelSchema>;
+export type MulmoReplicateMovieModel = z.infer<typeof mulmoReplicateMovieModelSchema>;
 export type MulmoTextSlideMedia = z.infer<typeof mulmoTextSlideMediaSchema>;
 export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
@@ -88,7 +90,7 @@ export type Text2HtmlAgentInfo = {
 export type BeatMediaType = "movie" | "image";
 export type StoryToScriptGenerateMode = (typeof storyToScriptGenerateMode)[keyof typeof storyToScriptGenerateMode];
 export type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
-export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie";
+export type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie" | "html";
 export type SessionProgressEvent = {
     kind: "session";
     sessionType: SessionType;

package/lib/utils/context.d.ts

@@ -271,7 +271,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 }> | undefined;
             } | undefined;
             movieParams?: {
-                provider?: "openai" | "google" | "replicate" | undefined;
+                provider?: "google" | "replicate" | undefined;
                 model?: string | undefined;
                 fillOption?: {
                     style: "aspectFit" | "aspectFill";
@@ -321,6 +321,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             movie: {};
             multiLingual: {};
             caption: {};
+            html: {};
         };
     };
     presentationStyle: {
@@ -389,7 +390,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             }> | undefined;
         } | undefined;
         movieParams?: {
-            provider?: "openai" | "google" | "replicate" | undefined;
+            provider?: "google" | "replicate" | undefined;
             model?: string | undefined;
             fillOption?: {
                 style: "aspectFit" | "aspectFill";

package/lib/utils/context.js

@@ -56,6 +56,7 @@ const initSessionState = () => {
             movie: {},
             multiLingual: {},
             caption: {},
+            html: {},
         },
     };
 };

package/lib/utils/file.js

@@ -81,10 +81,18 @@ export const getOutputVideoFilePath = (outDirPath, fileName, lang, caption) => {
 export const imageSuffix = "p";
 export const getBeatPngImagePath = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
+    if (beat?.id) {
+        return `${imageProjectDirPath}/${beat.id}.png`;
+    }
     return `${imageProjectDirPath}/${index}${imageSuffix}.png`;
 };
 export const getBeatMoviePath = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
+    if (beat?.id) {
+        return `${imageProjectDirPath}/${beat.id}.mov`;
+    }
     return `${imageProjectDirPath}/${index}.mov`;
 };
 export const getReferenceImagePath = (context, key, extension) => {

package/lib/utils/filters.js

@@ -22,18 +22,23 @@ export const fileCacheAgentFilter = async (context, next) => {
         }
     };
     if (await shouldUseCache()) {
-        GraphAILogger.debug("cache");
+        GraphAILogger.debug(`cache: ${path.basename(file)}`);
         return true;
     }
     try {
         MulmoStudioContextMethods.setBeatSessionState(mulmoContext, sessionType, index, true);
-        const output = (await next(context));
-        const buffer = output ? output["buffer"] : undefined;
+        const output = (await next(context)) || undefined;
+        const { buffer, text } = output ?? {};
         if (buffer) {
             writingMessage(file);
             await fsPromise.writeFile(file, buffer);
             return true;
         }
+        else if (text) {
+            writingMessage(file);
+            await fsPromise.writeFile(file, text, "utf-8");
+            return true;
+        }
         GraphAILogger.log("no cache, no buffer: " + file);
         return false;
     }

package/lib/utils/preprocess.d.ts

@@ -266,7 +266,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             }> | undefined;
         } | undefined;
         movieParams?: {
-            provider?: "openai" | "google" | "replicate" | undefined;
+            provider?: "google" | "replicate" | undefined;
             model?: string | undefined;
             fillOption?: {
                 style: "aspectFit" | "aspectFill";

package/lib/utils/utils.d.ts

@@ -19,3 +19,4 @@ export declare const localizedText: (beat: MulmoBeat, multiLingualData?: MulmoSt
 export declare const sleep: (milliseconds: number) => Promise<unknown>;
 export declare function userAssert(condition: boolean, message: string): asserts condition;
 export declare const settings2GraphAIConfig: (settings?: Record<string, string>) => ConfigDataDictionary<DefaultConfigData>;
+export declare const getExtention: (contentType: string | null, url: string) => string;

package/lib/utils/utils.js

@@ -102,3 +102,17 @@ export const settings2GraphAIConfig = (settings) => {
     }
     return config;
 };
+export const getExtention = (contentType, url) => {
+    if (contentType?.includes("jpeg") || contentType?.includes("jpg")) {
+        return "jpg";
+    }
+    else if (contentType?.includes("png")) {
+        return "png";
+    }
+    // Fall back to URL extension
+    const urlExtension = url.split(".").pop()?.toLowerCase();
+    if (urlExtension && ["jpg", "jpeg", "png"].includes(urlExtension)) {
+        return urlExtension === "jpeg" ? "jpg" : urlExtension;
+    }
+    return "png"; // default
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "",
   "type": "module",
   "main": "lib/index.js",
@@ -60,47 +60,47 @@
   "homepage": "https://github.com/receptron/mulmocast-cli#readme",
   "dependencies": {
     "@google-cloud/text-to-speech": "^6.1.0",
-    "@graphai/anthropic_agent": "^2.0.2",
+    "@graphai/anthropic_agent": "^2.0.5",
     "@graphai/browserless_agent": "^2.0.1",
     "@graphai/gemini_agent": "^2.0.0",
     "@graphai/groq_agent": "^2.0.0",
     "@graphai/input_agents": "^1.0.1",
     "@graphai/openai_agent": "^2.0.3",
     "@graphai/stream_agent_filter": "^2.0.2",
-    "@graphai/vanilla": "^2.0.4",
+    "@graphai/vanilla": "^2.0.5",
     "@graphai/vanilla_node_agents": "^2.0.1",
     "@modelcontextprotocol/sdk": "^1.13.1",
-    "@tavily/core": "^0.5.8",
+    "@tavily/core": "^0.5.9",
     "canvas": "^3.1.2",
     "clipboardy": "^4.0.0",
-    "dotenv": "^17.0.0",
+    "dotenv": "^17.1.0",
     "fluent-ffmpeg": "^2.1.3",
     "google-auth-library": "^9.15.1",
-    "graphai": "^2.0.9",
-    "inquirer": "^12.6.3",
+    "graphai": "^2.0.12",
+    "inquirer": "^12.7.0",
     "marked": "^16.0.0",
     "ora": "^8.2.0",
-    "puppeteer": "^24.11.1",
+    "puppeteer": "^24.12.0",
     "replicate": "^1.0.1",
     "yaml": "^2.8.0",
     "yargs": "^18.0.0",
-    "zod": "^3.25.67",
+    "zod": "^3.25.76",
     "zod-to-json-schema": "^3.24.6"
   },
   "devDependencies": {
     "@anatine/zod-mock": "^3.14.0",
-    "@faker-js/faker": "^9.8.0",
+    "@faker-js/faker": "^9.9.0",
     "@receptron/test_utils": "^2.0.0",
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
-    "eslint": "^9.30.0",
+    "eslint": "^9.30.1",
     "eslint-config-prettier": "^10.1.5",
     "eslint-plugin-prettier": "^5.5.1",
     "prettier": "^3.6.2",
     "ts-node": "^10.9.2",
     "tsx": "^4.20.3",
     "typescript": "^5.7.3",
-    "typescript-eslint": "^8.35.0"
+    "typescript-eslint": "^8.36.0"
   },
   "engines": {
     "node": ">=18.0.0"

package/scripts/templates/voice_over.json

@@ -0,0 +1,60 @@
+{
+  "$mulmocast": {
+    "version": "1.0"
+  },
+  "title": "Voice Over Test",
+  "captionParams": {
+    "lang": "en"
+  },
+  "canvasSize": {
+    "width": 1552,
+    "height": 2064
+  },
+  "beats": [
+    {
+      "text": "Description of this section of the movie",
+      "image": {
+        "type": "movie",
+        "source": {
+          "kind": "url",
+          "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/movies/actions.mp4"
+        }
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 8 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 8.0
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 14.5 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 14.5
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 21 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 21.0
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 25 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 25.0
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 30 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 30.0
+      }
+    }
+  ]
+}