mulmocast 0.0.28 → 0.1.1

package/lib/types/schema.js

@@ -20,7 +20,7 @@ export const speechOptionsSchema = z
 })
     .strict();
 const speakerIdSchema = z.string();
-export const text2SpeechProviderSchema = z.union([z.literal("openai"), z.literal("nijivoice"), z.literal("google"), z.literal("elevenlabs")]).default("openai");
+export const text2SpeechProviderSchema = z.enum(["openai", "nijivoice", "google", "elevenlabs"]).default("openai");
 export const speakerDataSchema = z
     .object({
     displayName: z.record(langSchema, z.string()).optional(),
@@ -117,6 +117,12 @@ export const mulmoBeatReferenceMediaSchema = z
     id: z.string().optional().describe("Specifies the beat to reference."),
 })
     .strict();
+export const mulmoVoiceOverMediaSchema = z
+    .object({
+    type: z.literal("voice_over"),
+    startAt: z.number().optional().describe("The time to start the voice over the video in seconds."),
+})
+    .strict();
 export const mulmoImageAssetSchema = z.union([
     mulmoMarkdownMediaSchema,
     mulmoWebMediaSchema,
@@ -131,6 +137,7 @@ export const mulmoImageAssetSchema = z.union([
     mulmoMermaidMediaSchema,
     mulmoHtmlTailwindMediaSchema,
     mulmoBeatReferenceMediaSchema,
+    mulmoVoiceOverMediaSchema,
 ]);
 const mulmoAudioMediaSchema = z
     .object({
@@ -152,8 +159,24 @@ export const mulmoFillOptionSchema = z
     style: z.enum(["aspectFit", "aspectFill"]).default("aspectFit"),
 })
     .describe("How to handle aspect ratio differences between image and canvas");
+export const text2ImageProviderSchema = z.enum(["openai", "google"]).default("openai");
+// NOTE: This is for UI only. (until we figure out how to use it in mulmoImageParamsSchema)
+export const mulmoOpenAIImageModelSchema = z
+    .object({
+    provider: z.literal("openai"),
+    model: z.enum(["dall-e-3", "gpt-image-1"]).optional(),
+})
+    .strict();
+// NOTE: This is for UI only. (until we figure out how to use it in mulmoImageParamsSchema)
+export const mulmoGoogleImageModelSchema = z
+    .object({
+    provider: z.literal("google"),
+    model: z.enum(["imagen-3.0-fast-generate-001", "imagen-3.0-generate-002", "imagen-3.0-capability-001"]).optional(),
+})
+    .strict();
 export const mulmoImageParamsSchema = z
     .object({
+    provider: text2ImageProviderSchema, // has default value
     model: z.string().optional(), // default: provider specific
     style: z.string().optional(), // optional image style
     moderation: z.string().optional(), // optional image style
@@ -209,7 +232,8 @@ export const mulmoBeatSchema = z
     audioParams: beatAudioParamsSchema.optional(), // beat specific parameters
     movieParams: z
         .object({
-        fillOption: mulmoFillOptionSchema,
+        fillOption: mulmoFillOptionSchema.optional(),
+        speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."),
     })
         .optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
@@ -241,9 +265,22 @@ export const mulmoSpeechParamsSchema = z
     speakers: speakerDictionarySchema,
 })
     .strict();
-export const text2ImageProviderSchema = z.union([z.literal("openai"), z.literal("google")]).default("openai");
-export const text2HtmlImageProviderSchema = z.union([z.literal("openai"), z.literal("anthropic")]).default("openai");
-export const text2MovieProviderSchema = z.union([z.literal("openai"), z.literal("google"), z.literal("replicate")]).default("google");
+export const text2HtmlImageProviderSchema = z.enum(["openai", "anthropic"]).default("openai");
+export const text2MovieProviderSchema = z.enum(["google", "replicate"]).default("google");
+// NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
+export const mulmoGoogleMovieModelSchema = z
+    .object({
+    provider: z.literal("google"),
+    model: z.enum(["veo-2.0-generate-001"]).optional(),
+})
+    .strict();
+// NOTE: This is UI only. (until we figure out how to use it in mulmoMovieParamsSchema)
+export const mulmoReplicateMovieModelSchema = z
+    .object({
+    provider: z.literal("replicate"),
+    model: z.enum(["bytedance/seedance-1-lite", "kwaivgi/kling-v2.1", "google/veo-3"]).optional(),
+})
+    .strict();
 export const mulmoTransitionSchema = z.object({
     type: z.enum(["fade", "slideout_left"]),
     duration: z.number().min(0).max(2).default(0.3), // transition duration in seconds
@@ -269,11 +306,7 @@ export const mulmoPresentationStyleSchema = z.object({
             },
         },
     }),
-    imageParams: mulmoImageParamsSchema
-        .extend({
-        provider: text2ImageProviderSchema, // has default value
-    })
-        .optional(),
+    imageParams: mulmoImageParamsSchema.optional(),
     movieParams: mulmoMovieParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema
         .extend({
@@ -296,7 +329,7 @@ export const mulmoReferenceSchema = z.object({
     url: URLStringSchema,
     title: z.string().optional(),
     description: z.string().optional(),
-    type: z.union([z.literal("article"), z.literal("paper"), z.literal("image"), z.literal("video"), z.literal("audio")]).default("article"),
+    type: z.enum(["article", "paper", "image", "video", "audio"]).default("article"),
 });
 export const mulmoScriptSchema = mulmoPresentationStyleSchema
     .extend({
@@ -315,6 +348,10 @@ export const mulmoStudioBeatSchema = z
     .object({
     hash: z.string().optional(),
     duration: z.number().optional(),
+    startAt: z.number().optional(),
+    audioDuration: z.number().optional(),
+    movieDuration: z.number().optional(),
+    silenceDuration: z.number().optional(),
     audioFile: z.string().optional(),
     imageFile: z.string().optional(), // path to the image
     movieFile: z.string().optional(), // path to the movie file

package/lib/types/type.d.ts

@@ -1,4 +1,4 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import { LLM } from "../utils/utils.js";
 import { z } from "zod";
@@ -31,6 +31,10 @@ export type MulmoStudioMultiLingual = z.infer<typeof mulmoStudioMultiLingualSche
 export type MulmoStudioMultiLingualData = z.infer<typeof mulmoStudioMultiLingualDataSchema>;
 export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
 export type MulmoMovieParams = z.infer<typeof mulmoMovieParamsSchema>;
+export type MulmoOpenAIImageModel = z.infer<typeof mulmoOpenAIImageModelSchema>;
+export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
+export type MulmoGoogleMovieModel = z.infer<typeof mulmoGoogleMovieModelSchema>;
+export type MulmoReplicateMovieModel = z.infer<typeof mulmoReplicateMovieModelSchema>;
 export type MulmoTextSlideMedia = z.infer<typeof mulmoTextSlideMediaSchema>;
 export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
@@ -74,7 +78,6 @@ export type ImageProcessorParams = {
 export type PDFMode = (typeof pdf_modes)[number];
 export type PDFSize = (typeof pdf_sizes)[number];
 export type Text2ImageAgentInfo = {
-    provider: Text2ImageProvider;
     agent: string;
     imageParams: MulmoImageParams;
 };

package/lib/utils/context.d.ts

@@ -7,7 +7,11 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
     studio: {
         beats: {
             duration?: number | undefined;
+            startAt?: number | undefined;
             hash?: string | undefined;
+            audioDuration?: number | undefined;
+            movieDuration?: number | undefined;
+            silenceDuration?: number | undefined;
             audioFile?: string | undefined;
             imageFile?: string | undefined;
             movieFile?: string | undefined;
@@ -149,6 +153,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | {
                     type: "beat";
                     id?: string | undefined;
+                } | {
+                    type: "voice_over";
+                    startAt?: number | undefined;
                 } | {
                     type: "movie";
                     source: {
@@ -188,6 +195,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
                 description?: string | undefined;
                 imageParams?: {
+                    provider: "openai" | "google";
                     style?: string | undefined;
                     model?: string | undefined;
                     moderation?: string | undefined;
@@ -212,9 +220,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     padding?: number | undefined;
                 } | undefined;
                 movieParams?: {
-                    fillOption: {
+                    speed?: number | undefined;
+                    fillOption?: {
                         style: "aspectFit" | "aspectFill";
-                    };
+                    } | undefined;
                 } | undefined;
                 htmlImageParams?: {
                     model?: string | undefined;
@@ -262,7 +271,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 }> | undefined;
             } | undefined;
             movieParams?: {
-                provider?: "openai" | "google" | "replicate" | undefined;
+                provider?: "google" | "replicate" | undefined;
                 model?: string | undefined;
                 fillOption?: {
                     style: "aspectFit" | "aspectFill";
@@ -380,7 +389,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
             }> | undefined;
         } | undefined;
         movieParams?: {
-            provider?: "openai" | "google" | "replicate" | undefined;
+            provider?: "google" | "replicate" | undefined;
             model?: string | undefined;
             fillOption?: {
                 style: "aspectFit" | "aspectFill";

package/lib/utils/file.js

@@ -81,10 +81,18 @@ export const getOutputVideoFilePath = (outDirPath, fileName, lang, caption) => {
 export const imageSuffix = "p";
 export const getBeatPngImagePath = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
+    if (beat?.id) {
+        return `${imageProjectDirPath}/${beat.id}.png`;
+    }
     return `${imageProjectDirPath}/${index}${imageSuffix}.png`;
 };
 export const getBeatMoviePath = (context, index) => {
     const imageProjectDirPath = MulmoStudioContextMethods.getImageProjectDirPath(context);
+    const beat = context.studio.script.beats[index]; // beat could be undefined only in a test case.
+    if (beat?.id) {
+        return `${imageProjectDirPath}/${beat.id}.mov`;
+    }
     return `${imageProjectDirPath}/${index}.mov`;
 };
 export const getReferenceImagePath = (context, key, extension) => {

package/lib/utils/image_plugins/index.d.ts

@@ -6,4 +6,5 @@ import * as pluginHtmlTailwind from "./html_tailwind.js";
 import * as pluginImage from "./image.js";
 import * as pluginMovie from "./movie.js";
 import * as pluginBeat from "./beat.js";
-export declare const findImagePlugin: (imageType?: string) => typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginChart | typeof pluginMermaid | typeof pluginHtmlTailwind | typeof pluginImage | typeof pluginMovie | typeof pluginBeat | undefined;
+import * as pluginVoiceOver from "./voice_over.js";
+export declare const findImagePlugin: (imageType?: string) => typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginChart | typeof pluginMermaid | typeof pluginHtmlTailwind | typeof pluginImage | typeof pluginMovie | typeof pluginBeat | typeof pluginVoiceOver | undefined;

package/lib/utils/image_plugins/index.js

@@ -6,7 +6,8 @@ import * as pluginHtmlTailwind from "./html_tailwind.js";
 import * as pluginImage from "./image.js";
 import * as pluginMovie from "./movie.js";
 import * as pluginBeat from "./beat.js";
-const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie, pluginHtmlTailwind, pluginBeat];
+import * as pluginVoiceOver from "./voice_over.js";
+const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie, pluginHtmlTailwind, pluginBeat, pluginVoiceOver];
 export const findImagePlugin = (imageType) => {
     return imagePlugins.find((plugin) => plugin.imageType === imageType);
 };

package/lib/utils/image_plugins/voice_over.d.ts

@@ -0,0 +1,5 @@
+import { ImageProcessorParams } from "../../types/index.js";
+export declare const imageType = "voice_over";
+export declare const processBeatReference: (__: ImageProcessorParams) => Promise<undefined>;
+export declare const process: (__: ImageProcessorParams) => Promise<undefined>;
+export declare const path: (__: ImageProcessorParams) => undefined;

package/lib/utils/image_plugins/voice_over.js

@@ -0,0 +1,9 @@
+export const imageType = "voice_over";
+export const processBeatReference = async (__) => {
+    // For voice-over, return undefined to indicate no image should be generated
+    return undefined;
+};
+export const process = processBeatReference;
+export const path = (__) => {
+    return undefined;
+};

package/lib/utils/preprocess.d.ts

@@ -2,7 +2,11 @@ import { MulmoStudio, MulmoScript } from "../types/index.js";
 export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string) => {
     beats: {
         duration?: number | undefined;
+        startAt?: number | undefined;
         hash?: string | undefined;
+        audioDuration?: number | undefined;
+        movieDuration?: number | undefined;
+        silenceDuration?: number | undefined;
         audioFile?: string | undefined;
         imageFile?: string | undefined;
         movieFile?: string | undefined;
@@ -144,6 +148,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | {
                 type: "beat";
                 id?: string | undefined;
+            } | {
+                type: "voice_over";
+                startAt?: number | undefined;
             } | {
                 type: "movie";
                 source: {
@@ -183,6 +190,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
             description?: string | undefined;
             imageParams?: {
+                provider: "openai" | "google";
                 style?: string | undefined;
                 model?: string | undefined;
                 moderation?: string | undefined;
@@ -207,9 +215,10 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 padding?: number | undefined;
             } | undefined;
             movieParams?: {
-                fillOption: {
+                speed?: number | undefined;
+                fillOption?: {
                     style: "aspectFit" | "aspectFill";
-                };
+                } | undefined;
             } | undefined;
             htmlImageParams?: {
                 model?: string | undefined;
@@ -257,7 +266,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             }> | undefined;
         } | undefined;
         movieParams?: {
-            provider?: "openai" | "google" | "replicate" | undefined;
+            provider?: "google" | "replicate" | undefined;
             model?: string | undefined;
             fillOption?: {
                 style: "aspectFit" | "aspectFill";

package/lib/utils/utils.d.ts

@@ -19,3 +19,4 @@ export declare const localizedText: (beat: MulmoBeat, multiLingualData?: MulmoSt
 export declare const sleep: (milliseconds: number) => Promise<unknown>;
 export declare function userAssert(condition: boolean, message: string): asserts condition;
 export declare const settings2GraphAIConfig: (settings?: Record<string, string>) => ConfigDataDictionary<DefaultConfigData>;
+export declare const getExtention: (contentType: string | null, url: string) => string;

package/lib/utils/utils.js

@@ -102,3 +102,17 @@ export const settings2GraphAIConfig = (settings) => {
     }
     return config;
 };
+export const getExtention = (contentType, url) => {
+    if (contentType?.includes("jpeg") || contentType?.includes("jpg")) {
+        return "jpg";
+    }
+    else if (contentType?.includes("png")) {
+        return "png";
+    }
+    // Fall back to URL extension
+    const urlExtension = url.split(".").pop()?.toLowerCase();
+    if (urlExtension && ["jpg", "jpeg", "png"].includes(urlExtension)) {
+        return urlExtension === "jpeg" ? "jpg" : urlExtension;
+    }
+    return "png"; // default
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.0.28",
+  "version": "0.1.1",
   "description": "",
   "type": "module",
   "main": "lib/index.js",
@@ -60,47 +60,47 @@
   "homepage": "https://github.com/receptron/mulmocast-cli#readme",
   "dependencies": {
     "@google-cloud/text-to-speech": "^6.1.0",
-    "@graphai/anthropic_agent": "^2.0.2",
+    "@graphai/anthropic_agent": "^2.0.5",
     "@graphai/browserless_agent": "^2.0.1",
     "@graphai/gemini_agent": "^2.0.0",
     "@graphai/groq_agent": "^2.0.0",
     "@graphai/input_agents": "^1.0.1",
     "@graphai/openai_agent": "^2.0.3",
     "@graphai/stream_agent_filter": "^2.0.2",
-    "@graphai/vanilla": "^2.0.4",
+    "@graphai/vanilla": "^2.0.5",
     "@graphai/vanilla_node_agents": "^2.0.1",
     "@modelcontextprotocol/sdk": "^1.13.1",
-    "@tavily/core": "^0.5.8",
+    "@tavily/core": "^0.5.9",
     "canvas": "^3.1.2",
     "clipboardy": "^4.0.0",
-    "dotenv": "^17.0.0",
+    "dotenv": "^17.0.1",
     "fluent-ffmpeg": "^2.1.3",
     "google-auth-library": "^9.15.1",
-    "graphai": "^2.0.9",
-    "inquirer": "^12.6.3",
+    "graphai": "^2.0.11",
+    "inquirer": "^12.7.0",
     "marked": "^16.0.0",
     "ora": "^8.2.0",
-    "puppeteer": "^24.11.1",
+    "puppeteer": "^24.11.2",
     "replicate": "^1.0.1",
     "yaml": "^2.8.0",
     "yargs": "^18.0.0",
-    "zod": "^3.25.67",
+    "zod": "^3.25.74",
     "zod-to-json-schema": "^3.24.6"
   },
   "devDependencies": {
     "@anatine/zod-mock": "^3.14.0",
-    "@faker-js/faker": "^9.8.0",
+    "@faker-js/faker": "^9.9.0",
     "@receptron/test_utils": "^2.0.0",
     "@types/fluent-ffmpeg": "^2.1.26",
     "@types/yargs": "^17.0.33",
-    "eslint": "^9.30.0",
+    "eslint": "^9.30.1",
     "eslint-config-prettier": "^10.1.5",
     "eslint-plugin-prettier": "^5.5.1",
     "prettier": "^3.6.2",
     "ts-node": "^10.9.2",
     "tsx": "^4.20.3",
     "typescript": "^5.7.3",
-    "typescript-eslint": "^8.35.0"
+    "typescript-eslint": "^8.35.1"
   },
   "engines": {
     "node": ">=18.0.0"

package/scripts/templates/voice_over.json

@@ -0,0 +1,60 @@
+{
+  "$mulmocast": {
+    "version": "1.0"
+  },
+  "title": "Voice Over Test",
+  "captionParams": {
+    "lang": "en"
+  },
+  "canvasSize": {
+    "width": 1552,
+    "height": 2064
+  },
+  "beats": [
+    {
+      "text": "Description of this section of the movie",
+      "image": {
+        "type": "movie",
+        "source": {
+          "kind": "url",
+          "url": "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/movies/actions.mp4"
+        }
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 8 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 8.0
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 14.5 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 14.5
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 21 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 21.0
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 25 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 25.0
+      }
+    },
+    {
+      "text": "Description of this section of the movie starting at 30 seconds",
+      "image": {
+        "type": "voice_over",
+        "startAt": 30.0
+      }
+    }
+  ]
+}