mulmocast 0.0.28 → 0.1.0

package/lib/types/schema.js

@@ -117,6 +117,12 @@ export const mulmoBeatReferenceMediaSchema = z
     id: z.string().optional().describe("Specifies the beat to reference."),
 })
     .strict();
+export const mulmoVoiceOverMediaSchema = z
+    .object({
+    type: z.literal("voice_over"),
+    startAt: z.number().optional().describe("The time to start the voice over the video in seconds."),
+})
+    .strict();
 export const mulmoImageAssetSchema = z.union([
     mulmoMarkdownMediaSchema,
     mulmoWebMediaSchema,
@@ -131,6 +137,7 @@ export const mulmoImageAssetSchema = z.union([
     mulmoMermaidMediaSchema,
     mulmoHtmlTailwindMediaSchema,
     mulmoBeatReferenceMediaSchema,
+    mulmoVoiceOverMediaSchema,
 ]);
 const mulmoAudioMediaSchema = z
     .object({
@@ -152,8 +159,24 @@ export const mulmoFillOptionSchema = z
     style: z.enum(["aspectFit", "aspectFill"]).default("aspectFit"),
 })
     .describe("How to handle aspect ratio differences between image and canvas");
+export const text2ImageProviderSchema = z.union([z.literal("openai"), z.literal("google")]).default("openai");
+// NOTE: This is for UI only. (until we figure out how to use it in mulmoImageParamsSchema)
+export const mulmoOpenAIImageModelSchema = z
+    .object({
+    provider: z.literal("openai"),
+    model: z.union([z.literal("dall-e-3"), z.literal("gpt-image-1")]).optional(),
+})
+    .strict();
+// NOTE: This is for UI only. (until we figure out how to use it in mulmoImageParamsSchema)
+export const mulmoGoogleImageModelSchema = z
+    .object({
+    provider: z.literal("google"),
+    model: z.union([z.literal("imagen-3.0-fast-generate-001"), z.literal("imagen-3.0-generate-002"), z.literal("imagen-3.0-capability-001")]).optional(),
+})
+    .strict();
 export const mulmoImageParamsSchema = z
     .object({
+    provider: text2ImageProviderSchema, // has default value
     model: z.string().optional(), // default: provider specific
     style: z.string().optional(), // optional image style
     moderation: z.string().optional(), // optional image style
@@ -209,7 +232,8 @@ export const mulmoBeatSchema = z
     audioParams: beatAudioParamsSchema.optional(), // beat specific parameters
     movieParams: z
         .object({
-        fillOption: mulmoFillOptionSchema,
+        fillOption: mulmoFillOptionSchema.optional(),
+        speed: z.number().optional().describe("Speed of the video. 1.0 is normal speed. 0.5 is half speed. 2.0 is double speed."),
     })
         .optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema.optional(),
@@ -241,7 +265,6 @@ export const mulmoSpeechParamsSchema = z
     speakers: speakerDictionarySchema,
 })
     .strict();
-export const text2ImageProviderSchema = z.union([z.literal("openai"), z.literal("google")]).default("openai");
 export const text2HtmlImageProviderSchema = z.union([z.literal("openai"), z.literal("anthropic")]).default("openai");
 export const text2MovieProviderSchema = z.union([z.literal("openai"), z.literal("google"), z.literal("replicate")]).default("google");
 export const mulmoTransitionSchema = z.object({
@@ -269,11 +292,7 @@ export const mulmoPresentationStyleSchema = z.object({
             },
         },
     }),
-    imageParams: mulmoImageParamsSchema
-        .extend({
-        provider: text2ImageProviderSchema, // has default value
-    })
-        .optional(),
+    imageParams: mulmoImageParamsSchema.optional(),
     movieParams: mulmoMovieParamsSchema.optional(),
     htmlImageParams: mulmoHtmlImageParamsSchema
         .extend({
@@ -315,6 +334,10 @@ export const mulmoStudioBeatSchema = z
     .object({
     hash: z.string().optional(),
     duration: z.number().optional(),
+    startAt: z.number().optional(),
+    audioDuration: z.number().optional(),
+    movieDuration: z.number().optional(),
+    silenceDuration: z.number().optional(),
     audioFile: z.string().optional(),
     imageFile: z.string().optional(), // path to the image
     movieFile: z.string().optional(), // path to the movie file

package/lib/types/type.d.ts

@@ -1,4 +1,4 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import { LLM } from "../utils/utils.js";
 import { z } from "zod";
@@ -31,6 +31,8 @@ export type MulmoStudioMultiLingual = z.infer<typeof mulmoStudioMultiLingualSche
 export type MulmoStudioMultiLingualData = z.infer<typeof mulmoStudioMultiLingualDataSchema>;
 export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
 export type MulmoMovieParams = z.infer<typeof mulmoMovieParamsSchema>;
+export type MulmoOpenAIImageModel = z.infer<typeof mulmoOpenAIImageModelSchema>;
+export type MulmoGoogleImageModel = z.infer<typeof mulmoGoogleImageModelSchema>;
 export type MulmoTextSlideMedia = z.infer<typeof mulmoTextSlideMediaSchema>;
 export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
@@ -74,7 +76,6 @@ export type ImageProcessorParams = {
 export type PDFMode = (typeof pdf_modes)[number];
 export type PDFSize = (typeof pdf_sizes)[number];
 export type Text2ImageAgentInfo = {
-    provider: Text2ImageProvider;
     agent: string;
     imageParams: MulmoImageParams;
 };

package/lib/utils/context.d.ts

@@ -7,7 +7,11 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
     studio: {
         beats: {
             duration?: number | undefined;
+            startAt?: number | undefined;
             hash?: string | undefined;
+            audioDuration?: number | undefined;
+            movieDuration?: number | undefined;
+            silenceDuration?: number | undefined;
             audioFile?: string | undefined;
             imageFile?: string | undefined;
             movieFile?: string | undefined;
@@ -149,6 +153,9 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | {
                     type: "beat";
                     id?: string | undefined;
+                } | {
+                    type: "voice_over";
+                    startAt?: number | undefined;
                 } | {
                     type: "movie";
                     source: {
@@ -188,6 +195,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 } | undefined;
                 description?: string | undefined;
                 imageParams?: {
+                    provider: "openai" | "google";
                     style?: string | undefined;
                     model?: string | undefined;
                     moderation?: string | undefined;
@@ -212,9 +220,10 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     padding?: number | undefined;
                 } | undefined;
                 movieParams?: {
-                    fillOption: {
+                    speed?: number | undefined;
+                    fillOption?: {
                         style: "aspectFit" | "aspectFill";
-                    };
+                    } | undefined;
                 } | undefined;
                 htmlImageParams?: {
                     model?: string | undefined;

package/lib/utils/image_plugins/index.d.ts

@@ -6,4 +6,5 @@ import * as pluginHtmlTailwind from "./html_tailwind.js";
 import * as pluginImage from "./image.js";
 import * as pluginMovie from "./movie.js";
 import * as pluginBeat from "./beat.js";
-export declare const findImagePlugin: (imageType?: string) => typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginChart | typeof pluginMermaid | typeof pluginHtmlTailwind | typeof pluginImage | typeof pluginMovie | typeof pluginBeat | undefined;
+import * as pluginVoiceOver from "./voice_over.js";
+export declare const findImagePlugin: (imageType?: string) => typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginChart | typeof pluginMermaid | typeof pluginHtmlTailwind | typeof pluginImage | typeof pluginMovie | typeof pluginBeat | typeof pluginVoiceOver | undefined;

package/lib/utils/image_plugins/index.js

@@ -6,7 +6,8 @@ import * as pluginHtmlTailwind from "./html_tailwind.js";
 import * as pluginImage from "./image.js";
 import * as pluginMovie from "./movie.js";
 import * as pluginBeat from "./beat.js";
-const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie, pluginHtmlTailwind, pluginBeat];
+import * as pluginVoiceOver from "./voice_over.js";
+const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie, pluginHtmlTailwind, pluginBeat, pluginVoiceOver];
 export const findImagePlugin = (imageType) => {
     return imagePlugins.find((plugin) => plugin.imageType === imageType);
 };

package/lib/utils/image_plugins/voice_over.d.ts

@@ -0,0 +1,5 @@
+import { ImageProcessorParams } from "../../types/index.js";
+export declare const imageType = "voice_over";
+export declare const processBeatReference: (__: ImageProcessorParams) => Promise<undefined>;
+export declare const process: (__: ImageProcessorParams) => Promise<undefined>;
+export declare const path: (__: ImageProcessorParams) => undefined;

package/lib/utils/image_plugins/voice_over.js

@@ -0,0 +1,9 @@
+export const imageType = "voice_over";
+export const processBeatReference = async (__) => {
+    // For voice-over, return undefined to indicate no image should be generated
+    return undefined;
+};
+export const process = processBeatReference;
+export const path = (__) => {
+    return undefined;
+};

package/lib/utils/preprocess.d.ts

@@ -2,7 +2,11 @@ import { MulmoStudio, MulmoScript } from "../types/index.js";
 export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, currentStudio: MulmoStudio | undefined, fileName: string, videoCaption?: string) => {
     beats: {
         duration?: number | undefined;
+        startAt?: number | undefined;
         hash?: string | undefined;
+        audioDuration?: number | undefined;
+        movieDuration?: number | undefined;
+        silenceDuration?: number | undefined;
         audioFile?: string | undefined;
         imageFile?: string | undefined;
         movieFile?: string | undefined;
@@ -144,6 +148,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | {
                 type: "beat";
                 id?: string | undefined;
+            } | {
+                type: "voice_over";
+                startAt?: number | undefined;
             } | {
                 type: "movie";
                 source: {
@@ -183,6 +190,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | undefined;
             description?: string | undefined;
             imageParams?: {
+                provider: "openai" | "google";
                 style?: string | undefined;
                 model?: string | undefined;
                 moderation?: string | undefined;
@@ -207,9 +215,10 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 padding?: number | undefined;
             } | undefined;
             movieParams?: {
-                fillOption: {
+                speed?: number | undefined;
+                fillOption?: {
                     style: "aspectFit" | "aspectFill";
-                };
+                } | undefined;
             } | undefined;
             htmlImageParams?: {
                 model?: string | undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.0.28",
+  "version": "0.1.0",
   "description": "",
   "type": "module",
   "main": "lib/index.js",