mulmocast 0.0.11 → 0.0.12

package/lib/types/schema.js

@@ -20,11 +20,13 @@ export const speechOptionsSchema = z
 })
     .strict();
 const speakerIdSchema = z.string();
+export const text2SpeechProviderSchema = z.union([z.literal("openai"), z.literal("nijivoice"), z.literal("google"), z.literal("elevenlabs")]).default("openai");
 const speakerDataSchema = z
     .object({
     displayName: z.record(langSchema, z.string()).optional(),
     voiceId: z.string(),
     speechOptions: speechOptionsSchema.optional(),
+    provider: text2SpeechProviderSchema.optional(),
 })
     .strict();
 export const speakerDictionarySchema = z.record(speakerIdSchema, speakerDataSchema);
@@ -103,6 +105,12 @@ export const mulmoHtmlTailwindMediaSchema = z
     html: stringOrStringArray,
 })
     .strict();
+export const mulmoBeatReferenceMediaSchema = z
+    .object({
+    type: z.literal("beat"),
+    id: z.string().optional().describe("Specifies the beat to reference."),
+})
+    .strict();
 export const mulmoImageAssetSchema = z.union([
     mulmoMarkdownMediaSchema,
     mulmoWebMediaSchema,
@@ -116,6 +124,7 @@ export const mulmoImageAssetSchema = z.union([
     mulmoChartMediaSchema,
     mulmoMermaidMediaSchema,
     mulmoHtmlTailwindMediaSchema,
+    mulmoBeatReferenceMediaSchema,
 ]);
 const mulmoAudioMediaSchema = z
     .object({
@@ -144,13 +153,6 @@ export const textSlideParamsSchema = z
     cssStyles: stringOrStringArray,
 })
     .strict();
-/* TODO: Add something later
-export const videoParamsSchema = z
-  .object({
-    padding: z.number().optional(), // msec
-  })
-  .strict();
-*/
 export const beatAudioParamsSchema = z
     .object({
     padding: z.number().optional().describe("Padding between beats"), // seconds
@@ -163,12 +165,14 @@ export const audioParamsSchema = z
     introPadding: z.number().describe("Padding at the beginning of the audio"), // seconds
     closingPadding: z.number().describe("Padding before the last beat"), // seconds
     outroPadding: z.number().describe("Padding at the end of the audio"), // seconds
+    bgm: mediaSourceSchema.optional(),
 })
     .strict();
 export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.default("Presenter"),
-    text: z.string().describe("Text to be spoken. If empty, the audio is not generated."),
+    text: z.string().default("").describe("Text to be spoken. If empty, the audio is not generated."),
+    id: z.string().optional().describe("Unique identifier for the beat."),
     description: z.string().optional(),
     image: mulmoImageAssetSchema.optional(),
     audio: mulmoAudioAssetSchema.optional(),
@@ -195,7 +199,6 @@ export const mulmoCastCreditSchema = z
     credit: z.literal("closing").optional(),
 })
     .strict();
-export const text2SpeechProviderSchema = z.union([z.literal("openai"), z.literal("nijivoice"), z.literal("google")]).default("openai");
 export const mulmoSpeechParamsSchema = z
     .object({
     provider: text2SpeechProviderSchema, // has default value
@@ -204,10 +207,15 @@ export const mulmoSpeechParamsSchema = z
     .strict();
 export const text2ImageProviderSchema = z.union([z.literal("openai"), z.literal("google")]).default("openai");
 export const text2MovieProviderSchema = z.union([z.literal("openai"), z.literal("google")]).default("google");
+export const mulmoTransitionSchema = z.object({
+    type: z.enum(["fade"]),
+    duration: z.number().min(0).max(2).default(0.3), // transition duration in seconds
+});
 export const mulmoMovieParamsSchema = z
     .object({
     provider: text2MovieProviderSchema.optional(),
     model: z.string().optional(), // default: provider specific
+    transition: mulmoTransitionSchema.optional(),
 })
     .strict();
 export const mulmoPresentationStyleSchema = z.object({
@@ -231,15 +239,12 @@ export const mulmoPresentationStyleSchema = z.object({
     movieParams: mulmoMovieParamsSchema.optional(),
     // for textSlides
     textSlideParams: textSlideParamsSchema.optional(),
-    // videoParams: videoParamsSchema.optional(),
     audioParams: audioParamsSchema.default({
         introPadding: 1.0,
         padding: 0.3,
         closingPadding: 0.8,
         outroPadding: 1.0,
     }),
-    // TODO: Switch to showCaptions later
-    omitCaptions: z.boolean().optional(), // default is false
 });
 export const mulmoReferenceSchema = z.object({
     url: URLStringSchema,
@@ -284,11 +289,11 @@ export const mulmoSessionStateSchema = z.object({
         pdf: z.boolean(),
     }),
     inBeatSession: z.object({
-        audio: z.set(z.number()),
-        image: z.set(z.number()),
-        movie: z.set(z.number()),
-        multiLingual: z.set(z.number()),
-        caption: z.set(z.number()),
+        audio: z.record(z.number().int(), z.boolean()),
+        image: z.record(z.number().int(), z.boolean()),
+        movie: z.record(z.number().int(), z.boolean()),
+        multiLingual: z.record(z.number().int(), z.boolean()),
+        caption: z.record(z.number().int(), z.boolean()),
     }),
 });
 export const mulmoStudioSchema = z
@@ -297,23 +302,6 @@ export const mulmoStudioSchema = z
     filename: z.string(),
     beats: z.array(mulmoStudioBeatSchema).min(1),
     multiLingual: mulmoStudioMultiLingualSchema,
-    state: mulmoSessionStateSchema.default({
-        inSession: {
-            audio: false,
-            image: false,
-            video: false,
-            multiLingual: false,
-            caption: false,
-            pdf: false,
-        },
-        inBeatSession: {
-            audio: new Set(),
-            image: new Set(),
-            movie: new Set(),
-            multiLingual: new Set(),
-            caption: new Set(),
-        },
-    }),
 })
     .strict();
 export const mulmoScriptTemplateSchema = z
@@ -325,6 +313,9 @@ export const mulmoScriptTemplateSchema = z
     presentationStyle: mulmoPresentationStyleSchema.optional(),
 })
     .strict();
+export const mulmoScriptTemplateFileSchema = mulmoScriptTemplateSchema.extend({
+    filename: z.string(),
+});
 export const mulmoStoryboardSceneSchema = z
     .object({
     description: z.string(),

package/lib/types/type.d.ts

@@ -1,4 +1,4 @@
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, text2ImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualDataSchema, speakerDictionarySchema, mulmoImageParamsSchema, mulmoMovieParamsSchema, mulmoSpeechParamsSchema, textSlideParamsSchema, speechOptionsSchema, mulmoCanvasDimensionSchema, mulmoScriptTemplateSchema, mulmoScriptTemplateFileSchema, text2ImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoMermaidMediaSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mulmoSessionStateSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "../utils/const.js";
 import { LLM } from "../utils/utils.js";
 import { z } from "zod";
@@ -22,6 +22,7 @@ export type MulmoStudioBeat = z.infer<typeof mulmoStudioBeatSchema>;
 export type MulmoMediaSource = z.infer<typeof mediaSourceSchema>;
 export type MulmoStudio = z.infer<typeof mulmoStudioSchema>;
 export type MulmoScriptTemplate = z.infer<typeof mulmoScriptTemplateSchema>;
+export type MulmoScriptTemplateFile = z.infer<typeof mulmoScriptTemplateFileSchema>;
 export type MulmoStudioMultiLingual = z.infer<typeof mulmoStudioMultiLingualSchema>;
 export type MulmoStudioMultiLingualData = z.infer<typeof mulmoStudioMultiLingualDataSchema>;
 export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
@@ -31,6 +32,7 @@ export type MulmoMarkdownMedia = z.infer<typeof mulmoMarkdownMediaSchema>;
 export type MulmoImageMedia = z.infer<typeof mulmoImageMediaSchema>;
 export type MulmoChartMedia = z.infer<typeof mulmoChartMediaSchema>;
 export type MulmoMermaidMedia = z.infer<typeof mulmoMermaidMediaSchema>;
+export type MulmoSessionState = z.infer<typeof mulmoSessionStateSchema>;
 export type FileDirs = {
     mulmoFilePath: string;
     mulmoFileDirPath: string;
@@ -45,6 +47,7 @@ export type MulmoStudioContext = {
     lang?: string;
     force: boolean;
     caption?: string;
+    sessionState: MulmoSessionState;
 };
 export type ScriptingParams = {
     urls: string[];

package/lib/utils/file.d.ts

@@ -1,4 +1,4 @@
-import { MulmoScript, MulmoScriptTemplate, MulmoMediaSource, MulmoStudioContext } from "../types/index.js";
+import { MulmoScript, MulmoScriptTemplateFile } from "../types/index.js";
 import { PDFMode } from "../types/index.js";
 import { ZodSchema } from "zod";
 export declare const updateNpmRoot: (_npmRoot: string) => void;
@@ -14,16 +14,8 @@ export declare function readMulmoScriptFile<T = MulmoScript>(path: string): {
 } | null;
 export declare const fetchMulmoScriptFile: (url: string) => Promise<{
     result: boolean;
-    status: number;
-    script?: undefined;
-} | {
-    result: boolean;
-    script: any;
-    status?: undefined;
-} | {
-    result: boolean;
-    status: string;
-    script?: undefined;
+    script?: MulmoScript;
+    status: string | number;
 }>;
 export declare const getOutputStudioFilePath: (outDirPath: string, fileName: string) => string;
 export declare const resolveDirPath: (dirPath: string, studioFileName: string) => string;
@@ -44,9 +36,6 @@ export declare const getBaseDirPath: (basedir?: string) => string;
 export declare const getFullPath: (baseDirPath: string | undefined, file: string) => string;
 export declare const readScriptTemplateFile: (scriptName: string) => any;
 export declare const readTemplatePrompt: (templateName: string) => string;
-export declare const getAvailableTemplates: () => (MulmoScriptTemplate & {
-    filename: string;
-})[];
+export declare const getAvailableTemplates: () => MulmoScriptTemplateFile[];
 export declare const writingMessage: (filePath: string) => void;
-export declare const resolveMediaSource: (source: MulmoMediaSource, context: MulmoStudioContext) => string | null;
 export declare const readAndParseJson: <S extends ZodSchema<any>>(filePath: string, schema: S) => ReturnType<S["parse"]>;

package/lib/utils/file.js

@@ -4,7 +4,6 @@ import { parse as yamlParse } from "yaml";
 import { fileURLToPath } from "url";
 import { GraphAILogger } from "graphai";
 import { MulmoScriptTemplateMethods } from "../methods/mulmo_script_template.js";
-import { MulmoStudioContextMethods } from "../methods/index.js";
 import { mulmoScriptTemplateSchema } from "../types/schema.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -17,7 +16,6 @@ export function readMulmoScriptFile(arg2, errorMessage) {
     if (!fs.existsSync(scriptPath)) {
         if (errorMessage) {
             GraphAILogger.info(errorMessage);
-            process.exit(1);
         }
         return null;
     }
@@ -34,7 +32,6 @@ export function readMulmoScriptFile(arg2, errorMessage) {
     catch (__error) {
         if (errorMessage) {
             GraphAILogger.info("read file format is broken.");
-            process.exit(1);
         }
         return null;
     }
@@ -45,10 +42,11 @@ export const fetchMulmoScriptFile = async (url) => {
         if (!res.ok) {
             return { result: false, status: res.status };
         }
-        const script = await res.json();
+        const script = (await res.json());
         return {
             result: true,
             script,
+            status: 200,
         };
     }
     catch {
@@ -162,15 +160,6 @@ export const getAvailableTemplates = () => {
 export const writingMessage = (filePath) => {
     GraphAILogger.debug(`writing: ${filePath}`);
 };
-export const resolveMediaSource = (source, context) => {
-    if (source.kind === "path") {
-        return MulmoStudioContextMethods.resolveAssetPath(context, source.path);
-    }
-    if (source.kind === "url") {
-        return source.url;
-    }
-    return null;
-};
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export const readAndParseJson = (filePath, schema) => {
     const fileContent = fs.readFileSync(filePath, "utf-8");

package/lib/utils/filters.js

@@ -5,10 +5,10 @@ import fsPromise from "fs/promises";
 import { GraphAILogger } from "graphai";
 import { writingMessage } from "./file.js";
 import { text2hash } from "./utils.js";
-import { MulmoStudioMethods } from "../methods/mulmo_studio.js";
+import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
 export const fileCacheAgentFilter = async (context, next) => {
     const { namedInputs } = context;
-    const { file, force, studio, index, sessionType } = namedInputs;
+    const { file, force, mulmoContext, index, sessionType } = namedInputs;
     const shouldUseCache = async () => {
         if (force) {
             return false;
@@ -25,7 +25,7 @@ export const fileCacheAgentFilter = async (context, next) => {
         return true;
     }
     try {
-        MulmoStudioMethods.setBeatSessionState(studio, sessionType, index, true);
+        MulmoStudioContextMethods.setBeatSessionState(mulmoContext, sessionType, index, true);
         const output = (await next(context));
         const buffer = output ? output["buffer"] : undefined;
         if (buffer) {
@@ -37,7 +37,7 @@ export const fileCacheAgentFilter = async (context, next) => {
         return false;
     }
     finally {
-        MulmoStudioMethods.setBeatSessionState(studio, sessionType, index, false);
+        MulmoStudioContextMethods.setBeatSessionState(mulmoContext, sessionType, index, false);
     }
 };
 export const browserlessCacheGenerator = (cacheDir) => {

package/lib/utils/image_plugins/beat.d.ts

@@ -0,0 +1,4 @@
+import { ImageProcessorParams } from "../../types/index.js";
+export declare const imageType = "beat";
+export declare const processBeatReference: (__: ImageProcessorParams) => Promise<undefined>;
+export declare const process: (__: ImageProcessorParams) => Promise<undefined>;

package/lib/utils/image_plugins/beat.js

@@ -0,0 +1,7 @@
+export const imageType = "beat";
+export const processBeatReference = async (__) => {
+    // For beat reference, return undefined to indicate no image should be generated
+    // The actual reference will be resolved in mergeResult
+    return undefined;
+};
+export const process = processBeatReference;

package/lib/utils/image_plugins/index.d.ts

@@ -5,4 +5,5 @@ import * as pluginChart from "./chart.js";
 import * as pluginMermaid from "./mermaid.js";
 import * as pluginMovie from "./movie.js";
 import * as pluginHtmlTailwind from "./html_tailwind.js";
-export declare const imagePlugins: (typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginImage | typeof pluginChart | typeof pluginMermaid | typeof pluginMovie | typeof pluginHtmlTailwind)[];
+import * as pluginBeat from "./beat.js";
+export declare const imagePlugins: (typeof pluginTextSlide | typeof pluginMarkdown | typeof pluginImage | typeof pluginChart | typeof pluginMermaid | typeof pluginMovie | typeof pluginHtmlTailwind | typeof pluginBeat)[];

package/lib/utils/image_plugins/index.js

@@ -5,4 +5,5 @@ import * as pluginChart from "./chart.js";
 import * as pluginMermaid from "./mermaid.js";
 import * as pluginMovie from "./movie.js";
 import * as pluginHtmlTailwind from "./html_tailwind.js";
-export const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie, pluginHtmlTailwind];
+import * as pluginBeat from "./beat.js";
+export const imagePlugins = [pluginTextSlide, pluginMarkdown, pluginImage, pluginChart, pluginMermaid, pluginMovie, pluginHtmlTailwind, pluginBeat];

package/lib/utils/image_plugins/source.js

@@ -1,11 +1,11 @@
 import { GraphAILogger } from "graphai";
-import { resolveMediaSource } from "../../utils/file.js";
+import { MulmoMediaSourceMethods } from "../../methods/mulmo_media_source.js";
 export const processSource = (imageType) => {
     return (params) => {
         const { beat, context } = params;
         if (!beat.image || beat.image.type !== imageType)
             return;
-        const path = resolveMediaSource(beat.image.source, context);
+        const path = MulmoMediaSourceMethods.resolve(beat.image.source, context);
         if (path) {
             return path;
         }

package/lib/utils/preprocess.d.ts

@@ -23,6 +23,19 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             introPadding: number;
             closingPadding: number;
             outroPadding: number;
+            bgm?: {
+                url: string;
+                kind: "url";
+            } | {
+                kind: "base64";
+                data: string;
+            } | {
+                text: string;
+                kind: "text";
+            } | {
+                path: string;
+                kind: "path";
+            } | undefined;
         };
         $mulmocast: {
             version: "1.0";
@@ -33,7 +46,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             height: number;
         };
         speechParams: {
-            provider: "openai" | "nijivoice" | "google";
+            provider: "openai" | "nijivoice" | "google" | "elevenlabs";
             speakers: Record<string, {
                 voiceId: string;
                 displayName?: Record<string, string> | undefined;
@@ -41,6 +54,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                     speed?: number | undefined;
                     instruction?: string | undefined;
                 } | undefined;
+                provider?: "openai" | "nijivoice" | "google" | "elevenlabs" | undefined;
             }>;
         };
         beats: {
@@ -133,6 +147,9 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             } | {
                 type: "html_tailwind";
                 html: string | string[];
+            } | {
+                type: "beat";
+                id?: string | undefined;
             } | {
                 type: "movie";
                 source: {
@@ -150,6 +167,7 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
                 };
                 mixAudio: number;
             } | undefined;
+            id?: string | undefined;
             audio?: {
                 type: "audio";
                 source: {
@@ -230,10 +248,13 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
             cssStyles: string | string[];
         } | undefined;
         movieParams?: {
-            model?: string | undefined;
             provider?: "openai" | "google" | undefined;
+            model?: string | undefined;
+            transition?: {
+                type: "fade";
+                duration: number;
+            } | undefined;
         } | undefined;
-        omitCaptions?: boolean | undefined;
         references?: {
             type: "image" | "audio" | "article" | "paper" | "video";
             url: string;
@@ -244,21 +265,4 @@ export declare const createOrUpdateStudioData: (_mulmoScript: MulmoScript, curre
         __test_invalid__?: boolean | undefined;
     };
     filename: string;
-    state: {
-        inSession: {
-            pdf: boolean;
-            image: boolean;
-            audio: boolean;
-            video: boolean;
-            multiLingual: boolean;
-            caption: boolean;
-        };
-        inBeatSession: {
-            image: Set<number>;
-            movie: Set<number>;
-            audio: Set<number>;
-            multiLingual: Set<number>;
-            caption: Set<number>;
-        };
-    };
 };

package/lib/utils/preprocess.js

@@ -1,9 +1,13 @@
+import { GraphAILogger } from "graphai";
 import { mulmoScriptSchema, mulmoBeatSchema, mulmoStudioSchema } from "../types/index.js";
 const rebuildStudio = (currentStudio, mulmoScript, fileName) => {
     const parsed = mulmoStudioSchema.safeParse(currentStudio);
     if (parsed.success) {
         return parsed.data;
     }
+    if (currentStudio) {
+        GraphAILogger.info("currentStudio is invalid", parsed.error);
+    }
     // We need to parse it to fill default values
     return mulmoStudioSchema.parse({
         script: mulmoScript,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "description": "",
   "type": "module",
   "main": "lib/index.js",

package/scripts/templates/movie_prompts_no_text_template.json

@@ -0,0 +1,50 @@
+{
+  "$mulmocast": {
+    "version": "1.0",
+    "credit": "closing"
+  },
+  "title": "[TITLE: Brief, engaging title for the topic]",
+  "lang": "en",
+  "references": [
+    {
+      "url": "[SOURCE_URL: URL of the source material]",
+      "title": "[SOURCE_TITLE: Title of the referenced article, or paper]",
+      "type": "[SOURCE_TYPE: article, paper]"
+    }
+  ],
+  "movieParams": {
+    "provider": "google"
+  },
+  "beats": [
+    {
+      "duration": 5.0,
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]",
+      "moviePrompt": "[MOVIE_PROMPT: A movie prompt for that image.]"
+    },
+    {
+      "duration": 5.0,
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]",
+      "moviePrompt": "[MOVIE_PROMPT: A movie prompt for that image.]"
+    },
+    {
+      "duration": 5.0,
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]",
+      "moviePrompt": "[MOVIE_PROMPT: A movie prompt for that image.]"
+    },
+    {
+      "duration": 5.0,
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]",
+      "moviePrompt": "[MOVIE_PROMPT: A movie prompt for that image.]"
+    },
+    {
+      "duration": 5.0,
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]",
+      "moviePrompt": "[MOVIE_PROMPT: A movie prompt for that image.]"
+    },
+    {
+      "duration": 5.0,
+      "imagePrompt": "[IMAGE_PROMPT: A prompt for the image to be generated for this beat.]",
+      "moviePrompt": "[MOVIE_PROMPT: A movie prompt for that image.]"
+    }
+  ]
+}