mulmocast 0.0.11 → 0.0.12

package/lib/agents/image_openai_agent.js

@@ -1,4 +1,5 @@
 import fs from "fs";
+import path from "path";
 import OpenAI, { toFile } from "openai";
 // https://platform.openai.com/docs/guides/image-generation
 export const imageOpenaiAgent = async ({ namedInputs, params }) => {
@@ -42,9 +43,11 @@ export const imageOpenaiAgent = async ({ namedInputs, params }) => {
     const response = await (async () => {
         const targetSize = imageOptions.size;
         if ((images ?? []).length > 0 && (targetSize === "1536x1024" || targetSize === "1024x1536" || targetSize === "1024x1024")) {
-            const imagelist = await Promise.all((images ?? []).map(async (file) => await toFile(fs.createReadStream(file), null, {
-                type: "image/png", // TODO: Support JPEG as well
-            })));
+            const imagelist = await Promise.all((images ?? []).map(async (file) => {
+                const ext = path.extname(file).toLowerCase();
+                const type = ext === ".jpg" || ext === ".jpeg" ? "image/jpeg" : "image/png";
+                return await toFile(fs.createReadStream(file), null, { type });
+            }));
             return await openai.images.edit({ ...imageOptions, size: targetSize, image: imagelist });
         }
         else {

package/lib/agents/index.d.ts

@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
 import combineAudioFilesAgent from "./combine_audio_files_agent.js";
 import imageGoogleAgent from "./image_google_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
+import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
 import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
@@ -9,4 +10,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };

package/lib/agents/index.js

@@ -2,6 +2,7 @@ import addBGMAgent from "./add_bgm_agent.js";
 import combineAudioFilesAgent from "./combine_audio_files_agent.js";
 import imageGoogleAgent from "./image_google_agent.js";
 import imageOpenaiAgent from "./image_openai_agent.js";
+import ttsElevenlabsAgent from "./tts_elevenlabs_agent.js";
 import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
@@ -10,4 +11,4 @@ import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGoogleAgent, imageOpenaiAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, validateSchemaAgent, };

package/lib/agents/tts_elevenlabs_agent.d.ts

@@ -0,0 +1,4 @@
+import type { AgentFunction, AgentFunctionInfo } from "graphai";
+export declare const ttsElevenlabsAgent: AgentFunction;
+declare const ttsElevenlabsAgentInfo: AgentFunctionInfo;
+export default ttsElevenlabsAgentInfo;

package/lib/agents/tts_elevenlabs_agent.js

@@ -0,0 +1,60 @@
+import { GraphAILogger } from "graphai";
+export const ttsElevenlabsAgent = async ({ namedInputs, params }) => {
+    const { text } = namedInputs;
+    const { voice, model, stability, similarityBoost, suppressError } = params;
+    const apiKey = process.env.ELEVENLABS_API_KEY;
+    if (!apiKey) {
+        throw new Error("ELEVENLABS_API_KEY environment variable is required");
+    }
+    if (!voice) {
+        throw new Error("Voice ID is required");
+    }
+    try {
+        const requestBody = {
+            text,
+            model_id: model ?? "eleven_monolingual_v1",
+            voice_settings: {
+                stability: stability ?? 0.5,
+                similarity_boost: similarityBoost ?? 0.75,
+            },
+        };
+        GraphAILogger.log("ElevenLabs TTS options", requestBody);
+        const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voice}`, {
+            method: "POST",
+            headers: {
+                Accept: "audio/mpeg",
+                "Content-Type": "application/json",
+                "xi-api-key": apiKey,
+            },
+            body: JSON.stringify(requestBody),
+        });
+        if (!response.ok) {
+            throw new Error(`Eleven Labs API error: ${response.status} ${response.statusText}`);
+        }
+        const arrayBuffer = await response.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+        return { buffer };
+    }
+    catch (e) {
+        if (suppressError) {
+            return {
+                error: e,
+            };
+        }
+        GraphAILogger.info(e);
+        throw new Error("TTS Eleven Labs Error");
+    }
+};
+const ttsElevenlabsAgentInfo = {
+    name: "ttsElevenlabsAgent",
+    agent: ttsElevenlabsAgent,
+    mock: ttsElevenlabsAgent,
+    samples: [],
+    description: "Eleven Labs TTS agent",
+    category: ["tts"],
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
+    license: "MIT",
+    environmentVariables: ["ELEVENLABS_API_KEY"],
+};
+export default ttsElevenlabsAgentInfo;

package/lib/agents/tts_google_agent.js

@@ -44,7 +44,7 @@ const ttsGoogleAgentInfo = {
     description: "Google TTS agent",
     category: ["tts"],
     author: "Receptron Team",
-    repository: "https://github.com/receptron/graphai-agents/tree/main/tts/tts-openai-agent",
+    repository: "https://github.com/receptron/mulmocast-cli/",
     license: "MIT",
     environmentVariables: ["OPENAI_API_KEY"],
 };

package/lib/agents/tts_nijivoice_agent.js

@@ -57,8 +57,9 @@ const ttsNijivoiceAgentInfo = {
     samples: [],
     description: "TTS nijivoice agent",
     category: ["tts"],
-    author: "isamu arimoto",
-    repository: "https://github.com/receptron/graphai/",
+    author: "Receptron Team",
+    repository: "https://github.com/receptron/mulmocast-cli/",
     license: "MIT",
+    environmentVariables: ["NIJIVOICE_API_KEY"],
 };
 export default ttsNijivoiceAgentInfo;

package/lib/agents/tts_openai_agent.js

@@ -36,7 +36,7 @@ const ttsOpenaiAgentInfo = {
     description: "OpenAI TTS agent",
     category: ["tts"],
     author: "Receptron Team",
-    repository: "https://github.com/receptron/graphai-agents/tree/main/tts/tts-openai-agent",
+    repository: "https://github.com/receptron/mulmocast-cli/",
     license: "MIT",
     environmentVariables: ["OPENAI_API_KEY"],
 };

package/lib/cli/commands/audio/handler.js

@@ -1,7 +1,10 @@
 import { audio } from "../../../actions/index.js";
-import { initializeContext, runTranslateIfNeeded } from "../../../cli/helpers.js";
+import { initializeContext, runTranslateIfNeeded } from "../../helpers.js";
 export const handler = async (argv) => {
     const context = await initializeContext(argv);
+    if (!context) {
+        process.exit(1);
+    }
     await runTranslateIfNeeded(context, argv);
     await audio(context);
 };

package/lib/cli/commands/image/handler.js

@@ -1,7 +1,10 @@
 import { images } from "../../../actions/index.js";
-import { initializeContext, runTranslateIfNeeded } from "../../../cli/helpers.js";
+import { initializeContext, runTranslateIfNeeded } from "../../helpers.js";
 export const handler = async (argv) => {
     const context = await initializeContext(argv);
+    if (!context) {
+        process.exit(1);
+    }
     await runTranslateIfNeeded(context, argv);
     await images(context);
 };

package/lib/cli/commands/movie/handler.js

@@ -1,7 +1,10 @@
 import { audio, images, movie, captions } from "../../../actions/index.js";
-import { initializeContext, runTranslateIfNeeded } from "../../../cli/helpers.js";
+import { initializeContext, runTranslateIfNeeded } from "../../helpers.js";
 export const handler = async (argv) => {
     const context = await initializeContext(argv);
+    if (!context) {
+        process.exit(1);
+    }
     await runTranslateIfNeeded(context, argv);
     await audio(context);
     await images(context);

package/lib/cli/commands/pdf/handler.js

@@ -1,7 +1,10 @@
 import { images, pdf } from "../../../actions/index.js";
-import { initializeContext, runTranslateIfNeeded } from "../../../cli/helpers.js";
+import { initializeContext, runTranslateIfNeeded } from "../../helpers.js";
 export const handler = async (argv) => {
     const context = await initializeContext(argv);
+    if (!context) {
+        process.exit(1);
+    }
     await runTranslateIfNeeded(context, argv);
     await images(context);
     await pdf(context, argv.pdf_mode, argv.pdf_size);

package/lib/cli/commands/translate/handler.js

@@ -1,6 +1,9 @@
 import { translate } from "../../../actions/index.js";
-import { initializeContext } from "../../../cli/helpers.js";
+import { initializeContext } from "../../helpers.js";
 export const handler = async (argv) => {
     const context = await initializeContext(argv);
+    if (!context) {
+        process.exit(1);
+    }
     await translate(context);
 };

package/lib/cli/helpers.d.ts

@@ -1,4 +1,4 @@
-import type { MulmoStudioContext } from "../types/type.js";
+import type { MulmoScript, MulmoStudioContext } from "../types/type.js";
 import type { CliArgs } from "../types/cli_types.js";
 export declare const setGraphAILogger: (verbose: boolean | undefined, logValues?: Record<string, unknown>) => void;
 export interface FileObject {
@@ -20,7 +20,7 @@ export declare const getFileObject: (args: {
     audiodir?: string;
     file: string;
 }) => FileObject;
-export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<any>;
+export declare const fetchScript: (isHttpPath: boolean, mulmoFilePath: string, fileOrUrl: string) => Promise<MulmoScript | null>;
 type InitOptions = {
     b?: string;
     o?: string;
@@ -30,7 +30,7 @@ type InitOptions = {
     l?: string;
     c?: string;
 };
-export declare const initializeContext: (argv: CliArgs<InitOptions>) => Promise<MulmoStudioContext>;
+export declare const initializeContext: (argv: CliArgs<InitOptions>) => Promise<MulmoStudioContext | null>;
 export declare const runTranslateIfNeeded: (context: MulmoStudioContext, argv: {
     l?: string;
     c?: string;

package/lib/cli/helpers.js

@@ -65,15 +65,15 @@ export const fetchScript = async (isHttpPath, mulmoFilePath, fileOrUrl) => {
         const res = await fetchMulmoScriptFile(fileOrUrl);
         if (!res.result || !res.script) {
             GraphAILogger.info(`ERROR: HTTP error! ${res.status} ${fileOrUrl}`);
-            process.exit(1);
+            return null;
         }
         return res.script;
     }
     if (!fs.existsSync(mulmoFilePath)) {
         GraphAILogger.info(`ERROR: File not exists ${mulmoFilePath}`);
-        process.exit(1);
+        return null;
     }
-    return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath).mulmoData;
+    return readMulmoScriptFile(mulmoFilePath, "ERROR: File does not exist " + mulmoFilePath)?.mulmoData ?? null;
 };
 export const initializeContext = async (argv) => {
     const files = getFileObject({
@@ -88,25 +88,43 @@ export const initializeContext = async (argv) => {
         files,
     });
     const mulmoScript = await fetchScript(isHttpPath, mulmoFilePath, fileOrUrl);
+    if (!mulmoScript) {
+        return null;
+    }
     // Create or update MulmoStudio file with MulmoScript
     const currentStudio = readMulmoScriptFile(outputStudioFilePath);
-    const studio = (() => {
-        try {
-            // validate mulmoStudioSchema. skip if __test_invalid__ is true
-            return createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName);
-        }
-        catch (error) {
-            GraphAILogger.info(`Error: invalid MulmoScript Schema: ${isHttpPath ? fileOrUrl : mulmoFilePath} \n ${error}`);
-            process.exit(1);
-        }
-    })();
-    return {
-        studio,
-        fileDirs: files,
-        force: Boolean(argv.f),
-        lang: argv.l,
-        caption: argv.c,
-    };
+    try {
+        // validate mulmoStudioSchema. skip if __test_invalid__ is true
+        const studio = createOrUpdateStudioData(mulmoScript, currentStudio?.mulmoData, fileName);
+        return {
+            studio,
+            fileDirs: files,
+            force: Boolean(argv.f),
+            lang: argv.l,
+            caption: argv.c,
+            sessionState: {
+                inSession: {
+                    audio: false,
+                    image: false,
+                    video: false,
+                    multiLingual: false,
+                    caption: false,
+                    pdf: false,
+                },
+                inBeatSession: {
+                    audio: {},
+                    image: {},
+                    movie: {},
+                    multiLingual: {},
+                    caption: {},
+                },
+            },
+        };
+    }
+    catch (error) {
+        GraphAILogger.info(`Error: invalid MulmoScript Schema: ${isHttpPath ? fileOrUrl : mulmoFilePath} \n ${error}`);
+        return null;
+    }
 };
 export const runTranslateIfNeeded = async (context, argv) => {
     if (argv.l || argv.c) {

package/lib/methods/mulmo_media_source.d.ts

@@ -1,4 +1,5 @@
 import { MulmoMediaSource, MulmoStudioContext } from "../types/index.js";
 export declare const MulmoMediaSourceMethods: {
     getText(mediaSource: MulmoMediaSource, context: MulmoStudioContext): Promise<string | null>;
+    resolve(mediaSource: MulmoMediaSource | undefined, context: MulmoStudioContext): string | null;
 };

package/lib/methods/mulmo_media_source.js

@@ -1,5 +1,6 @@
 import fs from "fs";
 import { getFullPath } from "../utils/file.js";
+import { MulmoStudioContextMethods } from "../methods/index.js";
 export const MulmoMediaSourceMethods = {
     async getText(mediaSource, context) {
         if (mediaSource.kind === "text") {
@@ -18,4 +19,15 @@ export const MulmoMediaSourceMethods = {
         }
         return null;
     },
+    resolve(mediaSource, context) {
+        if (!mediaSource)
+            return null;
+        if (mediaSource.kind === "path") {
+            return MulmoStudioContextMethods.resolveAssetPath(context, mediaSource.path);
+        }
+        if (mediaSource.kind === "url") {
+            return mediaSource.url;
+        }
+        return null;
+    },
 };

package/lib/methods/mulmo_script.d.ts

@@ -3,6 +3,7 @@ import { MulmoCanvasDimension, MulmoScript, MulmoBeat, SpeechOptions, Text2Speec
 export declare const MulmoScriptMethods: {
     getCanvasSize(script: MulmoScript): MulmoCanvasDimension;
     getSpeechProvider(script: MulmoScript): Text2SpeechProvider;
+    getAllSpeechProviders(script: MulmoScript): Set<Text2SpeechProvider>;
     getTextSlideStyle(script: MulmoScript, beat: MulmoBeat): string;
     getSpeechOptions(script: MulmoScript, beat: MulmoBeat): SpeechOptions | undefined;
     getImageAgentInfo(script: MulmoScript): Text2ImageAgentInfo;

package/lib/methods/mulmo_script.js

@@ -21,6 +21,15 @@ export const MulmoScriptMethods = {
     getSpeechProvider(script) {
         return text2SpeechProviderSchema.parse(script.speechParams?.provider);
     },
+    getAllSpeechProviders(script) {
+        const providers = new Set();
+        const defaultProvider = this.getSpeechProvider(script);
+        Object.values(script.speechParams.speakers).forEach((speaker) => {
+            const provider = speaker.provider ?? defaultProvider;
+            providers.add(provider);
+        });
+        return providers;
+    },
     getTextSlideStyle(script, beat) {
         const styles = script.textSlideParams?.cssStyles ?? [];
         // NOTES: Taking advantage of CSS override rule (you can redefine it to override)

package/lib/methods/mulmo_studio_context.d.ts

@@ -1,4 +1,9 @@
 import { MulmoStudioContext } from "../types/index.js";
+type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
+type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie";
 export declare const MulmoStudioContextMethods: {
     resolveAssetPath(context: MulmoStudioContext, relativePath: string): string;
+    setSessionState(context: MulmoStudioContext, sessionType: SessionType, value: boolean): void;
+    setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType, index: number, value: boolean): void;
 };
+export {};

package/lib/methods/mulmo_studio_context.js

@@ -1,6 +1,29 @@
 import path from "path";
+import { GraphAILogger } from "graphai";
+const notifyStateChange = (context, sessionType) => {
+    const prefix = context.sessionState.inSession[sessionType] ? "<" : " >";
+    GraphAILogger.info(`${prefix} ${sessionType}`);
+};
+const notifyBeatStateChange = (context, sessionType, index) => {
+    const prefix = context.sessionState.inBeatSession[sessionType][index] ? "{" : " }";
+    GraphAILogger.info(`${prefix} ${sessionType} ${index}`);
+};
 export const MulmoStudioContextMethods = {
     resolveAssetPath(context, relativePath) {
         return path.resolve(context.fileDirs.mulmoFileDirPath, relativePath);
     },
+    setSessionState(context, sessionType, value) {
+        context.sessionState.inSession[sessionType] = value;
+        notifyStateChange(context, sessionType);
+    },
+    setBeatSessionState(context, sessionType, index, value) {
+        if (value) {
+            context.sessionState.inBeatSession[sessionType][index] = true;
+        }
+        else {
+            // NOTE: Setting to false causes the parse error in rebuildStudio in preprocess.ts
+            delete context.sessionState.inBeatSession[sessionType][index];
+        }
+        notifyBeatStateChange(context, sessionType, index);
+    },
 };