mulmocast 0.0.1 → 0.0.3

package/lib/actions/audio.js

@@ -1,88 +1,67 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
+import "dotenv/config";
+import { GraphAI } from "graphai";
+import * as agents from "@graphai/vanilla";
+import ttsNijivoiceAgent from "../agents/tts_nijivoice_agent.js";
+import addBGMAgent from "../agents/add_bgm_agent.js";
+import combineAudioFilesAgent from "../agents/combine_audio_files_agent.js";
+import ttsOpenaiAgent from "../agents/tts_openai_agent.js";
+import { fileWriteAgent } from "@graphai/vanilla_node_agents";
+import { MulmoScriptMethods } from "../methods/index.js";
+import { fileCacheAgentFilter } from "../utils/filters.js";
+import { getAudioArtifactFilePath, getAudioSegmentDirPath, getAudioCombinedFilePath, getOutputStudioFilePath, defaultBGMPath, mkdir, writingMessage, getAudioSegmentFilePath, resolveMediaSource, } from "../utils/file.js";
+import { text2hash } from "../utils/utils.js";
+const { default: __, ...vanillaAgents } = agents;
+// const rion_takanashi_voice = "b9277ce3-ba1c-4f6f-9a65-c05ca102ded0"; // たかなし りおん
+// const ben_carter_voice = "bc06c63f-fef6-43b6-92f7-67f919bd5dae"; // ベン・カーター
+const provider_to_agent = {
+    nijivoice: "ttsNijivoiceAgent",
+    openai: "ttsOpenaiAgent",
+};
+const getAudioPath = (context, beat, audioFile, audioDirPath) => {
+    if (beat.audio?.type === "audio") {
+        const path = resolveMediaSource(beat.audio.source, context);
+        if (path) {
+            return path;
+        }
+        throw new Error("Invalid audio source");
     }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
+    return getAudioSegmentFilePath(audioDirPath, context.studio.filename, audioFile);
+};
+const preprocessor = (namedInputs) => {
+    const { beat, index, context, audioDirPath } = namedInputs;
+    const studioBeat = context.studio.beats[index];
+    const voiceId = context.studio.script.speechParams.speakers[beat.speaker].voiceId;
+    const speechOptions = MulmoScriptMethods.getSpeechOptions(context.studio.script, beat);
+    const hash_string = `${beat.text}${voiceId}${speechOptions?.instruction ?? ""}${speechOptions?.speed ?? 1.0}`;
+    const audioFile = `${context.studio.filename}_${index}_${text2hash(hash_string)}`;
+    const audioPath = getAudioPath(context, beat, audioFile, audioDirPath);
+    studioBeat.audioFile = audioPath;
+    return {
+        ttsAgent: provider_to_agent[context.studio.script.speechParams.provider],
+        studioBeat,
+        voiceId,
+        speechOptions,
+        audioPath,
     };
-})();
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
 };
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.audio = void 0;
-require("dotenv/config");
-const graphai_1 = require("graphai");
-const agents = __importStar(require("@graphai/vanilla"));
-const tts_nijivoice_agent_1 = __importDefault(require("../agents/tts_nijivoice_agent"));
-const add_bgm_agent_1 = __importDefault(require("../agents/add_bgm_agent"));
-const combine_audio_files_agent_1 = __importDefault(require("../agents/combine_audio_files_agent"));
-const tts_openai_agent_1 = __importDefault(require("../agents/tts_openai_agent"));
-const vanilla_node_agents_1 = require("@graphai/vanilla_node_agents");
-const methods_1 = require("../methods");
-const filters_1 = require("../utils/filters");
-const file_1 = require("../utils/file");
-// const rion_takanashi_voice = "b9277ce3-ba1c-4f6f-9a65-c05ca102ded0"; // たかなし りおん
-// const ben_carter_voice = "bc06c63f-fef6-43b6-92f7-67f919bd5dae"; // ベン・カーター
 const graph_tts = {
     nodes: {
         preprocessor: {
-            agent: (namedInputs) => {
-                const { beat, script, speakers } = namedInputs;
-                return {
-                    voiceId: speakers[beat.speaker].voiceId,
-                    speechOptions: methods_1.MulmoScriptMethods.getSpeechOptions(script, beat),
-                };
-            },
+            agent: preprocessor,
             inputs: {
                 beat: ":beat",
-                script: ":script",
-                speakers: ":script.speechParams.speakers",
-            },
-        },
-        ttsAgent: {
-            agent: (namedInputs) => {
-                if (namedInputs.provider === "nijivoice") {
-                    return "ttsNijivoiceAgent";
-                }
-                return "ttsOpenaiAgent";
-            },
-            inputs: {
-                provider: ":script.speechParams.provider",
+                index: ":__mapIndex",
+                context: ":context",
+                audioDirPath: ":audioDirPath",
             },
         },
         tts: {
             unless: ":beat.audio",
-            agent: ":ttsAgent",
+            agent: ":preprocessor.ttsAgent",
             inputs: {
                 text: ":beat.text",
-                file: "${:scratchpadDirPath}/${:beat.audioFile}.mp3", // TODO
+                file: ":preprocessor.audioPath",
+                force: ":context.force",
             },
             params: {
                 voice: ":preprocessor.voiceId",
@@ -97,13 +76,20 @@ const graph_data = {
     concurrency: 8,
     nodes: {
         context: {},
-        outputBGMFilePath: {},
-        outputAudioFilePath: {},
+        audioArtifactFilePath: {},
+        audioCombinedFilePath: {},
         outputStudioFilePath: {},
-        scratchpadDirPath: {},
+        audioDirPath: {},
+        audioSegmentDirPath: {},
         map: {
             agent: "mapAgent",
-            inputs: { rows: ":context.studio.beats", script: ":context.studio.script", scratchpadDirPath: ":scratchpadDirPath" },
+            inputs: {
+                rows: ":context.studio.script.beats",
+                studio: ":context.studio",
+                audioDirPath: ":audioDirPath",
+                audioSegmentDirPath: ":audioSegmentDirPath",
+                context: ":context",
+            },
             params: {
                 rowKey: "beat",
             },
@@ -114,8 +100,8 @@ const graph_data = {
             inputs: {
                 map: ":map",
                 context: ":context",
-                combinedFileName: ":outputAudioFilePath",
-                scratchpadDirPath: ":scratchpadDirPath",
+                combinedFileName: ":audioCombinedFilePath",
+                audioDirPath: ":audioDirPath",
             },
             isResult: true,
         },
@@ -129,12 +115,12 @@ const graph_data = {
         addBGM: {
             agent: "addBGMAgent",
             params: {
-                musicFile: process.env.PATH_BGM ?? file_1.defaultBGMPath,
+                musicFile: process.env.PATH_BGM ?? defaultBGMPath,
             },
             inputs: {
                 wait: ":combineFiles",
-                voiceFile: ":outputAudioFilePath",
-                outputFile: ":outputBGMFilePath",
+                voiceFile: ":audioCombinedFilePath",
+                outputFile: ":audioArtifactFilePath",
                 script: ":context.studio.script",
             },
             isResult: true,
@@ -154,33 +140,34 @@ const graph_data = {
 const agentFilters = [
     {
         name: "fileCacheAgentFilter",
-        agent: filters_1.fileCacheAgentFilter,
+        agent: fileCacheAgentFilter,
         nodeIds: ["tts"],
     },
 ];
-const audio = async (context, concurrency) => {
+export const audio = async (context) => {
     const { studio, fileDirs } = context;
-    const { outDirPath, scratchpadDirPath } = fileDirs;
-    const outputBGMFilePath = (0, file_1.getOutputBGMFilePath)(outDirPath, studio.filename);
-    const outputAudioFilePath = (0, file_1.getOutputAudioFilePath)(outDirPath, studio.filename);
-    const outputStudioFilePath = (0, file_1.getOutputStudioFilePath)(outDirPath, studio.filename);
-    (0, file_1.mkdir)(outDirPath);
-    (0, file_1.mkdir)(scratchpadDirPath);
-    graph_data.concurrency = concurrency;
-    const graph = new graphai_1.GraphAI(graph_data, {
-        ...agents,
-        fileWriteAgent: vanilla_node_agents_1.fileWriteAgent,
-        ttsOpenaiAgent: tts_openai_agent_1.default,
-        ttsNijivoiceAgent: tts_nijivoice_agent_1.default,
-        addBGMAgent: add_bgm_agent_1.default,
-        combineAudioFilesAgent: combine_audio_files_agent_1.default,
+    const { outDirPath, audioDirPath } = fileDirs;
+    const audioArtifactFilePath = getAudioArtifactFilePath(outDirPath, studio.filename);
+    const audioSegmentDirPath = getAudioSegmentDirPath(audioDirPath, studio.filename);
+    const audioCombinedFilePath = getAudioCombinedFilePath(audioDirPath, studio.filename);
+    const outputStudioFilePath = getOutputStudioFilePath(outDirPath, studio.filename);
+    mkdir(outDirPath);
+    mkdir(audioSegmentDirPath);
+    graph_data.concurrency = MulmoScriptMethods.getSpeechProvider(studio.script) === "nijivoice" ? 1 : 8;
+    const graph = new GraphAI(graph_data, {
+        ...vanillaAgents,
+        fileWriteAgent,
+        ttsOpenaiAgent,
+        ttsNijivoiceAgent,
+        addBGMAgent,
+        combineAudioFilesAgent,
     }, { agentFilters });
     graph.injectValue("context", context);
-    graph.injectValue("outputBGMFilePath", outputBGMFilePath);
-    graph.injectValue("outputAudioFilePath", outputAudioFilePath);
+    graph.injectValue("audioArtifactFilePath", audioArtifactFilePath);
+    graph.injectValue("audioCombinedFilePath", audioCombinedFilePath);
     graph.injectValue("outputStudioFilePath", outputStudioFilePath);
-    graph.injectValue("scratchpadDirPath", scratchpadDirPath);
+    graph.injectValue("audioSegmentDirPath", audioSegmentDirPath);
+    graph.injectValue("audioDirPath", audioDirPath);
     await graph.run();
-    (0, file_1.writingMessage)(outputAudioFilePath);
+    writingMessage(audioCombinedFilePath);
 };
-exports.audio = audio;

package/lib/actions/images.d.ts

@@ -1,2 +1,2 @@
-import { MulmoStudioContext } from "../types";
+import { MulmoStudioContext } from "../types/index.js";
 export declare const images: (context: MulmoStudioContext) => Promise<void>;

package/lib/actions/images.js

@@ -1,83 +1,42 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
+import dotenv from "dotenv";
+import { GraphAI, GraphAILogger } from "graphai";
+import * as agents from "@graphai/vanilla";
+import { fileWriteAgent } from "@graphai/vanilla_node_agents";
+import { getOutputStudioFilePath, mkdir } from "../utils/file.js";
+import { fileCacheAgentFilter } from "../utils/filters.js";
+import imageGoogleAgent from "../agents/image_google_agent.js";
+import imageOpenaiAgent from "../agents/image_openai_agent.js";
+import { MulmoScriptMethods } from "../methods/index.js";
+import { imagePlugins } from "../utils/image_plugins/index.js";
+const { default: __, ...vanillaAgents } = agents;
+dotenv.config();
+// const openai = new OpenAI();
+import { GoogleAuth } from "google-auth-library";
+const htmlStyle = (script, beat) => {
+    return {
+        canvasSize: MulmoScriptMethods.getCanvasSize(script),
+        textSlideStyle: MulmoScriptMethods.getTextSlideStyle(script, beat),
     };
-})();
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
 };
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.images = void 0;
-const dotenv_1 = __importDefault(require("dotenv"));
-const graphai_1 = require("graphai");
-const agents = __importStar(require("@graphai/vanilla"));
-const vanilla_node_agents_1 = require("@graphai/vanilla_node_agents");
-const file_1 = require("../utils/file");
-const filters_1 = require("../utils/filters");
-const markdown_1 = require("../utils/markdown");
-const image_google_agent_1 = __importDefault(require("../agents/image_google_agent"));
-const image_openai_agent_1 = __importDefault(require("../agents/image_openai_agent"));
-const methods_1 = require("../methods");
-dotenv_1.default.config();
-// const openai = new OpenAI();
-const google_auth_library_1 = require("google-auth-library");
-const preprocess_agent = async (namedInputs) => {
-    const { context, beat, index, suffix, imageDirPath } = namedInputs;
-    const imageParams = { ...context.studio.script.imageParams, ...beat.imageParams };
-    const prompt = (beat.imagePrompt || beat.text) + "\n" + (imageParams.style || "");
+const imagePreprocessAgent = async (namedInputs) => {
+    const { context, beat, index, suffix, imageDirPath, imageAgentInfo } = namedInputs;
+    const imageParams = { ...imageAgentInfo.imageParams, ...beat.imageParams };
     const imagePath = `${imageDirPath}/${context.studio.filename}/${index}${suffix}.png`;
-    const aspectRatio = methods_1.MulmoScriptMethods.getAspectRatio(context.studio.script);
+    const returnValue = {
+        aspectRatio: MulmoScriptMethods.getAspectRatio(context.studio.script),
+        imageParams,
+    };
     if (beat.image) {
-        if (beat.image.type === "textSlide") {
-            const slide = beat.image.slide;
-            const markdown = `# ${slide.title}` + slide.bullets.map((text) => `- ${text}`).join("\n");
-            await (0, markdown_1.convertMarkdownToImage)(markdown, methods_1.MulmoScriptMethods.getTextSlideStyle(context.studio.script, beat), imagePath);
-        }
-        else if (beat.image.type === "markdown") {
-            const markdown = Array.isArray(beat.image.markdown) ? beat.image.markdown.join("\n") : beat.image.markdown;
-            await (0, markdown_1.convertMarkdownToImage)(markdown, methods_1.MulmoScriptMethods.getTextSlideStyle(context.studio.script, beat), imagePath);
-        }
-        else if (beat.image.type === "image") {
-            if (beat.image.source.kind === "url") {
-                // undefined prompt indicates "no need to generate image"
-                return { path: beat.image.source.url, prompt: undefined, imageParams, aspectRatio };
-            }
-            else if (beat.image.source.kind === "path") {
-                const path = methods_1.MulmoStudioContextMethods.resolveAssetPath(context, beat.image.source.path);
-                return { path, prompt: undefined, imageParams, aspectRatio };
-            }
+        const plugin = imagePlugins.find((plugin) => plugin.imageType === beat?.image?.type);
+        if (plugin) {
+            const processorParams = { beat, context, imagePath, ...htmlStyle(context.studio.script, beat) };
+            const path = await plugin.process(processorParams);
+            // undefined prompt indicates that image generation is not needed
+            return { path, ...returnValue };
         }
     }
-    return { path: imagePath, prompt, imageParams, aspectRatio };
+    const prompt = (beat.imagePrompt || beat.text) + "\n" + (imageParams.style || "");
+    return { path: imagePath, prompt, ...returnValue };
 };
 const graph_data = {
     version: 0.5,
@@ -85,11 +44,11 @@ const graph_data = {
     nodes: {
         context: {},
         imageDirPath: {},
-        text2imageAgent: {},
+        imageAgentInfo: {},
         outputStudioFilePath: {},
         map: {
             agent: "mapAgent",
-            inputs: { rows: ":context.studio.beats", context: ":context", text2imageAgent: ":text2imageAgent", imageDirPath: ":imageDirPath" },
+            inputs: { rows: ":context.studio.script.beats", context: ":context", imageAgentInfo: ":imageAgentInfo", imageDirPath: ":imageDirPath" },
             isResult: true,
             params: {
                 rowKey: "beat",
@@ -98,18 +57,19 @@ const graph_data = {
             graph: {
                 nodes: {
                     preprocessor: {
-                        agent: preprocess_agent,
+                        agent: imagePreprocessAgent,
                         inputs: {
                             context: ":context",
                             beat: ":beat",
                             index: ":__mapIndex",
                             suffix: "p",
                             imageDirPath: ":imageDirPath",
+                            imageAgentInfo: ":imageAgentInfo",
                         },
                     },
                     imageGenerator: {
                         if: ":preprocessor.prompt",
-                        agent: ":text2imageAgent",
+                        agent: ":imageAgentInfo.agent",
                         params: {
                             model: ":preprocessor.imageParams.model",
                             size: ":preprocessor.imageParams.size",
@@ -120,6 +80,7 @@ const graph_data = {
                             prompt: ":preprocessor.prompt",
                             file: ":preprocessor.path", // only for fileCacheAgentFilter
                             text: ":preprocessor.prompt", // only for fileCacheAgentFilter
+                            force: ":context.force",
                         },
                         defaultValue: {},
                     },
@@ -164,30 +125,31 @@ const graph_data = {
     },
 };
 const googleAuth = async () => {
-    const auth = new google_auth_library_1.GoogleAuth({
+    const auth = new GoogleAuth({
         scopes: ["https://www.googleapis.com/auth/cloud-platform"],
     });
     const client = await auth.getClient();
     const accessToken = await client.getAccessToken();
     return accessToken.token;
 };
-const images = async (context) => {
+export const images = async (context) => {
     const { studio, fileDirs } = context;
     const { outDirPath, imageDirPath } = fileDirs;
-    (0, file_1.mkdir)(`${imageDirPath}/${studio.filename}`);
+    mkdir(`${imageDirPath}/${studio.filename}`);
     const agentFilters = [
         {
             name: "fileCacheAgentFilter",
-            agent: filters_1.fileCacheAgentFilter,
+            agent: fileCacheAgentFilter,
             nodeIds: ["imageGenerator"],
         },
     ];
     const options = {
         agentFilters,
     };
+    const imageAgentInfo = MulmoScriptMethods.getImageAgentInfo(studio.script);
     // We need to get google's auth token only if the google is the text2image provider.
-    if (methods_1.MulmoScriptMethods.getImageProvider(studio.script) === "google") {
-        console.log("google was specified as text2image engine");
+    if (imageAgentInfo.provider === "google") {
+        GraphAILogger.log("google was specified as text2image engine");
         const token = await googleAuth();
         options.config = {
             imageGoogleAgent: {
@@ -196,16 +158,16 @@ const images = async (context) => {
             },
         };
     }
+    GraphAILogger.info(`text2image: provider=${imageAgentInfo.provider} model=${imageAgentInfo.imageParams.model}`);
     const injections = {
         context,
-        text2imageAgent: methods_1.MulmoScriptMethods.getText2imageAgent(studio.script),
-        outputStudioFilePath: (0, file_1.getOutputStudioFilePath)(outDirPath, studio.filename),
+        imageAgentInfo,
+        outputStudioFilePath: getOutputStudioFilePath(outDirPath, studio.filename),
         imageDirPath,
     };
-    const graph = new graphai_1.GraphAI(graph_data, { ...agents, imageGoogleAgent: image_google_agent_1.default, imageOpenaiAgent: image_openai_agent_1.default, fileWriteAgent: vanilla_node_agents_1.fileWriteAgent }, options);
+    const graph = new GraphAI(graph_data, { ...vanillaAgents, imageGoogleAgent, imageOpenaiAgent, fileWriteAgent }, options);
     Object.keys(injections).forEach((key) => {
         graph.injectValue(key, injections[key]);
     });
     await graph.run();
 };
-exports.images = images;

package/lib/actions/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./audio.js";
+export * from "./images.js";
+export * from "./movie.js";
+export * from "./pdf.js";
+export * from "./translate.js";

package/lib/actions/index.js

@@ -0,0 +1,5 @@
+export * from "./audio.js";
+export * from "./images.js";
+export * from "./movie.js";
+export * from "./pdf.js";
+export * from "./translate.js";

package/lib/actions/movie.d.ts

@@ -1,2 +1,10 @@
-import { MulmoStudioContext } from "../types";
+import { MulmoStudioContext, MulmoCanvasDimension, BeatMediaType } from "../types/index.js";
+export declare const getVideoPart: (inputIndex: number, mediaType: BeatMediaType, duration: number, canvasInfo: MulmoCanvasDimension) => {
+    videoId: string;
+    videoPart: string;
+};
+export declare const getAudioPart: (inputIndex: number, duration: number, delay: number) => {
+    audioId: string;
+    audioPart: string;
+};
 export declare const movie: (context: MulmoStudioContext) => Promise<void>;