mulmocast 2.0.7 → 2.0.8

package/lib/actions/audio.js

@@ -2,7 +2,7 @@ import dotenv from "dotenv";
 import { GraphAI, TaskManager, GraphAILogger } from "graphai";
 import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-import { ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, ttsElevenlabsAgent, addBGMAgent, combineAudioFilesAgent, mediaMockAgent, } from "../agents/index.js";
+import { ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, ttsElevenlabsAgent, ttsKotodamaAgent, addBGMAgent, combineAudioFilesAgent, mediaMockAgent, } from "../agents/index.js";
 import { text2SpeechProviderSchema } from "../types/index.js";
 import { fileCacheAgentFilter, nijovoiceTextAgentFilter } from "../utils/filters.js";
 import { getAudioArtifactFilePath, getAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage } from "../utils/file.js";
@@ -108,6 +108,7 @@ const graph_tts = {
                     voice: ":preprocessor.voiceId",
                     speed: ":preprocessor.speechOptions.speed",
                     instructions: ":preprocessor.speechOptions.instruction",
+                    decoration: ":preprocessor.speechOptions.decoration",
                     model: ":preprocessor.model",
                 },
             },
@@ -222,6 +223,7 @@ const audioAgents = {
     ttsNijivoiceAgent,
     ttsGoogleAgent,
     ttsGeminiAgent,
+    ttsKotodamaAgent,
     ttsElevenlabsAgent,
     mediaMockAgent,
     addBGMAgent,

package/lib/agents/index.d.ts

@@ -12,6 +12,7 @@ import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import ttsGoogleAgent from "./tts_google_agent.js";
 import ttsGeminiAgent from "./tts_gemini_agent.js";
+import ttsKotodamaAgent from "./tts_kotodama_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
 import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
 import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
@@ -20,4 +21,4 @@ import { browserlessAgent } from "@graphai/browserless_agent";
 import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, ttsKotodamaAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/agents/index.js

@@ -12,6 +12,7 @@ import ttsNijivoiceAgent from "./tts_nijivoice_agent.js";
 import ttsOpenaiAgent from "./tts_openai_agent.js";
 import ttsGoogleAgent from "./tts_google_agent.js";
 import ttsGeminiAgent from "./tts_gemini_agent.js";
+import ttsKotodamaAgent from "./tts_kotodama_agent.js";
 import validateSchemaAgent from "./validate_schema_agent.js";
 import soundEffectReplicateAgent from "./sound_effect_replicate_agent.js";
 import lipSyncReplicateAgent from "./lipsync_replicate_agent.js";
@@ -21,4 +22,4 @@ import { textInputAgent } from "@graphai/input_agents";
 import { openAIAgent } from "@graphai/openai_agent";
 // import * as vanilla from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
-export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };
+export { openAIAgent, fileWriteAgent, browserlessAgent, textInputAgent, addBGMAgent, combineAudioFilesAgent, imageGenAIAgent, imageOpenaiAgent, imageReplicateAgent, tavilySearchAgent, movieGenAIAgent, movieReplicateAgent, mediaMockAgent, ttsElevenlabsAgent, ttsNijivoiceAgent, ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, ttsKotodamaAgent, validateSchemaAgent, soundEffectReplicateAgent, lipSyncReplicateAgent, puppeteerCrawlerAgent, };

package/lib/types/agent.d.ts

@@ -116,6 +116,9 @@ export type NijivoiceTTSAgentParams = TTSAgentParams & {
     speed: number;
     speed_global: number;
 };
+export type KotodamaTTSAgentParams = TTSAgentParams & {
+    decoration: string;
+};
 export type GoogleTTSAgentParams = TTSAgentParams & {
     speed: number;
 };

package/lib/types/schema.d.ts

@@ -19,6 +19,7 @@ export declare const multiLingualTextsSchema: z.ZodRecord<z.ZodString, z.ZodObje
 export declare const speechOptionsSchema: z.ZodObject<{
     speed: z.ZodOptional<z.ZodNumber>;
     instruction: z.ZodOptional<z.ZodString>;
+    decoration: z.ZodOptional<z.ZodString>;
 }, z.core.$strict>;
 export declare const defaultSpeaker = "Presenter";
 export declare const text2SpeechProviderSchema: z.ZodDefault<z.ZodEnum<{
@@ -31,6 +32,7 @@ export declare const speakerDataSchema: z.ZodObject<{
     speechOptions: z.ZodOptional<z.ZodObject<{
         speed: z.ZodOptional<z.ZodNumber>;
         instruction: z.ZodOptional<z.ZodString>;
+        decoration: z.ZodOptional<z.ZodString>;
     }, z.core.$strict>>;
     provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
         [x: string]: string;
@@ -44,6 +46,7 @@ export declare const speakerSchema: z.ZodObject<{
     speechOptions: z.ZodOptional<z.ZodObject<{
         speed: z.ZodOptional<z.ZodNumber>;
         instruction: z.ZodOptional<z.ZodString>;
+        decoration: z.ZodOptional<z.ZodString>;
     }, z.core.$strict>>;
     provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
         [x: string]: string;
@@ -56,6 +59,7 @@ export declare const speakerSchema: z.ZodObject<{
         speechOptions: z.ZodOptional<z.ZodObject<{
             speed: z.ZodOptional<z.ZodNumber>;
             instruction: z.ZodOptional<z.ZodString>;
+            decoration: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
         provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
             [x: string]: string;
@@ -70,6 +74,7 @@ export declare const speakerDictionarySchema: z.ZodRecord<z.ZodString, z.ZodObje
     speechOptions: z.ZodOptional<z.ZodObject<{
         speed: z.ZodOptional<z.ZodNumber>;
         instruction: z.ZodOptional<z.ZodString>;
+        decoration: z.ZodOptional<z.ZodString>;
     }, z.core.$strict>>;
     provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
         [x: string]: string;
@@ -82,6 +87,7 @@ export declare const speakerDictionarySchema: z.ZodRecord<z.ZodString, z.ZodObje
         speechOptions: z.ZodOptional<z.ZodObject<{
             speed: z.ZodOptional<z.ZodNumber>;
             instruction: z.ZodOptional<z.ZodString>;
+            decoration: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
         provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
             [x: string]: string;
@@ -97,6 +103,7 @@ export declare const mulmoSpeechParamsSchema: z.ZodDefault<z.ZodObject<{
         speechOptions: z.ZodOptional<z.ZodObject<{
             speed: z.ZodOptional<z.ZodNumber>;
             instruction: z.ZodOptional<z.ZodString>;
+            decoration: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
         provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
             [x: string]: string;
@@ -109,6 +116,7 @@ export declare const mulmoSpeechParamsSchema: z.ZodDefault<z.ZodObject<{
             speechOptions: z.ZodOptional<z.ZodObject<{
                 speed: z.ZodOptional<z.ZodNumber>;
                 instruction: z.ZodOptional<z.ZodString>;
+                decoration: z.ZodOptional<z.ZodString>;
             }, z.core.$strict>>;
             provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                 [x: string]: string;
@@ -629,6 +637,7 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     speechOptions: z.ZodOptional<z.ZodObject<{
         speed: z.ZodOptional<z.ZodNumber>;
         instruction: z.ZodOptional<z.ZodString>;
+        decoration: z.ZodOptional<z.ZodString>;
     }, z.core.$strict>>;
     textSlideParams: z.ZodOptional<z.ZodObject<{
         cssStyles: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
@@ -716,6 +725,7 @@ export declare const mulmoPresentationStyleSchema: z.ZodObject<{
             speechOptions: z.ZodOptional<z.ZodObject<{
                 speed: z.ZodOptional<z.ZodNumber>;
                 instruction: z.ZodOptional<z.ZodString>;
+                decoration: z.ZodOptional<z.ZodString>;
             }, z.core.$strict>>;
             provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                 [x: string]: string;
@@ -728,6 +738,7 @@ export declare const mulmoPresentationStyleSchema: z.ZodObject<{
                 speechOptions: z.ZodOptional<z.ZodObject<{
                     speed: z.ZodOptional<z.ZodNumber>;
                     instruction: z.ZodOptional<z.ZodString>;
+                    decoration: z.ZodOptional<z.ZodString>;
                 }, z.core.$strict>>;
                 provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                     [x: string]: string;
@@ -852,6 +863,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
             speechOptions: z.ZodOptional<z.ZodObject<{
                 speed: z.ZodOptional<z.ZodNumber>;
                 instruction: z.ZodOptional<z.ZodString>;
+                decoration: z.ZodOptional<z.ZodString>;
             }, z.core.$strict>>;
             provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                 [x: string]: string;
@@ -864,6 +876,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
                 speechOptions: z.ZodOptional<z.ZodObject<{
                     speed: z.ZodOptional<z.ZodNumber>;
                     instruction: z.ZodOptional<z.ZodString>;
+                    decoration: z.ZodOptional<z.ZodString>;
                 }, z.core.$strict>>;
                 provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                     [x: string]: string;
@@ -1133,6 +1146,7 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         speechOptions: z.ZodOptional<z.ZodObject<{
             speed: z.ZodOptional<z.ZodNumber>;
             instruction: z.ZodOptional<z.ZodString>;
+            decoration: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
         textSlideParams: z.ZodOptional<z.ZodObject<{
             cssStyles: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
@@ -1264,6 +1278,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
                 speechOptions: z.ZodOptional<z.ZodObject<{
                     speed: z.ZodOptional<z.ZodNumber>;
                     instruction: z.ZodOptional<z.ZodString>;
+                    decoration: z.ZodOptional<z.ZodString>;
                 }, z.core.$strict>>;
                 provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                     [x: string]: string;
@@ -1276,6 +1291,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
                     speechOptions: z.ZodOptional<z.ZodObject<{
                         speed: z.ZodOptional<z.ZodNumber>;
                         instruction: z.ZodOptional<z.ZodString>;
+                        decoration: z.ZodOptional<z.ZodString>;
                     }, z.core.$strict>>;
                     provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                         [x: string]: string;
@@ -1545,6 +1561,7 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             speechOptions: z.ZodOptional<z.ZodObject<{
                 speed: z.ZodOptional<z.ZodNumber>;
                 instruction: z.ZodOptional<z.ZodString>;
+                decoration: z.ZodOptional<z.ZodString>;
             }, z.core.$strict>>;
             textSlideParams: z.ZodOptional<z.ZodObject<{
                 cssStyles: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodString>]>;
@@ -1612,6 +1629,7 @@ export declare const mulmoPromptTemplateSchema: z.ZodObject<{
                 speechOptions: z.ZodOptional<z.ZodObject<{
                     speed: z.ZodOptional<z.ZodNumber>;
                     instruction: z.ZodOptional<z.ZodString>;
+                    decoration: z.ZodOptional<z.ZodString>;
                 }, z.core.$strict>>;
                 provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                     [x: string]: string;
@@ -1624,6 +1642,7 @@ export declare const mulmoPromptTemplateSchema: z.ZodObject<{
                     speechOptions: z.ZodOptional<z.ZodObject<{
                         speed: z.ZodOptional<z.ZodNumber>;
                         instruction: z.ZodOptional<z.ZodString>;
+                        decoration: z.ZodOptional<z.ZodString>;
                     }, z.core.$strict>>;
                     provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                         [x: string]: string;
@@ -1742,6 +1761,7 @@ export declare const mulmoPromptTemplateFileSchema: z.ZodObject<{
                 speechOptions: z.ZodOptional<z.ZodObject<{
                     speed: z.ZodOptional<z.ZodNumber>;
                     instruction: z.ZodOptional<z.ZodString>;
+                    decoration: z.ZodOptional<z.ZodString>;
                 }, z.core.$strict>>;
                 provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                     [x: string]: string;
@@ -1754,6 +1774,7 @@ export declare const mulmoPromptTemplateFileSchema: z.ZodObject<{
                     speechOptions: z.ZodOptional<z.ZodObject<{
                         speed: z.ZodOptional<z.ZodNumber>;
                         instruction: z.ZodOptional<z.ZodString>;
+                        decoration: z.ZodOptional<z.ZodString>;
                     }, z.core.$strict>>;
                     provider: z.ZodOptional<z.ZodDefault<z.ZodEnum<{
                         [x: string]: string;

package/lib/types/schema.js

@@ -18,8 +18,9 @@ export const localizedTextSchema = z
 export const multiLingualTextsSchema = z.record(langSchema, localizedTextSchema);
 export const speechOptionsSchema = z
     .object({
-    speed: z.number().optional(), // default: 1.0
-    instruction: z.string().optional(),
+    speed: z.number().optional(), // default: 1.0 for google and niji voice
+    instruction: z.string().optional(), // for tts openai
+    decoration: z.string().optional(), // for kotodama. default: neutral
 })
     .strict();
 const speakerIdSchema = z.string();

package/lib/utils/context.d.ts

@@ -17,6 +17,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                 speechOptions?: {
                     speed?: number | undefined;
                     instruction?: string | undefined;
+                    decoration?: string | undefined;
                 } | undefined;
                 provider?: string | undefined;
                 model?: string | undefined;
@@ -27,6 +28,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
                     speechOptions?: {
                         speed?: number | undefined;
                         instruction?: string | undefined;
+                        decoration?: string | undefined;
                     } | undefined;
                     provider?: string | undefined;
                     model?: string | undefined;
@@ -242,6 +244,7 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             speechOptions?: {
                 speed?: number | undefined;
                 instruction?: string | undefined;
+                decoration?: string | undefined;
             } | undefined;
             textSlideParams?: {
                 cssStyles: string | string[];
@@ -332,6 +335,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     speechOptions?: {
                         speed?: number | undefined;
                         instruction?: string | undefined;
+                        decoration?: string | undefined;
                     } | undefined;
                     provider?: string | undefined;
                     model?: string | undefined;
@@ -342,6 +346,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                         speechOptions?: {
                             speed?: number | undefined;
                             instruction?: string | undefined;
+                            decoration?: string | undefined;
                         } | undefined;
                         provider?: string | undefined;
                         model?: string | undefined;
@@ -557,6 +562,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 speechOptions?: {
                     speed?: number | undefined;
                     instruction?: string | undefined;
+                    decoration?: string | undefined;
                 } | undefined;
                 textSlideParams?: {
                     cssStyles: string | string[];
@@ -654,6 +660,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 speechOptions?: {
                     speed?: number | undefined;
                     instruction?: string | undefined;
+                    decoration?: string | undefined;
                 } | undefined;
                 provider?: string | undefined;
                 model?: string | undefined;
@@ -664,6 +671,7 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                     speechOptions?: {
                         speed?: number | undefined;
                         instruction?: string | undefined;
+                        decoration?: string | undefined;
                     } | undefined;
                     provider?: string | undefined;
                     model?: string | undefined;

package/lib/utils/provider2agent.d.ts

@@ -29,6 +29,13 @@ export declare const provider2TTSAgent: {
         models: string[];
         keyName: string;
     };
+    kotodama: {
+        agentName: string;
+        hasLimitedConcurrency: boolean;
+        defaultVoice: string;
+        defaultDecoration: string;
+        keyName: string;
+    };
     mock: {
         agentName: string;
         hasLimitedConcurrency: boolean;

package/lib/utils/provider2agent.js

@@ -32,6 +32,13 @@ export const provider2TTSAgent = {
         models: ["eleven_multilingual_v2", "eleven_turbo_v2_5", "eleven_turbo_v2", "eleven_flash_v2_5", "eleven_flash_v2"],
         keyName: "ELEVENLABS_API_KEY",
     },
+    kotodama: {
+        agentName: "ttsKotodamaAgent",
+        hasLimitedConcurrency: true,
+        defaultVoice: "Atla",
+        defaultDecoration: "neutral",
+        keyName: "KOTODAMA_API_KEY",
+    },
     mock: {
         agentName: "mediaMockAgent",
         hasLimitedConcurrency: true,

package/lib/utils/utils.js

@@ -82,6 +82,9 @@ export const settings2GraphAIConfig = (settings, env) => {
         ttsElevenlabsAgent: {
             apiKey: getKey("TTS", "ELEVENLABS_API_KEY"),
         },
+        ttsKotodamaAgent: {
+            apiKey: getKey("TTS", "KOTODAMA_API_KEY"),
+        },
         soundEffectReplicateAgent: {
             apiKey: getKey("SOUND_EFFECT", "REPLICATE_API_TOKEN"),
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.0.7",
+  "version": "2.0.8",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",

package/scripts/test/test_kotodama.json

@@ -0,0 +1,57 @@
+{
+  "$mulmocast": {
+    "version": "1.1"
+  },
+  "lang": "en",
+  "title": "Media Test",
+  "speechParams": {
+    "speakers": {
+      "Presenter": {
+        "provider": "kotodama",
+        "voiceId": "Poporo"
+      },
+      "Shion": {
+        "provider": "kotodama",
+        "speechOptions": {
+          "decoration": "laughing"
+        },
+        "voiceId": "Shion"
+      }
+    }
+  },
+  "beats": [
+    {
+      "speaker": "Presenter",
+      "text": "こんにちは",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Hello, kotodama"
+        }
+      }
+    },
+    {
+      "speaker": "Shion",
+      "text": "こんにちは",
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Hello, kotodama"
+        }
+      }
+    },
+    {
+      "speaker": "Presenter",
+      "text": "ハッピーだね",
+      "speechOptions": {
+        "decoration": "happy"
+      },
+      "image": {
+        "type": "textSlide",
+        "slide": {
+          "title": "Hello, kotodama"
+        }
+      }
+    }
+  ]
+}