mulmocast 2.0.6 → 2.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/agents/image_genai_agent.js +62 -56
- package/lib/agents/image_replicate_agent.js +9 -1
- package/lib/agents/movie_genai_agent.d.ts +0 -4
- package/lib/agents/movie_genai_agent.js +3 -12
- package/lib/agents/test.d.ts +1 -0
- package/lib/agents/test.js +12 -0
- package/lib/agents/tts_elevenlabs_agent.js +42 -32
- package/lib/agents/tts_gemini_agent.js +8 -2
- package/lib/agents/tts_kotodama_agent.d.ts +5 -0
- package/lib/agents/tts_kotodama_agent.js +76 -0
- package/lib/agents/tts_openai_agent.js +1 -1
- package/lib/agents/utils.d.ts +1 -0
- package/lib/agents/utils.js +1 -0
- package/lib/utils/const.d.ts +1 -0
- package/lib/utils/const.js +1 -0
- package/lib/utils/error_cause.d.ts +10 -0
- package/lib/utils/error_cause.js +22 -0
- package/lib/utils/utils.d.ts +4 -0
- package/lib/utils/utils.js +18 -6
- package/package.json +4 -4
- package/scripts/test/README.md +161 -0
- package/scripts/test/test_all_elevenlabs_tts_model.json +111 -0
- package/scripts/test/test_all_gemini_tts_model.json +433 -0
- package/scripts/test/test_all_image.json +40 -0
- package/scripts/test/test_all_image.json~ +45 -0
- package/scripts/test/test_all_movie.json +33 -0
- package/scripts/test/test_all_movie.json~ +37 -0
- package/scripts/test/test_all_tts.json +83 -0
- package/scripts/test/test_all_tts.json~ +83 -0
- package/scripts/test/test_kotodama.json~ +0 -0
|
@@ -1,18 +1,11 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import { GraphAILogger } from "graphai";
|
|
3
3
|
import { provider2ImageAgent } from "../utils/provider2agent.js";
|
|
4
|
-
import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, imageAction, imageFileTarget, hasCause } from "../utils/error_cause.js";
|
|
4
|
+
import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, agentInvalidResponseError, imageAction, imageFileTarget, hasCause, getGenAIErrorReason, resultify, } from "../utils/error_cause.js";
|
|
5
|
+
import { getAspectRatio } from "../utils/utils.js";
|
|
6
|
+
import { ASPECT_RATIOS } from "../utils/const.js";
|
|
5
7
|
import { GoogleGenAI, PersonGeneration } from "@google/genai";
|
|
6
8
|
import { blankImagePath, blankSquareImagePath, blankVerticalImagePath } from "../utils/file.js";
|
|
7
|
-
const getAspectRatio = (canvasSize) => {
|
|
8
|
-
if (canvasSize.width > canvasSize.height) {
|
|
9
|
-
return "16:9";
|
|
10
|
-
}
|
|
11
|
-
else if (canvasSize.width < canvasSize.height) {
|
|
12
|
-
return "9:16";
|
|
13
|
-
}
|
|
14
|
-
return "1:1";
|
|
15
|
-
};
|
|
16
9
|
export const ratio2BlankPath = (aspectRatio) => {
|
|
17
10
|
if (aspectRatio === "9:16") {
|
|
18
11
|
return blankVerticalImagePath();
|
|
@@ -61,9 +54,24 @@ const geminiFlashResult = (response) => {
|
|
|
61
54
|
cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
|
|
62
55
|
});
|
|
63
56
|
};
|
|
57
|
+
const errorProcess = (error) => {
|
|
58
|
+
GraphAILogger.info("Failed to generate image:", error);
|
|
59
|
+
if (hasCause(error) && error.cause) {
|
|
60
|
+
throw error;
|
|
61
|
+
}
|
|
62
|
+
const reasonDetail = getGenAIErrorReason(error);
|
|
63
|
+
if (reasonDetail && reasonDetail.reason && reasonDetail.reason === "API_KEY_INVALID") {
|
|
64
|
+
throw new Error("Failed to generate image: 400 Incorrect API key provided with gemini", {
|
|
65
|
+
cause: agentIncorrectAPIKeyError("imageGenAIAgent", imageAction, imageFileTarget),
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
throw new Error("Failed to generate image with Google GenAI", {
|
|
69
|
+
cause: agentGenerationError("imageGenAIAgent", imageAction, imageFileTarget),
|
|
70
|
+
});
|
|
71
|
+
};
|
|
64
72
|
export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
65
73
|
const { prompt, referenceImages } = namedInputs;
|
|
66
|
-
const aspectRatio = getAspectRatio(params.canvasSize);
|
|
74
|
+
const aspectRatio = getAspectRatio(params.canvasSize, ASPECT_RATIOS);
|
|
67
75
|
const model = params.model ?? provider2ImageAgent["google"].defaultModel;
|
|
68
76
|
const apiKey = config?.apiKey;
|
|
69
77
|
if (!apiKey) {
|
|
@@ -71,61 +79,60 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
71
79
|
cause: apiKeyMissingError("imageGenAIAgent", imageAction, "GEMINI_API_KEY"),
|
|
72
80
|
});
|
|
73
81
|
}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
+
const ai = new GoogleGenAI({ apiKey });
|
|
83
|
+
if (model === "gemini-2.5-flash-image" || model === "gemini-3-pro-image-preview") {
|
|
84
|
+
const contentParams = (() => {
|
|
85
|
+
if (model === "gemini-2.5-flash-image") {
|
|
86
|
+
const contents = getGeminiContents(prompt, referenceImages, aspectRatio);
|
|
87
|
+
return { model, contents };
|
|
88
|
+
}
|
|
89
|
+
// gemini-3-pro-image-preview
|
|
82
90
|
const contents = getGeminiContents(prompt, referenceImages);
|
|
83
|
-
const
|
|
91
|
+
const PRO_ASPECT_RATIOS = ["1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"];
|
|
92
|
+
return {
|
|
84
93
|
model,
|
|
85
94
|
contents,
|
|
86
95
|
config: {
|
|
87
96
|
imageConfig: {
|
|
88
|
-
|
|
89
|
-
aspectRatio,
|
|
97
|
+
aspectRatio: getAspectRatio(params.canvasSize, PRO_ASPECT_RATIOS),
|
|
90
98
|
},
|
|
91
99
|
},
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
model,
|
|
98
|
-
prompt,
|
|
99
|
-
config: {
|
|
100
|
-
numberOfImages: 1, // default is 4!
|
|
101
|
-
aspectRatio,
|
|
102
|
-
personGeneration: PersonGeneration.ALLOW_ALL,
|
|
103
|
-
// safetyFilterLevel: SafetyFilterLevel.BLOCK_ONLY_HIGH,
|
|
104
|
-
},
|
|
105
|
-
});
|
|
106
|
-
if (!response.generatedImages || response.generatedImages.length === 0) {
|
|
107
|
-
throw new Error("ERROR: generateImage returned no generated images", {
|
|
108
|
-
cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
|
|
109
|
-
});
|
|
110
|
-
}
|
|
111
|
-
const image = response.generatedImages[0].image;
|
|
112
|
-
if (image && image.imageBytes) {
|
|
113
|
-
return { buffer: Buffer.from(image.imageBytes, "base64") };
|
|
114
|
-
}
|
|
115
|
-
throw new Error("ERROR: generateImage returned no image bytes", {
|
|
116
|
-
cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
|
|
117
|
-
});
|
|
100
|
+
};
|
|
101
|
+
})();
|
|
102
|
+
const res = await resultify(() => ai.models.generateContent(contentParams));
|
|
103
|
+
if (res.ok) {
|
|
104
|
+
return geminiFlashResult(res.value);
|
|
118
105
|
}
|
|
106
|
+
return errorProcess(res.error);
|
|
119
107
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
108
|
+
// other case,
|
|
109
|
+
const generateParams = {
|
|
110
|
+
model,
|
|
111
|
+
prompt,
|
|
112
|
+
config: {
|
|
113
|
+
numberOfImages: 1, // default is 4!
|
|
114
|
+
aspectRatio,
|
|
115
|
+
personGeneration: PersonGeneration.ALLOW_ALL,
|
|
116
|
+
// safetyFilterLevel: SafetyFilterLevel.BLOCK_ONLY_HIGH,
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
const res = await resultify(() => ai.models.generateImages(generateParams));
|
|
120
|
+
if (!res.ok) {
|
|
121
|
+
return errorProcess(res.error);
|
|
122
|
+
}
|
|
123
|
+
const response = res.value;
|
|
124
|
+
if (!response.generatedImages || response.generatedImages.length === 0) {
|
|
125
|
+
throw new Error("ERROR: generateImage returned no generated images", {
|
|
126
|
+
cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
|
|
127
127
|
});
|
|
128
128
|
}
|
|
129
|
+
const image = response.generatedImages[0].image;
|
|
130
|
+
if (image && image.imageBytes) {
|
|
131
|
+
return { buffer: Buffer.from(image.imageBytes, "base64") };
|
|
132
|
+
}
|
|
133
|
+
throw new Error("ERROR: generateImage returned no image bytes", {
|
|
134
|
+
cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
|
|
135
|
+
});
|
|
129
136
|
};
|
|
130
137
|
const imageGenAIAgentInfo = {
|
|
131
138
|
name: "imageGenAIAgent",
|
|
@@ -136,7 +143,6 @@ const imageGenAIAgentInfo = {
|
|
|
136
143
|
category: ["image"],
|
|
137
144
|
author: "Receptron Team",
|
|
138
145
|
repository: "https://github.com/receptron/mulmocast-cli/",
|
|
139
|
-
// source: "https://github.com/receptron/mulmocast-cli/blob/main/src/agents/image_google_agent.ts",
|
|
140
146
|
license: "MIT",
|
|
141
147
|
environmentVariables: [],
|
|
142
148
|
};
|
|
@@ -2,7 +2,7 @@ import { readFileSync } from "fs";
|
|
|
2
2
|
import { GraphAILogger } from "graphai";
|
|
3
3
|
import Replicate from "replicate";
|
|
4
4
|
import { getAspectRatio } from "./movie_replicate_agent.js";
|
|
5
|
-
import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, imageAction, imageFileTarget, hasCause } from "../utils/error_cause.js";
|
|
5
|
+
import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, agentInvalidResponseError, imageAction, imageFileTarget, hasCause, } from "../utils/error_cause.js";
|
|
6
6
|
import { provider2ImageAgent } from "../utils/provider2agent.js";
|
|
7
7
|
export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
|
|
8
8
|
const { prompt, referenceImages } = namedInputs;
|
|
@@ -51,6 +51,14 @@ export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
|
|
|
51
51
|
if (hasCause(error) && error.cause) {
|
|
52
52
|
throw error;
|
|
53
53
|
}
|
|
54
|
+
if (typeof error === "object" && error !== null && "response" in error) {
|
|
55
|
+
const errorWithResponse = error;
|
|
56
|
+
if (errorWithResponse.response?.status === 401) {
|
|
57
|
+
throw new Error("Failed to generate image: 401 Incorrect API key provided with replicate", {
|
|
58
|
+
cause: agentIncorrectAPIKeyError("imageGenAIAgent", imageAction, imageFileTarget),
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
54
62
|
throw new Error("Failed to generate image with Replicate", {
|
|
55
63
|
cause: agentGenerationError("imageReplicateAgent", imageAction, imageFileTarget),
|
|
56
64
|
});
|
|
@@ -1,9 +1,5 @@
|
|
|
1
1
|
import type { AgentFunction, AgentFunctionInfo } from "graphai";
|
|
2
2
|
import type { AgentBufferResult, GenAIImageAgentConfig, GoogleMovieAgentParams, MovieAgentInputs } from "../types/agent.js";
|
|
3
|
-
export declare const getAspectRatio: (canvasSize: {
|
|
4
|
-
width: number;
|
|
5
|
-
height: number;
|
|
6
|
-
}) => string;
|
|
7
3
|
export declare const movieGenAIAgent: AgentFunction<GoogleMovieAgentParams, AgentBufferResult, MovieAgentInputs, GenAIImageAgentConfig>;
|
|
8
4
|
declare const movieGenAIAgentInfo: AgentFunctionInfo;
|
|
9
5
|
export default movieGenAIAgentInfo;
|
|
@@ -2,18 +2,9 @@ import { readFileSync } from "fs";
|
|
|
2
2
|
import { GraphAILogger, sleep } from "graphai";
|
|
3
3
|
import { GoogleGenAI, PersonGeneration } from "@google/genai";
|
|
4
4
|
import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, imageAction, movieFileTarget, videoDurationTarget, hasCause, } from "../utils/error_cause.js";
|
|
5
|
+
import { getAspectRatio } from "../utils/utils.js";
|
|
6
|
+
import { ASPECT_RATIOS } from "../utils/const.js";
|
|
5
7
|
import { getModelDuration, provider2MovieAgent } from "../utils/provider2agent.js";
|
|
6
|
-
export const getAspectRatio = (canvasSize) => {
|
|
7
|
-
if (canvasSize.width > canvasSize.height) {
|
|
8
|
-
return "16:9";
|
|
9
|
-
}
|
|
10
|
-
else if (canvasSize.width < canvasSize.height) {
|
|
11
|
-
return "9:16";
|
|
12
|
-
}
|
|
13
|
-
else {
|
|
14
|
-
return "1:1";
|
|
15
|
-
}
|
|
16
|
-
};
|
|
17
8
|
const pollUntilDone = async (ai, operation) => {
|
|
18
9
|
const response = { operation };
|
|
19
10
|
while (!response.operation.done) {
|
|
@@ -115,7 +106,7 @@ const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath,
|
|
|
115
106
|
};
|
|
116
107
|
export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
117
108
|
const { prompt, imagePath, movieFile } = namedInputs;
|
|
118
|
-
const aspectRatio = getAspectRatio(params.canvasSize);
|
|
109
|
+
const aspectRatio = getAspectRatio(params.canvasSize, ASPECT_RATIOS);
|
|
119
110
|
const model = params.model ?? provider2MovieAgent.google.defaultModel;
|
|
120
111
|
const apiKey = config?.apiKey;
|
|
121
112
|
if (!apiKey) {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import "dotenv/config";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import "dotenv/config";
|
|
2
|
+
import { ttsKotodamaAgent } from "./tts_kotodama_agent.js";
|
|
3
|
+
const kotodamaApiKey = process.env.KOTODAMA_API_KEY ?? "";
|
|
4
|
+
const main = async () => {
|
|
5
|
+
const result = await ttsKotodamaAgent({
|
|
6
|
+
namedInputs: { text: "こんにちは" },
|
|
7
|
+
params: { voice: "Atla", decoration: "neutral", suppressError: false },
|
|
8
|
+
config: { apiKey: kotodamaApiKey },
|
|
9
|
+
});
|
|
10
|
+
console.log("Result:", result);
|
|
11
|
+
};
|
|
12
|
+
main();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
2
|
import { provider2TTSAgent } from "../utils/provider2agent.js";
|
|
3
|
-
import { apiKeyMissingError, agentGenerationError, audioAction, audioFileTarget } from "../utils/error_cause.js";
|
|
3
|
+
import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, audioAction, audioFileTarget } from "../utils/error_cause.js";
|
|
4
4
|
export const ttsElevenlabsAgent = async ({ namedInputs, params, config, }) => {
|
|
5
5
|
const { text } = namedInputs;
|
|
6
6
|
const { voice, model, stability, similarityBoost, suppressError } = params;
|
|
@@ -15,45 +15,55 @@ export const ttsElevenlabsAgent = async ({ namedInputs, params, config, }) => {
|
|
|
15
15
|
cause: agentGenerationError("ttsElevenlabsAgent", audioAction, audioFileTarget),
|
|
16
16
|
});
|
|
17
17
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
18
|
+
const requestBody = {
|
|
19
|
+
text,
|
|
20
|
+
model_id: model ?? provider2TTSAgent.elevenlabs.defaultModel,
|
|
21
|
+
voice_settings: {
|
|
22
|
+
stability: stability ?? 0.5,
|
|
23
|
+
similarity_boost: similarityBoost ?? 0.75,
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
GraphAILogger.log("ElevenLabs TTS options", requestBody);
|
|
27
|
+
const response = await (async () => {
|
|
28
|
+
try {
|
|
29
|
+
return await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voice}`, {
|
|
30
|
+
method: "POST",
|
|
31
|
+
headers: {
|
|
32
|
+
Accept: "audio/mpeg",
|
|
33
|
+
"Content-Type": "application/json",
|
|
34
|
+
"xi-api-key": apiKey,
|
|
35
|
+
},
|
|
36
|
+
body: JSON.stringify(requestBody),
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
catch (e) {
|
|
40
|
+
if (suppressError) {
|
|
41
|
+
return {
|
|
42
|
+
error: e,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
GraphAILogger.info(e);
|
|
46
|
+
throw new Error("TTS Eleven Labs Error", {
|
|
39
47
|
cause: agentGenerationError("ttsElevenlabsAgent", audioAction, audioFileTarget),
|
|
40
48
|
});
|
|
41
49
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
return
|
|
50
|
+
})();
|
|
51
|
+
if ("error" in response) {
|
|
52
|
+
return response;
|
|
45
53
|
}
|
|
46
|
-
|
|
47
|
-
if (
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
};
|
|
54
|
+
if (!response.ok) {
|
|
55
|
+
if (response.status === 401) {
|
|
56
|
+
throw new Error("Failed to generate audio: 401 Incorrect API key provided with ElevenLabs", {
|
|
57
|
+
cause: agentIncorrectAPIKeyError("ttsElevenlabsAgent", audioAction, audioFileTarget),
|
|
58
|
+
});
|
|
51
59
|
}
|
|
52
|
-
|
|
53
|
-
throw new Error("TTS Eleven Labs Error", {
|
|
60
|
+
throw new Error(`Eleven Labs API error: ${response.status} ${response.statusText}`, {
|
|
54
61
|
cause: agentGenerationError("ttsElevenlabsAgent", audioAction, audioFileTarget),
|
|
55
62
|
});
|
|
56
63
|
}
|
|
64
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
65
|
+
const buffer = Buffer.from(arrayBuffer);
|
|
66
|
+
return { buffer };
|
|
57
67
|
};
|
|
58
68
|
const ttsElevenlabsAgentInfo = {
|
|
59
69
|
name: "ttsElevenlabsAgent",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
2
|
import { GoogleGenAI } from "@google/genai";
|
|
3
3
|
import { provider2TTSAgent } from "../utils/provider2agent.js";
|
|
4
|
-
import { apiKeyMissingError, agentGenerationError, audioAction, audioFileTarget } from "../utils/error_cause.js";
|
|
4
|
+
import { agentIncorrectAPIKeyError, apiKeyMissingError, agentGenerationError, audioAction, audioFileTarget, getGenAIErrorReason, } from "../utils/error_cause.js";
|
|
5
5
|
import { pcmToMp3 } from "../utils/ffmpeg_utils.js";
|
|
6
6
|
export const ttsGeminiAgent = async ({ namedInputs, params, config, }) => {
|
|
7
7
|
const { text } = namedInputs;
|
|
@@ -29,7 +29,7 @@ export const ttsGeminiAgent = async ({ namedInputs, params, config, }) => {
|
|
|
29
29
|
const inlineData = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
|
|
30
30
|
const pcmBase64 = inlineData?.data;
|
|
31
31
|
const mimeType = inlineData?.mimeType;
|
|
32
|
-
if (!pcmBase64)
|
|
32
|
+
if (!pcmBase64 || typeof pcmBase64 !== "string")
|
|
33
33
|
throw new Error("No audio data returned");
|
|
34
34
|
// Extract sample rate from mimeType (e.g., "audio/L16;codec=pcm;rate=24000")
|
|
35
35
|
const rateMatch = mimeType?.match(/rate=(\d+)/);
|
|
@@ -44,6 +44,12 @@ export const ttsGeminiAgent = async ({ namedInputs, params, config, }) => {
|
|
|
44
44
|
};
|
|
45
45
|
}
|
|
46
46
|
GraphAILogger.info(e);
|
|
47
|
+
const reasonDetail = getGenAIErrorReason(e);
|
|
48
|
+
if (reasonDetail && reasonDetail.reason && reasonDetail.reason === "API_KEY_INVALID") {
|
|
49
|
+
throw new Error("Failed to generate tts: 400 Incorrect API key provided with gemini", {
|
|
50
|
+
cause: agentIncorrectAPIKeyError("ttsGeminiAgent", audioAction, audioFileTarget),
|
|
51
|
+
});
|
|
52
|
+
}
|
|
47
53
|
throw new Error("TTS Gemini Error", {
|
|
48
54
|
cause: agentGenerationError("ttsGeminiAgent", audioAction, audioFileTarget),
|
|
49
55
|
});
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { AgentFunction, AgentFunctionInfo } from "graphai";
|
|
2
|
+
import type { KotodamaTTSAgentParams, AgentBufferResult, AgentTextInputs, AgentErrorResult, AgentConfig } from "../types/agent.js";
|
|
3
|
+
export declare const ttsKotodamaAgent: AgentFunction<KotodamaTTSAgentParams, AgentBufferResult | AgentErrorResult, AgentTextInputs, AgentConfig>;
|
|
4
|
+
declare const ttsKotodamaAgentInfo: AgentFunctionInfo;
|
|
5
|
+
export default ttsKotodamaAgentInfo;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { GraphAILogger } from "graphai";
|
|
2
|
+
import { provider2TTSAgent } from "../utils/provider2agent.js";
|
|
3
|
+
import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, audioAction, audioFileTarget } from "../utils/error_cause.js";
|
|
4
|
+
export const ttsKotodamaAgent = async ({ namedInputs, params, config, }) => {
|
|
5
|
+
const { text } = namedInputs;
|
|
6
|
+
const { voice, decoration, suppressError } = params;
|
|
7
|
+
const { apiKey } = config ?? {};
|
|
8
|
+
if (!apiKey) {
|
|
9
|
+
throw new Error("Kotodama API key is required (KOTODAMA_API_KEY)", {
|
|
10
|
+
cause: apiKeyMissingError("ttsKotodamaAgent", audioAction, "KOTODAMA_API_KEY"),
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
const url = "https://tts3.spiral-ai-app.com/api/tts_generate";
|
|
14
|
+
const body = {
|
|
15
|
+
text,
|
|
16
|
+
speaker_id: voice ?? provider2TTSAgent.kotodama.defaultVoice,
|
|
17
|
+
decoration_id: decoration ?? provider2TTSAgent.kotodama.defaultDecoration,
|
|
18
|
+
audio_format: "mp3",
|
|
19
|
+
};
|
|
20
|
+
try {
|
|
21
|
+
const response = await fetch(url, {
|
|
22
|
+
method: "POST",
|
|
23
|
+
headers: {
|
|
24
|
+
"Content-Type": "application/json",
|
|
25
|
+
"X-API-Key": apiKey,
|
|
26
|
+
},
|
|
27
|
+
body: JSON.stringify(body),
|
|
28
|
+
});
|
|
29
|
+
if (!response.ok) {
|
|
30
|
+
if (response.status === 401) {
|
|
31
|
+
throw new Error("Failed to generate audio: 401 Incorrect API key provided with Kotodama", {
|
|
32
|
+
cause: agentIncorrectAPIKeyError("ttsKotodamaAgent", audioAction, audioFileTarget),
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
throw new Error(`Kotodama API error: ${response.status} ${response.statusText}`, {
|
|
36
|
+
cause: agentGenerationError("ttsKotodamaAgent", audioAction, audioFileTarget),
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
// Response is JSON with base64-encoded audio in "audios" array
|
|
40
|
+
const json = await response.json();
|
|
41
|
+
if (!json.audios || !json.audios[0]) {
|
|
42
|
+
throw new Error("TTS Kotodama Error: No audio data in response", {
|
|
43
|
+
cause: agentGenerationError("ttsKotodamaAgent", audioAction, audioFileTarget),
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
const buffer = Buffer.from(json.audios[0], "base64");
|
|
47
|
+
return { buffer };
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
if (suppressError) {
|
|
51
|
+
return {
|
|
52
|
+
error,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
GraphAILogger.error(error);
|
|
56
|
+
if (error && typeof error === "object" && "cause" in error) {
|
|
57
|
+
throw error;
|
|
58
|
+
}
|
|
59
|
+
throw new Error("TTS Kotodama Error", {
|
|
60
|
+
cause: agentGenerationError("ttsKotodamaAgent", audioAction, audioFileTarget),
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
const ttsKotodamaAgentInfo = {
|
|
65
|
+
name: "ttsKotodamaAgent",
|
|
66
|
+
agent: ttsKotodamaAgent,
|
|
67
|
+
mock: ttsKotodamaAgent,
|
|
68
|
+
samples: [],
|
|
69
|
+
description: "Kotodama TTS agent (SpiralAI)",
|
|
70
|
+
category: ["tts"],
|
|
71
|
+
author: "Receptron Team",
|
|
72
|
+
repository: "https://github.com/receptron/mulmocast-cli",
|
|
73
|
+
license: "MIT",
|
|
74
|
+
environmentVariables: ["KOTODAMA_API_KEY"],
|
|
75
|
+
};
|
|
76
|
+
export default ttsKotodamaAgentInfo;
|
|
@@ -34,7 +34,7 @@ export const ttsOpenaiAgent = async ({ namedInputs, params, config, }) => {
|
|
|
34
34
|
}
|
|
35
35
|
GraphAILogger.error(error);
|
|
36
36
|
if (error instanceof AuthenticationError) {
|
|
37
|
-
throw new Error("Failed to generate
|
|
37
|
+
throw new Error("Failed to generate audio: 401 Incorrect API key provided with OpenAI", {
|
|
38
38
|
cause: agentIncorrectAPIKeyError("ttsOpenaiAgent", audioAction, audioFileTarget),
|
|
39
39
|
});
|
|
40
40
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/lib/utils/const.d.ts
CHANGED
package/lib/utils/const.js
CHANGED
|
@@ -197,3 +197,13 @@ export declare const translateApiKeyMissingError: () => {
|
|
|
197
197
|
export declare const hasCause: (err: unknown) => err is Error & {
|
|
198
198
|
cause: unknown;
|
|
199
199
|
};
|
|
200
|
+
type Result<T> = {
|
|
201
|
+
ok: true;
|
|
202
|
+
value: T;
|
|
203
|
+
} | {
|
|
204
|
+
ok: false;
|
|
205
|
+
error: Error;
|
|
206
|
+
};
|
|
207
|
+
export declare function resultify<T>(fn: () => Promise<T>): Promise<Result<T>>;
|
|
208
|
+
export declare const getGenAIErrorReason: (error: Error) => any;
|
|
209
|
+
export {};
|
package/lib/utils/error_cause.js
CHANGED
|
@@ -226,3 +226,25 @@ export const translateApiKeyMissingError = () => {
|
|
|
226
226
|
export const hasCause = (err) => {
|
|
227
227
|
return err instanceof Error && "cause" in err;
|
|
228
228
|
};
|
|
229
|
+
export async function resultify(fn) {
|
|
230
|
+
try {
|
|
231
|
+
return { ok: true, value: await fn() };
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
return { ok: false, error: error };
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
export const getGenAIErrorReason = (error) => {
|
|
238
|
+
try {
|
|
239
|
+
if (error instanceof Error && error.message && error.message[0] === "{") {
|
|
240
|
+
const reasonDetail = JSON.parse(error.message).error.details.find((detail) => detail.reason);
|
|
241
|
+
if (reasonDetail) {
|
|
242
|
+
return reasonDetail;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
catch (__error) {
|
|
247
|
+
// nothing.
|
|
248
|
+
}
|
|
249
|
+
return undefined;
|
|
250
|
+
};
|
package/lib/utils/utils.d.ts
CHANGED
|
@@ -35,4 +35,8 @@ export declare const multiLingualObjectToArray: (multiLingual: MulmoStudioMultiL
|
|
|
35
35
|
}>;
|
|
36
36
|
cacheKey?: string | undefined;
|
|
37
37
|
}[];
|
|
38
|
+
export declare const getAspectRatio: (canvasSize: {
|
|
39
|
+
width: number;
|
|
40
|
+
height: number;
|
|
41
|
+
}, ASPECT_RATIOS: string[]) => string;
|
|
38
42
|
export {};
|
package/lib/utils/utils.js
CHANGED
|
@@ -47,9 +47,8 @@ export const settings2GraphAIConfig = (settings, env) => {
|
|
|
47
47
|
apiKey: getKey("LLM", "OPENAI_API_KEY"),
|
|
48
48
|
baseURL: getKey("LLM", "OPENAI_BASE_URL"),
|
|
49
49
|
},
|
|
50
|
-
|
|
51
|
-
apiKey: getKey("
|
|
52
|
-
baseURL: getKey("TTS", "OPENAI_BASE_URL"),
|
|
50
|
+
anthropicAgent: {
|
|
51
|
+
apiKey: getKey("LLM", "ANTHROPIC_API_TOKEN"),
|
|
53
52
|
},
|
|
54
53
|
imageOpenaiAgent: {
|
|
55
54
|
apiKey: getKey("IMAGE", "OPENAI_API_KEY"),
|
|
@@ -61,18 +60,22 @@ export const settings2GraphAIConfig = (settings, env) => {
|
|
|
61
60
|
imageGenAIAgent: {
|
|
62
61
|
apiKey: getKey("IMAGE", "GEMINI_API_KEY"),
|
|
63
62
|
},
|
|
64
|
-
anthropicAgent: {
|
|
65
|
-
apiKey: getKey("LLM", "ANTHROPIC_API_TOKEN"),
|
|
66
|
-
},
|
|
67
63
|
movieReplicateAgent: {
|
|
68
64
|
apiKey: getKey("MOVIE", "REPLICATE_API_TOKEN"),
|
|
69
65
|
},
|
|
70
66
|
movieGenAIAgent: {
|
|
71
67
|
apiKey: getKey("MOVIE", "GEMINI_API_KEY"),
|
|
72
68
|
},
|
|
69
|
+
ttsOpenaiAgent: {
|
|
70
|
+
apiKey: getKey("TTS", "OPENAI_API_KEY"),
|
|
71
|
+
baseURL: getKey("TTS", "OPENAI_BASE_URL"),
|
|
72
|
+
},
|
|
73
73
|
ttsNijivoiceAgent: {
|
|
74
74
|
apiKey: getKey("TTS", "NIJIVOICE_API_KEY"),
|
|
75
75
|
},
|
|
76
|
+
ttsGoogleAgent: {
|
|
77
|
+
apiKey: getKey("TTS", "GEMINI_API_KEY"),
|
|
78
|
+
},
|
|
76
79
|
ttsGeminiAgent: {
|
|
77
80
|
apiKey: getKey("TTS", "GEMINI_API_KEY"),
|
|
78
81
|
},
|
|
@@ -126,3 +129,12 @@ export const multiLingualObjectToArray = (multiLingual, beats) => {
|
|
|
126
129
|
return { multiLingualTexts: {} };
|
|
127
130
|
});
|
|
128
131
|
};
|
|
132
|
+
export const getAspectRatio = (canvasSize, ASPECT_RATIOS) => {
|
|
133
|
+
const target = canvasSize.width / canvasSize.height;
|
|
134
|
+
return ASPECT_RATIOS.reduce((best, ratio) => {
|
|
135
|
+
const [w, h] = ratio.split(":").map(Number);
|
|
136
|
+
const r = w / h;
|
|
137
|
+
const diff = Math.abs(target - r);
|
|
138
|
+
return diff < best.diff ? { ratio, diff } : best;
|
|
139
|
+
}, { ratio: ASPECT_RATIOS[0], diff: Infinity }).ratio;
|
|
140
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mulmocast",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.7",
|
|
4
4
|
"description": "",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.node.js",
|
|
@@ -75,12 +75,12 @@
|
|
|
75
75
|
"dependencies": {
|
|
76
76
|
"@google-cloud/text-to-speech": "^6.4.0",
|
|
77
77
|
"@google/genai": "^1.30.0",
|
|
78
|
-
"@graphai/anthropic_agent": "^2.0.
|
|
78
|
+
"@graphai/anthropic_agent": "^2.0.12",
|
|
79
79
|
"@graphai/browserless_agent": "^2.0.1",
|
|
80
80
|
"@graphai/gemini_agent": "^2.0.1",
|
|
81
81
|
"@graphai/groq_agent": "^2.0.2",
|
|
82
82
|
"@graphai/input_agents": "^1.0.2",
|
|
83
|
-
"@graphai/openai_agent": "^2.0.
|
|
83
|
+
"@graphai/openai_agent": "^2.0.8",
|
|
84
84
|
"@graphai/stream_agent_filter": "^2.0.2",
|
|
85
85
|
"@graphai/vanilla": "^2.0.12",
|
|
86
86
|
"@graphai/vanilla_node_agents": "^2.0.4",
|
|
@@ -114,7 +114,7 @@
|
|
|
114
114
|
"eslint-config-prettier": "^10.1.8",
|
|
115
115
|
"eslint-plugin-prettier": "^5.5.4",
|
|
116
116
|
"eslint-plugin-sonarjs": "^3.0.5",
|
|
117
|
-
"prettier": "^3.
|
|
117
|
+
"prettier": "^3.7.1",
|
|
118
118
|
"tsx": "^4.20.6",
|
|
119
119
|
"typescript": "^5.9.3",
|
|
120
120
|
"typescript-eslint": "^8.48.0"
|