mulmocast 2.6.23 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/lib/actions/bundle.js +2 -1
- package/lib/actions/pdf.js +4 -1
- package/lib/agents/image_genai_agent.js +5 -2
- package/lib/agents/image_openai_agent.js +2 -1
- package/lib/agents/image_replicate_agent.js +2 -1
- package/lib/agents/lipsync_replicate_agent.js +2 -1
- package/lib/agents/media_mock_agent.js +2 -1
- package/lib/agents/movie_genai_agent.js +17 -3
- package/lib/agents/movie_replicate_agent.js +2 -1
- package/lib/agents/sound_effect_replicate_agent.js +6 -2
- package/lib/agents/tts_elevenlabs_agent.js +4 -3
- package/lib/agents/tts_gemini_agent.js +4 -1
- package/lib/agents/tts_kotodama_agent.js +7 -3
- package/lib/cli/commands/audio/builder.d.ts +2 -14
- package/lib/cli/commands/audio/builder.js +2 -2
- package/lib/cli/commands/audio/handler.d.ts +2 -0
- package/lib/cli/commands/audio/handler.js +5 -1
- package/lib/cli/commands/image/builder.d.ts +2 -14
- package/lib/cli/commands/image/builder.js +2 -2
- package/lib/cli/commands/image/handler.d.ts +2 -0
- package/lib/cli/commands/image/handler.js +5 -1
- package/lib/cli/commands/movie/builder.d.ts +2 -14
- package/lib/cli/commands/movie/builder.js +2 -2
- package/lib/cli/commands/movie/handler.d.ts +2 -0
- package/lib/cli/commands/movie/handler.js +5 -1
- package/lib/cli/commands/pdf/builder.d.ts +2 -14
- package/lib/cli/commands/pdf/builder.js +2 -2
- package/lib/cli/commands/pdf/handler.d.ts +2 -0
- package/lib/cli/commands/pdf/handler.js +5 -1
- package/lib/cli/commands/translate/builder.d.ts +3 -15
- package/lib/cli/commands/translate/builder.js +2 -2
- package/lib/cli/commands/translate/handler.d.ts +2 -0
- package/lib/cli/commands/translate/handler.js +5 -1
- package/lib/cli/common.d.ts +5 -0
- package/lib/cli/common.js +13 -0
- package/lib/cli/helpers.d.ts +2 -0
- package/lib/cli/helpers.js +19 -0
- package/lib/index.common.d.ts +2 -0
- package/lib/index.common.js +2 -0
- package/lib/methods/mulmo_media_source.js +9 -13
- package/lib/methods/mulmo_presentation_style.d.ts +2 -0
- package/lib/methods/mulmo_presentation_style.js +9 -6
- package/lib/types/index.d.ts +1 -0
- package/lib/types/index.js +1 -0
- package/lib/types/provider2agent.d.ts +17 -0
- package/lib/types/provider2agent.js +79 -0
- package/lib/types/usage.d.ts +21 -0
- package/lib/utils/estimate_usage.d.ts +11 -0
- package/lib/utils/estimate_usage.js +332 -0
- package/lib/utils/estimate_usage_format.d.ts +2 -0
- package/lib/utils/estimate_usage_format.js +64 -0
- package/lib/utils/fetch.d.ts +28 -0
- package/lib/utils/fetch.js +52 -0
- package/lib/utils/file.js +2 -1
- package/lib/utils/image_plugins/bg_image_util.js +7 -20
- package/lib/utils/openai_client.js +6 -0
- package/lib/utils/replicate_usage.d.ts +4 -1
- package/lib/utils/replicate_usage.js +25 -5
- package/lib/utils/sdk_timeout.d.ts +3 -0
- package/lib/utils/sdk_timeout.js +7 -0
- package/lib/utils/zip.js +2 -2
- package/package.json +13 -12
package/README.md
CHANGED
|
@@ -419,6 +419,30 @@ Payload shape:
|
|
|
419
419
|
|
|
420
420
|
`byModel` groups by `provider:model` (different models have different rate cards, so summing across them isn't meaningful). The full per-call snapshot is also included so a billing layer can apply its own grouping. See [docs/api.md § Usage tracking](./docs/api.md#usage-tracking) for the per-agent field-population matrix and the programmatic (library) API.
|
|
421
421
|
|
|
422
|
+
### Usage estimation (before running anything)
|
|
423
|
+
|
|
424
|
+
Every generation command accepts `--estimate`: it prints what the command would consume — scoped to that command's pipeline — and exits without generating:
|
|
425
|
+
|
|
426
|
+
```bash
|
|
427
|
+
mulmo movie script.json --estimate # table per provider:model + total cost
|
|
428
|
+
mulmo audio script.json --estimate --json # raw UsageEstimate[] records (pipe-friendly)
|
|
429
|
+
```
|
|
430
|
+
|
|
431
|
+
`mulmo audio --estimate` covers TTS (plus translation when `-l` differs from the script language); `mulmo images` / `mulmo pdf` cover image, htmlImage, movie, sound-effect, lip-sync and reference-media generation; `mulmo movie` covers everything; `mulmo translate` only translation. Values marked `~` are heuristic estimates; unmarked values are exact for the given script.
|
|
432
|
+
|
|
433
|
+
The library API can estimate the same thing — per beat, per process — without calling any API:
|
|
434
|
+
|
|
435
|
+
```typescript
|
|
436
|
+
import { estimateUsage } from "mulmocast";
|
|
437
|
+
|
|
438
|
+
const estimates = estimateUsage(script, { targetLangs: ["ja"] });
|
|
439
|
+
// [{ process: "tts", beatIndex: 0, provider: "openai", model: "gpt-4o-mini-tts",
|
|
440
|
+
// inputChars: { value: 50, precision: "exact" }, outputTokens: { value: 150, precision: "estimated" },
|
|
441
|
+
// costUSD: 0.0019, pricingAsOf: "2026-07-03" }, ...]
|
|
442
|
+
```
|
|
443
|
+
|
|
444
|
+
Each metric carries a `precision` flag: `"exact"` for values that are deterministic from the script (TTS character counts, tokenized OpenAI prompts, fixed image-output token tables) and `"estimated"` for heuristics (LLM output length, speech duration derived from text). `costUSD` appears when pricing data exists in `modelPricing` (`src/types/provider2agent.ts`), where every price records the date it was last verified. See [docs/api.md § Pre-run estimation](./docs/api.md#pre-run-estimation-estimateusage).
|
|
445
|
+
|
|
422
446
|
## Cache and Re-run
|
|
423
447
|
When running the same `mulmo` command multiple times, previously generated files are treated as cache. For example, audio or image files will not be regenerated if they already exist.
|
|
424
448
|
|
package/lib/actions/bundle.js
CHANGED
|
@@ -7,8 +7,9 @@ import { ZipBuilder } from "../utils/zip.js";
|
|
|
7
7
|
import { bundleTargetLang } from "../types/const.js";
|
|
8
8
|
import { createSilentAudio } from "../utils/ffmpeg_utils.js";
|
|
9
9
|
import { silentMp3 } from "../utils/context.js";
|
|
10
|
+
import { safeFetch, FETCH_MEDIA_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
10
11
|
const downloadFile = async (url, destPath) => {
|
|
11
|
-
const response = await
|
|
12
|
+
const response = await safeFetch(url, {}, FETCH_MEDIA_DOWNLOAD_TIMEOUT_MS);
|
|
12
13
|
if (!response.ok) {
|
|
13
14
|
throw new Error(`Failed to download file from ${url}: ${response.statusText}`);
|
|
14
15
|
}
|
package/lib/actions/pdf.js
CHANGED
|
@@ -6,6 +6,7 @@ import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
|
6
6
|
import { localizedText, isHttp } from "../utils/utils.js";
|
|
7
7
|
import { getOutputPdfFilePath, writingMessage, getHTMLFile, mulmoCreditPath } from "../utils/file.js";
|
|
8
8
|
import { interpolate } from "../utils/html_render.js";
|
|
9
|
+
import { safeFetch, FETCH_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
9
10
|
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
10
11
|
const isCI = process.env.CI === "true";
|
|
11
12
|
const getPdfSize = (pdfSize) => {
|
|
@@ -13,7 +14,9 @@ const getPdfSize = (pdfSize) => {
|
|
|
13
14
|
};
|
|
14
15
|
const loadImage = async (imagePath, index) => {
|
|
15
16
|
try {
|
|
16
|
-
const imageData = isHttp(imagePath)
|
|
17
|
+
const imageData = isHttp(imagePath)
|
|
18
|
+
? Buffer.from(await (await safeFetch(imagePath, {}, FETCH_DOWNLOAD_TIMEOUT_MS)).arrayBuffer())
|
|
19
|
+
: fs.readFileSync(imagePath);
|
|
17
20
|
const ext = path.extname(imagePath).toLowerCase().replace(".", "");
|
|
18
21
|
const mimeType = ext === "jpg" ? "jpeg" : ext;
|
|
19
22
|
return `data:image/${mimeType};base64,${imageData.toString("base64")}`;
|
|
@@ -5,6 +5,9 @@ import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, ag
|
|
|
5
5
|
import { getAspectRatio } from "../utils/utils.js";
|
|
6
6
|
import { ASPECT_RATIOS, PRO_ASPECT_RATIOS } from "../types/const.js";
|
|
7
7
|
import { GoogleGenAI, PersonGeneration } from "@google/genai";
|
|
8
|
+
// Per-request timeout so a stalled GenAI image call rejects (and GraphAI retry
|
|
9
|
+
// can recover) instead of hanging. Generous vs. normal generation time.
|
|
10
|
+
const GENAI_REQUEST_TIMEOUT_MS = 120_000;
|
|
8
11
|
const isDeprecatedGoogleImageModel = (model) => model in deprecatedGoogleImageModelHints;
|
|
9
12
|
export const buildDeprecatedGoogleImageModelMessage = (model) => {
|
|
10
13
|
if (!isDeprecatedGoogleImageModel(model))
|
|
@@ -86,7 +89,7 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
86
89
|
if (vertexAIGlobalOnlyImageModels.has(model) && location !== "global") {
|
|
87
90
|
GraphAILogger.warn(`imageGenAIAgent: model "${model}" on Vertex AI is only available in location "global", but got "${location}". Set imageParams.vertexai_location to "global".`);
|
|
88
91
|
}
|
|
89
|
-
return new GoogleGenAI({ vertexai: true, project: params.vertexai_project, location });
|
|
92
|
+
return new GoogleGenAI({ vertexai: true, project: params.vertexai_project, location, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
|
|
90
93
|
})()
|
|
91
94
|
: (() => {
|
|
92
95
|
if (!apiKey) {
|
|
@@ -94,7 +97,7 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
94
97
|
cause: apiKeyMissingError("imageGenAIAgent", imageAction, "GEMINI_API_KEY"),
|
|
95
98
|
});
|
|
96
99
|
}
|
|
97
|
-
return new GoogleGenAI({ apiKey });
|
|
100
|
+
return new GoogleGenAI({ apiKey, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
|
|
98
101
|
})();
|
|
99
102
|
if (model === "gemini-2.5-flash-image" || model === "gemini-3.1-flash-image-preview" || model === "gemini-3-pro-image-preview") {
|
|
100
103
|
const contentParams = (() => {
|
|
@@ -3,6 +3,7 @@ import path from "path";
|
|
|
3
3
|
import { GraphAILogger } from "graphai";
|
|
4
4
|
import { toFile, AuthenticationError, RateLimitError, APIError } from "openai";
|
|
5
5
|
import { createOpenAIClient } from "../utils/openai_client.js";
|
|
6
|
+
import { safeFetch, FETCH_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
6
7
|
import { provider2ImageAgent, gptImages, deprecatedOpenAIImageModelHints } from "../types/provider2agent.js";
|
|
7
8
|
import { apiKeyMissingError, agentGenerationError, openAIAgentGenerationError, agentIncorrectAPIKeyError, agentAPIRateLimitError, agentInvalidResponseError, imageAction, imageFileTarget, unsupportedModelTarget, } from "../utils/error_cause.js";
|
|
8
9
|
const isDeprecatedOpenAIImageModel = (model) => model in deprecatedOpenAIImageModelHints;
|
|
@@ -142,7 +143,7 @@ export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
|
|
|
142
143
|
return { buffer: Buffer.from(image_base64, "base64"), usage };
|
|
143
144
|
}
|
|
144
145
|
// URL response handling (legacy OpenAI image API response format)
|
|
145
|
-
const res = await
|
|
146
|
+
const res = await safeFetch(url, {}, FETCH_DOWNLOAD_TIMEOUT_MS);
|
|
146
147
|
if (!res.ok) {
|
|
147
148
|
throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`, {
|
|
148
149
|
cause: agentGenerationError("imageOpenaiAgent", imageAction, imageFileTarget),
|
|
@@ -5,6 +5,7 @@ import { getAspectRatio } from "./movie_replicate_agent.js";
|
|
|
5
5
|
import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, agentInvalidResponseError, imageAction, imageFileTarget, hasCause, } from "../utils/error_cause.js";
|
|
6
6
|
import { provider2ImageAgent } from "../types/provider2agent.js";
|
|
7
7
|
import { runReplicateWithMetrics } from "../utils/replicate_usage.js";
|
|
8
|
+
import { safeFetch, FETCH_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
8
9
|
// Replicate image models return one of: FileOutput (object with url() method),
|
|
9
10
|
// Array<FileOutput>, string URL, or { url: string }. Normalize to URL string/URL.
|
|
10
11
|
const extractImageUrl = (output) => {
|
|
@@ -52,7 +53,7 @@ export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
|
|
|
52
53
|
cause: agentInvalidResponseError("imageReplicateAgent", imageAction, imageFileTarget),
|
|
53
54
|
});
|
|
54
55
|
}
|
|
55
|
-
const imageResponse = await
|
|
56
|
+
const imageResponse = await safeFetch(imageUrl, {}, FETCH_DOWNLOAD_TIMEOUT_MS);
|
|
56
57
|
if (!imageResponse.ok) {
|
|
57
58
|
throw new Error(`Error downloading image: ${imageResponse.status} - ${imageResponse.statusText}`, {
|
|
58
59
|
cause: agentGenerationError("imageReplicateAgent", imageAction, imageFileTarget),
|
|
@@ -4,6 +4,7 @@ import Replicate from "replicate";
|
|
|
4
4
|
import { provider2LipSyncAgent } from "../types/provider2agent.js";
|
|
5
5
|
import { apiKeyMissingError, agentGenerationError, agentFileNotExistError, imageAction, movieFileTarget, audioFileTarget, hasCause, } from "../utils/error_cause.js";
|
|
6
6
|
import { runReplicateWithMetrics } from "../utils/replicate_usage.js";
|
|
7
|
+
import { safeFetch, FETCH_MEDIA_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
7
8
|
export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) => {
|
|
8
9
|
const { movieFile, audioFile, imageFile } = namedInputs;
|
|
9
10
|
const apiKey = config?.apiKey;
|
|
@@ -64,7 +65,7 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
|
|
|
64
65
|
const { output, predictSec } = await runReplicateWithMetrics(replicate, model_identifier, input);
|
|
65
66
|
if (output && typeof output === "object" && "url" in output) {
|
|
66
67
|
const videoUrl = output.url();
|
|
67
|
-
const videoResponse = await
|
|
68
|
+
const videoResponse = await safeFetch(videoUrl, {}, FETCH_MEDIA_DOWNLOAD_TIMEOUT_MS);
|
|
68
69
|
if (!videoResponse.ok) {
|
|
69
70
|
throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`, {
|
|
70
71
|
cause: agentGenerationError("lipSyncReplicateAgent", imageAction, movieFileTarget),
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
2
|
import fs from "fs";
|
|
3
3
|
import { silent60secPath, mulmoCreditPath } from "../utils/file.js";
|
|
4
|
+
import { safeFetch, FETCH_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
4
5
|
export const mediaMockAgent = async ({ namedInputs }) => {
|
|
5
6
|
if (namedInputs.media === "audio") {
|
|
6
7
|
const buffer = fs.readFileSync(silent60secPath());
|
|
@@ -12,7 +13,7 @@ export const mediaMockAgent = async ({ namedInputs }) => {
|
|
|
12
13
|
}
|
|
13
14
|
if (namedInputs.media === "movie") {
|
|
14
15
|
const url = "https://github.com/receptron/mulmocast-media/raw/refs/heads/main/test/pingpong.mov";
|
|
15
|
-
const res = await
|
|
16
|
+
const res = await safeFetch(url, {}, FETCH_DOWNLOAD_TIMEOUT_MS);
|
|
16
17
|
if (!res.ok) {
|
|
17
18
|
throw new Error(`Failed to fetch: ${res.status} ${res.statusText}`);
|
|
18
19
|
}
|
|
@@ -6,9 +6,19 @@ import { getAspectRatio } from "../utils/utils.js";
|
|
|
6
6
|
import { ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
|
|
7
7
|
import { ASPECT_RATIOS } from "../types/const.js";
|
|
8
8
|
import { getModelDuration, provider2MovieAgent, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS } from "../types/provider2agent.js";
|
|
9
|
+
// Per-request timeout so a stalled GenAI video API call rejects instead of hanging.
|
|
10
|
+
const GENAI_REQUEST_TIMEOUT_MS = 120_000;
|
|
11
|
+
// Wall-clock cap on the long-running video operation poll loop (Veo runs minutes).
|
|
12
|
+
const VIDEO_POLL_TIMEOUT_MS = 1_200_000;
|
|
9
13
|
const pollUntilDone = async (ai, operation) => {
|
|
10
14
|
const response = { operation };
|
|
15
|
+
const deadline = Date.now() + VIDEO_POLL_TIMEOUT_MS;
|
|
11
16
|
while (!response.operation.done) {
|
|
17
|
+
if (Date.now() > deadline) {
|
|
18
|
+
throw new Error(`Video generation did not complete within ${VIDEO_POLL_TIMEOUT_MS}ms`, {
|
|
19
|
+
cause: agentGenerationError("movieGenAIAgent", imageAction, movieFileTarget),
|
|
20
|
+
});
|
|
21
|
+
}
|
|
12
22
|
await sleep(5000);
|
|
13
23
|
response.operation = await ai.operations.getVideosOperation(response);
|
|
14
24
|
}
|
|
@@ -190,6 +200,7 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
190
200
|
vertexai: true,
|
|
191
201
|
project: params.vertexai_project,
|
|
192
202
|
location: params.vertexai_location ?? "us-central1",
|
|
203
|
+
httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS },
|
|
193
204
|
})
|
|
194
205
|
: (() => {
|
|
195
206
|
if (!apiKey) {
|
|
@@ -197,7 +208,7 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
197
208
|
cause: apiKeyMissingError("movieGenAIAgent", imageAction, "GEMINI_API_KEY"),
|
|
198
209
|
});
|
|
199
210
|
}
|
|
200
|
-
return new GoogleGenAI({ apiKey });
|
|
211
|
+
return new GoogleGenAI({ apiKey, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
|
|
201
212
|
})();
|
|
202
213
|
// Veo 3.1: Video extension mode for videos longer than 8s
|
|
203
214
|
if (model === "veo-3.1-generate-preview" && requestedDuration > 8 && params.canvasSize) {
|
|
@@ -207,11 +218,14 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
207
218
|
return generateStandardVideo(ai, model, prompt, aspectRatio, imagePath, lastFrameImagePath, referenceImages, duration, movieFile, isVertexAI);
|
|
208
219
|
}
|
|
209
220
|
catch (error) {
|
|
210
|
-
GraphAILogger.info("Failed to generate movie:", error.message);
|
|
211
221
|
if (hasCause(error) && error.cause) {
|
|
212
222
|
throw error;
|
|
213
223
|
}
|
|
214
|
-
|
|
224
|
+
GraphAILogger.info("Failed to generate movie:", error.message);
|
|
225
|
+
// Preserve the underlying message (e.g. a timeout/abort deadline) instead of
|
|
226
|
+
// collapsing every failure to a static label. (Same template as #1452.)
|
|
227
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
228
|
+
throw new Error(`Failed to generate movie with Google GenAI: ${detail}`, {
|
|
215
229
|
cause: agentGenerationError("movieGenAIAgent", imageAction, movieFileTarget),
|
|
216
230
|
});
|
|
217
231
|
}
|
|
@@ -4,6 +4,7 @@ import Replicate from "replicate";
|
|
|
4
4
|
import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, hasCause, imageAction, movieFileTarget, videoDurationTarget, unsupportedModelTarget, } from "../utils/error_cause.js";
|
|
5
5
|
import { provider2MovieAgent, getModelDuration, AUDIO_MODE_OPTIONAL, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS } from "../types/provider2agent.js";
|
|
6
6
|
import { runReplicateWithMetrics } from "../utils/replicate_usage.js";
|
|
7
|
+
import { safeFetch, FETCH_MEDIA_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
7
8
|
function replicate_get_videoUrl(output) {
|
|
8
9
|
if (typeof output === "string")
|
|
9
10
|
return output;
|
|
@@ -99,7 +100,7 @@ async function generateMovie(model, apiKey, prompt, imagePath, lastFrameImagePat
|
|
|
99
100
|
// Some models return a FileOutput object with a url() method; others return a plain string URL.
|
|
100
101
|
const videoUrl = replicate_get_videoUrl(output);
|
|
101
102
|
if (videoUrl) {
|
|
102
|
-
const videoResponse = await
|
|
103
|
+
const videoResponse = await safeFetch(videoUrl, {}, FETCH_MEDIA_DOWNLOAD_TIMEOUT_MS);
|
|
103
104
|
if (!videoResponse.ok) {
|
|
104
105
|
throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`, {
|
|
105
106
|
cause: agentGenerationError("movieReplicateAgent", imageAction, movieFileTarget),
|
|
@@ -4,6 +4,7 @@ import Replicate from "replicate";
|
|
|
4
4
|
import { provider2SoundEffectAgent } from "../types/provider2agent.js";
|
|
5
5
|
import { apiKeyMissingError, agentGenerationError, imageAction, movieFileTarget, hasCause } from "../utils/error_cause.js";
|
|
6
6
|
import { runReplicateWithMetrics } from "../utils/replicate_usage.js";
|
|
7
|
+
import { safeFetch, FETCH_MEDIA_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
7
8
|
export const soundEffectReplicateAgent = async ({ namedInputs, params, config }) => {
|
|
8
9
|
const { prompt, movieFile } = namedInputs;
|
|
9
10
|
const apiKey = config?.apiKey;
|
|
@@ -32,7 +33,7 @@ export const soundEffectReplicateAgent = async ({ namedInputs, params, config })
|
|
|
32
33
|
const { output, predictSec } = await runReplicateWithMetrics(replicate, model_identifier, input);
|
|
33
34
|
if (output && typeof output === "object" && "url" in output) {
|
|
34
35
|
const videoUrl = output.url();
|
|
35
|
-
const videoResponse = await
|
|
36
|
+
const videoResponse = await safeFetch(videoUrl, {}, FETCH_MEDIA_DOWNLOAD_TIMEOUT_MS);
|
|
36
37
|
if (!videoResponse.ok) {
|
|
37
38
|
throw new Error(`Error downloading video: ${videoResponse.status} - ${videoResponse.statusText}`, {
|
|
38
39
|
cause: agentGenerationError("soundEffectReplicateAgent", imageAction, movieFileTarget),
|
|
@@ -49,7 +50,10 @@ export const soundEffectReplicateAgent = async ({ namedInputs, params, config })
|
|
|
49
50
|
if (hasCause(error) && error.cause) {
|
|
50
51
|
throw error;
|
|
51
52
|
}
|
|
52
|
-
|
|
53
|
+
// Preserve the underlying message (e.g. a safeFetch timeout) rather than
|
|
54
|
+
// collapsing every failure to a static label. (Same template as #1452.)
|
|
55
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
56
|
+
throw new Error(`Failed to generate sound effect with Replicate: ${detail}`, {
|
|
53
57
|
cause: agentGenerationError("soundEffectReplicateAgent", imageAction, movieFileTarget),
|
|
54
58
|
});
|
|
55
59
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
2
|
import { provider2TTSAgent } from "../types/provider2agent.js";
|
|
3
3
|
import { apiKeyMissingError, agentVoiceLimitReachedError, agentIncorrectAPIKeyError, agentGenerationError, audioAction, audioFileTarget, } from "../utils/error_cause.js";
|
|
4
|
+
import { safeFetch, FETCH_API_TIMEOUT_MS } from "../utils/fetch.js";
|
|
4
5
|
export const ttsElevenlabsAgent = async ({ namedInputs, params, config, }) => {
|
|
5
6
|
const { text } = namedInputs;
|
|
6
7
|
const { voice, model, stability, similarityBoost, speed, suppressError } = params;
|
|
@@ -27,7 +28,7 @@ export const ttsElevenlabsAgent = async ({ namedInputs, params, config, }) => {
|
|
|
27
28
|
GraphAILogger.log("ElevenLabs TTS options", requestBody);
|
|
28
29
|
const response = await (async () => {
|
|
29
30
|
try {
|
|
30
|
-
return await
|
|
31
|
+
return await safeFetch(`https://api.elevenlabs.io/v1/text-to-speech/${voice}`, {
|
|
31
32
|
method: "POST",
|
|
32
33
|
headers: {
|
|
33
34
|
Accept: "audio/mpeg",
|
|
@@ -35,7 +36,7 @@ export const ttsElevenlabsAgent = async ({ namedInputs, params, config, }) => {
|
|
|
35
36
|
"xi-api-key": apiKey,
|
|
36
37
|
},
|
|
37
38
|
body: JSON.stringify(requestBody),
|
|
38
|
-
});
|
|
39
|
+
}, FETCH_API_TIMEOUT_MS);
|
|
39
40
|
}
|
|
40
41
|
catch (e) {
|
|
41
42
|
if (suppressError) {
|
|
@@ -64,7 +65,7 @@ export const ttsElevenlabsAgent = async ({ namedInputs, params, config, }) => {
|
|
|
64
65
|
});
|
|
65
66
|
}
|
|
66
67
|
const errorDetail = await response.json();
|
|
67
|
-
if (errorDetail.detail
|
|
68
|
+
if (errorDetail.detail?.status === "voice_limit_reached") {
|
|
68
69
|
throw new Error("Failed to generate audio: 400 You have reached your maximum amount of custom voices with ElevenLabs", {
|
|
69
70
|
cause: agentVoiceLimitReachedError("ttsElevenlabsAgent", audioAction, audioFileTarget),
|
|
70
71
|
});
|
|
@@ -3,6 +3,9 @@ import { GoogleGenAI } from "@google/genai";
|
|
|
3
3
|
import { provider2TTSAgent } from "../types/provider2agent.js";
|
|
4
4
|
import { agentIncorrectAPIKeyError, apiKeyMissingError, agentGenerationError, audioAction, audioFileTarget, getGenAIErrorReason, } from "../utils/error_cause.js";
|
|
5
5
|
import { pcmToMp3 } from "../utils/ffmpeg_utils.js";
|
|
6
|
+
// Per-request timeout so a stalled GenAI TTS call rejects (and GraphAI retry can
|
|
7
|
+
// recover) instead of hanging. Generous vs. normal generation time.
|
|
8
|
+
const GENAI_REQUEST_TIMEOUT_MS = 120_000;
|
|
6
9
|
const getPrompt = (text, instructions) => {
|
|
7
10
|
// https://ai.google.dev/gemini-api/docs/speech-generation?hl=ja#controllable
|
|
8
11
|
if (instructions) {
|
|
@@ -21,7 +24,7 @@ export const ttsGeminiAgent = async ({ namedInputs, params, config, }) => {
|
|
|
21
24
|
}
|
|
22
25
|
const geminiResult = await (async () => {
|
|
23
26
|
try {
|
|
24
|
-
const ai = new GoogleGenAI({ apiKey });
|
|
27
|
+
const ai = new GoogleGenAI({ apiKey, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
|
|
25
28
|
const response = await ai.models.generateContent({
|
|
26
29
|
model: model ?? provider2TTSAgent.gemini.defaultModel,
|
|
27
30
|
contents: [{ parts: [{ text: getPrompt(text, instructions) }] }],
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { GraphAILogger } from "graphai";
|
|
2
2
|
import { provider2TTSAgent } from "../types/provider2agent.js";
|
|
3
3
|
import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, audioAction, audioFileTarget } from "../utils/error_cause.js";
|
|
4
|
+
import { safeFetch, FETCH_API_TIMEOUT_MS } from "../utils/fetch.js";
|
|
4
5
|
export const ttsKotodamaAgent = async ({ namedInputs, params, config, }) => {
|
|
5
6
|
const { text } = namedInputs;
|
|
6
7
|
const { voice, decoration, suppressError } = params;
|
|
@@ -18,14 +19,14 @@ export const ttsKotodamaAgent = async ({ namedInputs, params, config, }) => {
|
|
|
18
19
|
audio_format: "mp3",
|
|
19
20
|
};
|
|
20
21
|
try {
|
|
21
|
-
const response = await
|
|
22
|
+
const response = await safeFetch(url, {
|
|
22
23
|
method: "POST",
|
|
23
24
|
headers: {
|
|
24
25
|
"Content-Type": "application/json",
|
|
25
26
|
"X-API-Key": apiKey,
|
|
26
27
|
},
|
|
27
28
|
body: JSON.stringify(body),
|
|
28
|
-
});
|
|
29
|
+
}, FETCH_API_TIMEOUT_MS);
|
|
29
30
|
if (!response.ok) {
|
|
30
31
|
if (response.status === 401) {
|
|
31
32
|
throw new Error("Failed to generate audio: 401 Incorrect API key provided with Kotodama", {
|
|
@@ -64,7 +65,10 @@ export const ttsKotodamaAgent = async ({ namedInputs, params, config, }) => {
|
|
|
64
65
|
if (error && typeof error === "object" && "cause" in error) {
|
|
65
66
|
throw error;
|
|
66
67
|
}
|
|
67
|
-
|
|
68
|
+
// Preserve the underlying message (e.g. a safeFetch timeout) rather than
|
|
69
|
+
// collapsing every failure to a static label. (Same template as #1452.)
|
|
70
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
71
|
+
throw new Error(`TTS Kotodama Error: ${detail}`, {
|
|
68
72
|
cause: agentGenerationError("ttsKotodamaAgent", audioAction, audioFileTarget),
|
|
69
73
|
});
|
|
70
74
|
}
|
|
@@ -1,20 +1,8 @@
|
|
|
1
1
|
import type { Argv } from "yargs";
|
|
2
2
|
export declare const builder: (yargs: Argv) => Argv<{
|
|
3
|
-
|
|
3
|
+
estimate: boolean;
|
|
4
4
|
} & {
|
|
5
|
-
|
|
6
|
-
} & {
|
|
7
|
-
l: string | undefined;
|
|
8
|
-
} & {
|
|
9
|
-
f: boolean;
|
|
10
|
-
} & {
|
|
11
|
-
g: boolean;
|
|
12
|
-
} & {
|
|
13
|
-
backup: boolean;
|
|
14
|
-
} & {
|
|
15
|
-
p: string | undefined;
|
|
16
|
-
} & {
|
|
17
|
-
file: string | undefined;
|
|
5
|
+
json: boolean;
|
|
18
6
|
} & {
|
|
19
7
|
a: string | undefined;
|
|
20
8
|
}>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { commonOptions } from "../../common.js";
|
|
2
|
-
export const builder = (yargs) => commonOptions(yargs).option("a", {
|
|
1
|
+
import { commonOptions, estimateOptions } from "../../common.js";
|
|
2
|
+
export const builder = (yargs) => estimateOptions(commonOptions(yargs)).option("a", {
|
|
3
3
|
alias: "audiodir",
|
|
4
4
|
describe: "Audio output directory",
|
|
5
5
|
type: "string",
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { audio } from "../../../actions/index.js";
|
|
2
|
-
import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded } from "../../helpers.js";
|
|
2
|
+
import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded, printUsageEstimate } from "../../helpers.js";
|
|
3
3
|
export const handler = async (argv) => {
|
|
4
4
|
const context = await initializeContext(argv);
|
|
5
5
|
if (!context) {
|
|
6
6
|
process.exit(1);
|
|
7
7
|
}
|
|
8
|
+
if (argv.estimate) {
|
|
9
|
+
printUsageEstimate(context, "audio", argv.json);
|
|
10
|
+
return;
|
|
11
|
+
}
|
|
8
12
|
await runTranslateIfNeeded(context);
|
|
9
13
|
await audio(context);
|
|
10
14
|
dumpUsageIfRequested(context);
|
|
@@ -1,20 +1,8 @@
|
|
|
1
1
|
import type { Argv } from "yargs";
|
|
2
2
|
export declare const builder: (yargs: Argv) => Argv<{
|
|
3
|
-
|
|
3
|
+
estimate: boolean;
|
|
4
4
|
} & {
|
|
5
|
-
|
|
6
|
-
} & {
|
|
7
|
-
l: string | undefined;
|
|
8
|
-
} & {
|
|
9
|
-
f: boolean;
|
|
10
|
-
} & {
|
|
11
|
-
g: boolean;
|
|
12
|
-
} & {
|
|
13
|
-
backup: boolean;
|
|
14
|
-
} & {
|
|
15
|
-
p: string | undefined;
|
|
16
|
-
} & {
|
|
17
|
-
file: string | undefined;
|
|
5
|
+
json: boolean;
|
|
18
6
|
} & {
|
|
19
7
|
i: string | undefined;
|
|
20
8
|
}>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { commonOptions } from "../../common.js";
|
|
2
|
-
export const builder = (yargs) => commonOptions(yargs).option("i", {
|
|
1
|
+
import { commonOptions, estimateOptions } from "../../common.js";
|
|
2
|
+
export const builder = (yargs) => estimateOptions(commonOptions(yargs)).option("i", {
|
|
3
3
|
alias: "imagedir",
|
|
4
4
|
describe: "Image output directory",
|
|
5
5
|
type: "string",
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { images } from "../../../actions/index.js";
|
|
2
|
-
import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded } from "../../helpers.js";
|
|
2
|
+
import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded, printUsageEstimate } from "../../helpers.js";
|
|
3
3
|
export const handler = async (argv) => {
|
|
4
4
|
const context = await initializeContext(argv);
|
|
5
5
|
if (!context) {
|
|
6
6
|
process.exit(1);
|
|
7
7
|
}
|
|
8
|
+
if (argv.estimate) {
|
|
9
|
+
printUsageEstimate(context, "images", argv.json);
|
|
10
|
+
return;
|
|
11
|
+
}
|
|
8
12
|
await runTranslateIfNeeded(context);
|
|
9
13
|
await images(context);
|
|
10
14
|
dumpUsageIfRequested(context);
|
|
@@ -1,20 +1,8 @@
|
|
|
1
1
|
import type { Argv } from "yargs";
|
|
2
2
|
export declare const builder: (yargs: Argv) => Argv<{
|
|
3
|
-
|
|
3
|
+
estimate: boolean;
|
|
4
4
|
} & {
|
|
5
|
-
|
|
6
|
-
} & {
|
|
7
|
-
l: string | undefined;
|
|
8
|
-
} & {
|
|
9
|
-
f: boolean;
|
|
10
|
-
} & {
|
|
11
|
-
g: boolean;
|
|
12
|
-
} & {
|
|
13
|
-
backup: boolean;
|
|
14
|
-
} & {
|
|
15
|
-
p: string | undefined;
|
|
16
|
-
} & {
|
|
17
|
-
file: string | undefined;
|
|
5
|
+
json: boolean;
|
|
18
6
|
} & {
|
|
19
7
|
a: string | undefined;
|
|
20
8
|
} & {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { commonOptions } from "../../common.js";
|
|
1
|
+
import { commonOptions, estimateOptions } from "../../common.js";
|
|
2
2
|
import { languages } from "../../../types/const.js";
|
|
3
|
-
export const builder = (yargs) => commonOptions(yargs)
|
|
3
|
+
export const builder = (yargs) => estimateOptions(commonOptions(yargs))
|
|
4
4
|
.option("a", {
|
|
5
5
|
alias: "audiodir",
|
|
6
6
|
describe: "Audio output directory",
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { audio, images, movie, captions } from "../../../actions/index.js";
|
|
2
|
-
import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded } from "../../helpers.js";
|
|
2
|
+
import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded, printUsageEstimate } from "../../helpers.js";
|
|
3
3
|
export const handler = async (argv) => {
|
|
4
4
|
const context = await initializeContext(argv);
|
|
5
5
|
if (!context) {
|
|
6
6
|
process.exit(1);
|
|
7
7
|
}
|
|
8
|
+
if (argv.estimate) {
|
|
9
|
+
printUsageEstimate(context, "movie", argv.json);
|
|
10
|
+
return;
|
|
11
|
+
}
|
|
8
12
|
await runTranslateIfNeeded(context, true);
|
|
9
13
|
await audio(context).then(images).then(captions).then(movie);
|
|
10
14
|
dumpUsageIfRequested(context);
|
|
@@ -1,20 +1,8 @@
|
|
|
1
1
|
import type { Argv } from "yargs";
|
|
2
2
|
export declare const builder: (yargs: Argv) => Argv<{
|
|
3
|
-
|
|
3
|
+
estimate: boolean;
|
|
4
4
|
} & {
|
|
5
|
-
|
|
6
|
-
} & {
|
|
7
|
-
l: string | undefined;
|
|
8
|
-
} & {
|
|
9
|
-
f: boolean;
|
|
10
|
-
} & {
|
|
11
|
-
g: boolean;
|
|
12
|
-
} & {
|
|
13
|
-
backup: boolean;
|
|
14
|
-
} & {
|
|
15
|
-
p: string | undefined;
|
|
16
|
-
} & {
|
|
17
|
-
file: string | undefined;
|
|
5
|
+
json: boolean;
|
|
18
6
|
} & {
|
|
19
7
|
i: string | undefined;
|
|
20
8
|
} & {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { commonOptions } from "../../common.js";
|
|
1
|
+
import { commonOptions, estimateOptions } from "../../common.js";
|
|
2
2
|
import { pdf_modes, pdf_sizes } from "../../../types/const.js";
|
|
3
|
-
export const builder = (yargs) => commonOptions(yargs)
|
|
3
|
+
export const builder = (yargs) => estimateOptions(commonOptions(yargs))
|
|
4
4
|
.option("i", {
|
|
5
5
|
alias: "imagedir",
|
|
6
6
|
describe: "Image output directory",
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { images, pdf } from "../../../actions/index.js";
|
|
2
|
-
import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded } from "../../helpers.js";
|
|
2
|
+
import { dumpUsageIfRequested, initializeContext, runTranslateIfNeeded, printUsageEstimate } from "../../helpers.js";
|
|
3
3
|
export const handler = async (argv) => {
|
|
4
4
|
const context = await initializeContext(argv);
|
|
5
5
|
if (!context) {
|
|
6
6
|
process.exit(1);
|
|
7
7
|
}
|
|
8
|
+
if (argv.estimate) {
|
|
9
|
+
printUsageEstimate(context, "pdf", argv.json);
|
|
10
|
+
return;
|
|
11
|
+
}
|
|
8
12
|
await runTranslateIfNeeded(context);
|
|
9
13
|
await images(context);
|
|
10
14
|
await pdf(context, argv.pdf_mode, argv.pdf_size);
|
|
@@ -1,20 +1,8 @@
|
|
|
1
1
|
import type { Argv } from "yargs";
|
|
2
|
-
export declare const builder: (yargs: Argv) => Argv<
|
|
3
|
-
|
|
2
|
+
export declare const builder: (yargs: Argv) => Argv<{
|
|
3
|
+
estimate: boolean;
|
|
4
4
|
} & {
|
|
5
|
-
|
|
5
|
+
json: boolean;
|
|
6
6
|
} & {
|
|
7
|
-
l: string | undefined;
|
|
8
|
-
} & {
|
|
9
|
-
f: boolean;
|
|
10
|
-
} & {
|
|
11
|
-
g: boolean;
|
|
12
|
-
} & {
|
|
13
|
-
backup: boolean;
|
|
14
|
-
} & {
|
|
15
|
-
p: string | undefined;
|
|
16
|
-
} & {
|
|
17
|
-
file: string | undefined;
|
|
18
|
-
}, "file"> & {
|
|
19
7
|
file: string | undefined;
|
|
20
8
|
}>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { commonOptions } from "../../common.js";
|
|
2
|
-
export const builder = (yargs) => commonOptions(yargs).positional("file", {
|
|
1
|
+
import { commonOptions, estimateOptions } from "../../common.js";
|
|
2
|
+
export const builder = (yargs) => estimateOptions(commonOptions(yargs)).positional("file", {
|
|
3
3
|
describe: "Mulmo Script File",
|
|
4
4
|
type: "string",
|
|
5
5
|
});
|