mulmocast 2.6.20 → 2.6.22

package/README.md

@@ -137,6 +137,14 @@ ELEVENLABS_API_KEY=your_elevenlabs_api_key
 BROWSERLESS_API_TOKEN=your_browserless_api_token # to access web in mulmo tool
 ```
 
+#### (Optional) Local usage tracking
+```bash
+MULMOCAST_DUMP_USAGE=1                 # print JSON usage dump to stdout after each CLI action
+MULMOCAST_DUMP_USAGE=/tmp/usage.json   # ...or write it to a file
+```
+
+Token / per-second / per-char usage per provider:model. See [docs/api.md](./docs/api.md#usage-tracking) for the full reference.
+
 ### Google Vertex AI
 
 For enterprise/production environments, or to use Veo movie models that are only available on Vertex AI, use Vertex AI with Application Default Credentials (ADC):
@@ -383,6 +391,34 @@ mulmo movie script.json
 mulmo translate script.json
 ```
 
+### Usage tracking (token / character / predict-second per provider:model)
+
+Any of the commands above can dump a JSON breakdown of upstream AI consumption when `MULMOCAST_DUMP_USAGE` is set:
+
+```bash
+# print JSON to stdout (pipe-friendly: `... | jq .`)
+MULMOCAST_DUMP_USAGE=1 mulmo audio script.json
+
+# ...or write it to a file
+MULMOCAST_DUMP_USAGE=/tmp/usage.json mulmo movie script.json
+```
+
+Payload shape:
+
+```json
+{
+  "records": 3,
+  "byModel": [
+    { "provider": "openai",     "model": "gpt-4o-mini-tts", "records": 1, "inputChars": 20 },
+    { "provider": "gemini",     "model": "gemini-2.5-flash-preview-tts", "records": 1, "inputTokens": 9, "outputTokens": 59, "totalTokens": 68 },
+    { "provider": "elevenlabs", "model": "eleven_multilingual_v2", "records": 1, "inputChars": 18 }
+  ],
+  "snapshot": [ /* per-call UsageRecord[] */ ]
+}
+```
+
+`byModel` groups by `provider:model` (different models have different rate cards, so summing across them isn't meaningful). The full per-call snapshot is also included so a billing layer can apply its own grouping. See [docs/api.md § Usage tracking](./docs/api.md#usage-tracking) for the per-agent field-population matrix and the programmatic (library) API.
+
 ## Cache and Re-run
 When running the same `mulmo` command multiple times, previously generated files are treated as cache. For example, audio or image files will not be regenerated if they already exist.
 

package/lib/actions/audio.js

@@ -4,6 +4,7 @@ import * as agents from "@graphai/vanilla";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, ttsElevenlabsAgent, ttsKotodamaAgent, addBGMAgent, combineAudioFilesAgent, mediaMockAgent, } from "../agents/index.js";
 import { audioGraphOption } from "./graph_option.js";
+import { createUsageCallback } from "../utils/usage_callback.js";
 import { getAudioArtifactFilePath, getAudioFilePath, getGroupedAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage, } from "../utils/file.js";
 import { localizedText } from "../utils/utils.js";
 import { text2hash } from "../utils/utils_node.js";
@@ -232,6 +233,7 @@ export const generateBeatAudio = async (index, context, args) => {
         mkdir(audioSegmentDirPath);
         const graph = new GraphAI(langs ? graph_tts_map : graph_tts, audioAgents, await audioGraphOption(context, settings));
         callbacks?.forEach((callback) => graph.registerCallback(callback));
+        graph.registerCallback(createUsageCallback(context));
         graph.injectValue("__mapIndex", index);
         graph.injectValue("beat", context.studio.script.beats[index]);
         graph.injectValue("studioBeat", context.studio.beats[index]);
@@ -268,6 +270,7 @@ export const audio = async (context, args) => {
         mkdir(audioSegmentDirPath);
         const graph = new GraphAI(audio_graph_data, audioAgents, await audioGraphOption(context, settings));
         callbacks?.forEach((callback) => graph.registerCallback(callback));
+        graph.registerCallback(createUsageCallback(context));
         graph.injectValue("context", context);
         graph.injectValue("audioArtifactFilePath", audioArtifactFilePath);
         graph.injectValue("audioCombinedFilePath", audioCombinedFilePath);

package/lib/actions/image_references.js

@@ -1,6 +1,7 @@
 import { GraphAI, GraphAILogger } from "graphai";
 import { getReferenceImagePath } from "../utils/file.js";
 import { imageGraphOption } from "./graph_option.js";
+import { createUsageCallback } from "../utils/usage_callback.js";
 import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods } from "../methods/index.js";
 import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent } from "../agents/index.js";
 import { agentGenerationError, imageReferenceAction, imageFileTarget, movieFileTarget } from "../utils/error_cause.js";
@@ -43,6 +44,7 @@ export const generateReferenceImage = async (inputs) => {
     try {
         const options = await imageGraphOption(context);
         const graph = new GraphAI(image_graph_data, { imageGenAIAgent, imageOpenaiAgent, mediaMockAgent, imageReplicateAgent }, options);
+        graph.registerCallback(createUsageCallback(context));
         await graph.run();
         return imagePath;
     }
@@ -129,6 +131,7 @@ const generateReferenceMovie = async (inputs) => {
     try {
         const options = await imageGraphOption(context);
         const graph = new GraphAI(movie_graph_data, { movieGenAIAgent, movieReplicateAgent, mediaMockAgent }, options);
+        graph.registerCallback(createUsageCallback(context));
         await graph.run();
         return moviePath;
     }

package/lib/actions/images.js

@@ -15,6 +15,7 @@ import { extractImageFromMovie, ffmpegGetMediaDuration, trimMusic } from "../uti
 import { getMediaRefs, resolveBeatLocalRefs } from "./image_references.js";
 import { imagePreprocessAgent, imagePluginAgent, htmlImageGeneratorAgent } from "./image_agents.js";
 import { imageGraphOption } from "./graph_option.js";
+import { createUsageCallback } from "../utils/usage_callback.js";
 const vanillaAgents = vanilla.default ?? vanilla;
 const imageAgents = {
     imageGenAIAgent,
@@ -468,6 +469,7 @@ const generateImages = async (context, args) => {
             graph.registerCallback(callback);
         });
     }
+    graph.registerCallback(createUsageCallback(context));
     const res = await graph.run();
     return res.mergeResult;
 };
@@ -519,6 +521,7 @@ export const generateBeatImage = async (inputs) => {
                 graph.registerCallback(callback);
             });
         }
+        graph.registerCallback(createUsageCallback(context));
         await graph.run();
     }
     catch (error) {

package/lib/actions/translate.d.ts

@@ -19,6 +19,8 @@ export declare const translateTextGraph: {
             };
             output: {
                 text: string;
+                usage: string;
+                model: string;
             };
             agent: string;
         };

package/lib/actions/translate.js

@@ -5,6 +5,7 @@ import * as agents from "@graphai/vanilla";
 import { openAIAgent } from "@graphai/openai_agent";
 import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { splitText } from "../utils/string.js";
+import { createUsageCallback } from "../utils/usage_callback.js";
 import { settings2GraphAIConfig, beatId, multiLingualObjectToArray } from "../utils/utils.js";
 import { getMultiLingual } from "../utils/context.js";
 import { currentMulmoScriptVersion } from "../types/const.js";
@@ -36,6 +37,8 @@ export const translateTextGraph = {
             },
             output: {
                 text: ".text",
+                usage: ".usage",
+                model: ".model",
             },
             // return { lang, text } <- localizedText
             agent: "openAIAgent",
@@ -253,6 +256,7 @@ export const translateBeat = async (index, context, targetLangs, args) => {
                 graph.registerCallback(callback);
             });
         }
+        graph.registerCallback(createUsageCallback(context));
         const results = await graph.run();
         const multiLingual = getMultiLingual(outputMultilingualFilePath, context.studio.beats);
         const key = beatId(context.studio.script.beats[index]?.id, index);
@@ -294,6 +298,7 @@ export const translate = async (context, args) => {
                 graph.registerCallback(callback);
             });
         }
+        graph.registerCallback(createUsageCallback(context));
         const results = await graph.run();
         writingMessage(outputMultilingualFilePath);
         if (results.mergeStudioResult) {

package/lib/agents/image_genai_agent.js

@@ -21,12 +21,21 @@ const getGeminiContents = (prompt, referenceImages) => {
     });
     return contents;
 };
-const geminiFlashResult = (response) => {
+const geminiFlashResult = (response, model) => {
     if (!response.candidates?.[0]?.content?.parts) {
         throw new Error("ERROR: generateContent returned no candidates", {
             cause: agentInvalidResponseError("imageGenAIAgent", imageAction, imageFileTarget),
         });
     }
+    const usage = response.usageMetadata
+        ? {
+            provider: "google",
+            model,
+            inputTokens: response.usageMetadata.promptTokenCount,
+            outputTokens: response.usageMetadata.candidatesTokenCount,
+            totalTokens: response.usageMetadata.totalTokenCount,
+        }
+        : undefined;
     for (const part of response.candidates[0].content.parts) {
         if (part.text) {
             GraphAILogger.info("Gemini image generation response:", part.text);
@@ -39,7 +48,7 @@ const geminiFlashResult = (response) => {
                 });
             }
             const buffer = Buffer.from(imageData, "base64");
-            return { buffer };
+            return { buffer, usage };
         }
     }
     throw new Error("ERROR: generateContent returned no image data", {
@@ -102,7 +111,7 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
         })();
         const res = await resultify(() => ai.models.generateContent(contentParams));
         if (res.ok) {
-            return geminiFlashResult(res.value);
+            return geminiFlashResult(res.value, model);
         }
         return errorProcess(res.error);
     }

package/lib/agents/image_openai_agent.js

@@ -95,17 +95,22 @@ export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
             }
             if (error instanceof APIError) {
                 if (error.code && error.type) {
-                    throw new Error("Failed to generate image with OpenAI", {
+                    throw new Error(`Failed to generate image with OpenAI: ${error.message}`, {
                         cause: openAIAgentGenerationError("imageOpenaiAgent", imageAction, error.code, error.type),
                     });
                 }
                 if (error.type === "invalid_request_error" && error?.error?.message?.includes("Your organization must be verified")) {
-                    throw new Error("Failed to generate image with OpenAI", {
+                    throw new Error(`Failed to generate image with OpenAI: ${error.message}`, {
                         cause: openAIAgentGenerationError("imageOpenaiAgent", imageAction, "need_verified_organization", error.type),
                     });
                 }
             }
-            throw new Error("Failed to generate image with OpenAI", {
+            // Catch-all: include the underlying message so unknown errors
+            // (OpenAI SDK exceptions w/o APIError shape, fetch resets) are
+            // diagnosable from the thrown error alone — not just the log
+            // line above. Same template as #1452 / #1453 / #1454 / #1455 / #1456.
+            const detail = error instanceof Error ? error.message : String(error);
+            throw new Error(`Failed to generate image with OpenAI: ${detail}`, {
                 cause: agentGenerationError("imageOpenaiAgent", imageAction, imageFileTarget),
             });
         }
@@ -115,6 +120,16 @@ export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
             cause: agentInvalidResponseError("imageOpenaiAgent", imageAction, imageFileTarget),
         });
     }
+    // gpt-image-1 surfaces token usage; legacy DALL·E response shapes don't.
+    const usage = response.usage
+        ? {
+            provider: "openai",
+            model,
+            inputTokens: response.usage.input_tokens,
+            outputTokens: response.usage.output_tokens,
+            totalTokens: response.usage.total_tokens,
+        }
+        : undefined;
     const url = response.data[0].url;
     if (!url) {
         // For gpt-image-1
@@ -124,7 +139,7 @@ export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
                 cause: agentInvalidResponseError("imageOpenaiAgent", imageAction, imageFileTarget),
             });
         }
-        return { buffer: Buffer.from(image_base64, "base64") };
+        return { buffer: Buffer.from(image_base64, "base64"), usage };
     }
     // URL response handling (legacy OpenAI image API response format)
     const res = await fetch(url);
@@ -136,7 +151,7 @@ export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
     // 2. Read the response as an ArrayBuffer
     const arrayBuffer = await res.arrayBuffer();
     // 3. Convert the ArrayBuffer to a Node.js Buffer and return it along with url
-    return { buffer: Buffer.from(arrayBuffer) };
+    return { buffer: Buffer.from(arrayBuffer), usage };
 };
 const imageOpenaiAgentInfo = {
     name: "imageOpenaiAgent",

package/lib/agents/image_replicate_agent.js

@@ -4,6 +4,7 @@ import Replicate from "replicate";
 import { getAspectRatio } from "./movie_replicate_agent.js";
 import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, agentInvalidResponseError, imageAction, imageFileTarget, hasCause, } from "../utils/error_cause.js";
 import { provider2ImageAgent } from "../types/provider2agent.js";
+import { runReplicateWithMetrics } from "../utils/replicate_usage.js";
 // Replicate image models return one of: FileOutput (object with url() method),
 // Array<FileOutput>, string URL, or { url: string }. Normalize to URL string/URL.
 const extractImageUrl = (output) => {
@@ -44,7 +45,7 @@ export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
         });
     }
     try {
-        const output = await replicate.run(model, { input });
+        const { output, predictSec } = await runReplicateWithMetrics(replicate, model, input);
         const imageUrl = extractImageUrl(output);
         if (!imageUrl) {
             throw new Error("ERROR: generateImage returned undefined", {
@@ -59,7 +60,8 @@ export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
         }
         const arrayBuffer = await imageResponse.arrayBuffer();
         const buffer = Buffer.from(arrayBuffer);
-        return { buffer };
+        const usage = predictSec !== undefined ? { provider: "replicate", model, predictSec } : undefined;
+        return { buffer, usage };
     }
     catch (error) {
         GraphAILogger.info("Replicate generation error:", error);
@@ -74,7 +76,11 @@ export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
                 });
             }
         }
-        throw new Error("Failed to generate image with Replicate", {
+        // Include the underlying message so the catch-all path doesn't
+        // mask Replicate SDK / fetch / arrayBuffer failures behind a
+        // generic label (same fix as tts_gemini #1452, whisper #1453).
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to generate image with Replicate: ${detail}`, {
             cause: agentGenerationError("imageReplicateAgent", imageAction, imageFileTarget),
         });
     }

package/lib/agents/lipsync_replicate_agent.js

@@ -3,6 +3,7 @@ import { GraphAILogger } from "graphai";
 import Replicate from "replicate";
 import { provider2LipSyncAgent } from "../types/provider2agent.js";
 import { apiKeyMissingError, agentGenerationError, agentFileNotExistError, imageAction, movieFileTarget, audioFileTarget, hasCause, } from "../utils/error_cause.js";
+import { runReplicateWithMetrics } from "../utils/replicate_usage.js";
 export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) => {
     const { movieFile, audioFile, imageFile } = namedInputs;
     const apiKey = config?.apiKey;
@@ -60,9 +61,7 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
     }
     const model_identifier = provider2LipSyncAgent.replicate.modelParams[model]?.identifier ?? model;
     try {
-        const output = await replicate.run(model_identifier, {
-            input,
-        });
+        const { output, predictSec } = await runReplicateWithMetrics(replicate, model_identifier, input);
         if (output && typeof output === "object" && "url" in output) {
             const videoUrl = output.url();
             const videoResponse = await fetch(videoUrl);
@@ -72,7 +71,8 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
                 });
             }
             const arrayBuffer = await videoResponse.arrayBuffer();
-            return { buffer: Buffer.from(arrayBuffer) };
+            const usage = predictSec !== undefined ? { provider: "replicate", model, predictSec } : undefined;
+            return { buffer: Buffer.from(arrayBuffer), usage };
         }
         return undefined;
     }
@@ -81,7 +81,12 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
         if (hasCause(error) && error.cause) {
             throw error;
         }
-        throw new Error("Failed to lipSync with Replicate", {
+        // Include the underlying message so the catch-all path doesn't
+        // mask Replicate SDK / fetch / arrayBuffer failures behind a
+        // generic label (same fix as tts_gemini #1452, whisper #1453,
+        // image_replicate #1454).
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to lipSync with Replicate: ${detail}`, {
             cause: agentGenerationError("lipSyncReplicateAgent", imageAction, movieFileTarget),
         });
     }

package/lib/agents/movie_genai_agent.js

@@ -3,6 +3,7 @@ import { GraphAILogger, sleep } from "graphai";
 import { GoogleGenAI, PersonGeneration } from "@google/genai";
 import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, imageAction, movieFileTarget, videoDurationTarget, unsupportedModelTarget, hasCause, } from "../utils/error_cause.js";
 import { getAspectRatio } from "../utils/utils.js";
+import { ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
 import { ASPECT_RATIOS } from "../types/const.js";
 import { getModelDuration, provider2MovieAgent, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS } from "../types/provider2agent.js";
 const pollUntilDone = async (ai, operation) => {
@@ -35,7 +36,22 @@ const loadImageAsBase64 = (imagePath) => {
         mimeType: "image/png",
     };
 };
-const downloadVideo = async (ai, video, movieFile, isVertexAI) => {
+// Veo bills per second of generated video. The SDK response carries no usage
+// metadata (GenerateVideosResponse only has generatedVideos and RAI flags), so
+// we ffprobe the downloaded file to get the actual duration. Falls back to
+// undefined if ffprobe fails, which lets the billing layer use the requested
+// duration as a fallback.
+const probeDurationSec = async (movieFile) => {
+    try {
+        const { duration } = await ffmpegGetMediaDuration(movieFile);
+        return duration > 0 ? duration : undefined;
+    }
+    catch (e) {
+        GraphAILogger.warn("movieGenAIAgent: ffprobe failed, predictSec will be omitted", e);
+        return undefined;
+    }
+};
+const downloadVideo = async (ai, video, movieFile, isVertexAI, model) => {
     if (isVertexAI) {
         // Vertex AI returns videoBytes directly
         writeFileSync(movieFile, Buffer.from(video.videoBytes, "base64"));
@@ -48,7 +64,9 @@ const downloadVideo = async (ai, video, movieFile, isVertexAI) => {
         });
         await sleep(5000); // HACK: Without this, the file is not ready yet.
     }
-    return { saved: movieFile };
+    const predictSec = await probeDurationSec(movieFile);
+    const usage = predictSec !== undefined ? { provider: "google", model, predictSec } : undefined;
+    return { saved: movieFile, usage };
 };
 const createVeo31Payload = (model, prompt, aspectRatio, source) => ({
     model,
@@ -92,7 +110,7 @@ const generateExtendedVideo = async (ai, model, prompt, aspectRatio, imagePath,
             cause: agentInvalidResponseError("movieGenAIAgent", imageAction, movieFileTarget),
         });
     }
-    return downloadVideo(ai, result.video, movieFile, isVertexAI);
+    return downloadVideo(ai, result.video, movieFile, isVertexAI, model);
 };
 const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath, lastFrameImagePath, referenceImages, duration, movieFile, isVertexAI) => {
     const capabilities = provider2MovieAgent.google.modelParams[model];
@@ -139,7 +157,7 @@ const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath,
     const operation = await ai.models.generateVideos(payload);
     const response = await pollUntilDone(ai, operation);
     const video = getVideoFromResponse(response);
-    return downloadVideo(ai, video, movieFile, isVertexAI);
+    return downloadVideo(ai, video, movieFile, isVertexAI, model);
 };
 export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
     const { prompt, imagePath, lastFrameImagePath, referenceImages, movieFile } = namedInputs;

package/lib/agents/movie_replicate_agent.js

@@ -3,6 +3,7 @@ import { GraphAILogger } from "graphai";
 import Replicate from "replicate";
 import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, hasCause, imageAction, movieFileTarget, videoDurationTarget, unsupportedModelTarget, } from "../utils/error_cause.js";
 import { provider2MovieAgent, getModelDuration, AUDIO_MODE_OPTIONAL, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS } from "../types/provider2agent.js";
+import { runReplicateWithMetrics } from "../utils/replicate_usage.js";
 function replicate_get_videoUrl(output) {
     if (typeof output === "string")
         return output;
@@ -93,7 +94,7 @@ async function generateMovie(model, apiKey, prompt, imagePath, lastFrameImagePat
         }
     }
     try {
-        const output = await replicate.run(model, { input });
+        const { output, predictSec } = await runReplicateWithMetrics(replicate, model, input);
         // Download the generated video
         // Some models return a FileOutput object with a url() method; others return a plain string URL.
         const videoUrl = replicate_get_videoUrl(output);
@@ -105,7 +106,7 @@ async function generateMovie(model, apiKey, prompt, imagePath, lastFrameImagePat
                 });
             }
             const arrayBuffer = await videoResponse.arrayBuffer();
-            return Buffer.from(arrayBuffer);
+            return { buffer: Buffer.from(arrayBuffer), predictSec };
         }
         return undefined;
     }
@@ -154,9 +155,10 @@ export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
         });
     }
     try {
-        const buffer = await generateMovie(model, apiKey, prompt, imagePath, lastFrameImagePath, referenceImages, aspectRatio, duration, params.generateAudio);
-        if (buffer) {
-            return { buffer };
+        const result = await generateMovie(model, apiKey, prompt, imagePath, lastFrameImagePath, referenceImages, aspectRatio, duration, params.generateAudio);
+        if (result) {
+            const usage = result.predictSec !== undefined ? { provider: "replicate", model, predictSec: result.predictSec } : undefined;
+            return { buffer: result.buffer, usage };
         }
     }
     catch (error) {
@@ -164,6 +166,15 @@ export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
             throw error;
         }
         GraphAILogger.info("Failed to generate movie:", error.message);
+        // Throw a properly-labeled error with the underlying message
+        // preserved rather than falling through to the "returned
+        // undefined" line below — that message is only meaningful when
+        // `result` actually IS undefined, not when generation threw.
+        // (Same template as #1452 / #1453 / #1454 / #1455.)
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to generate movie with Replicate: ${detail}`, {
+            cause: agentGenerationError("movieReplicateAgent", imageAction, movieFileTarget),
+        });
     }
     throw new Error("ERROR: generateMovie returned undefined", {
         cause: agentInvalidResponseError("movieReplicateAgent", imageAction, movieFileTarget),

package/lib/agents/sound_effect_replicate_agent.js

@@ -3,6 +3,7 @@ import { GraphAILogger } from "graphai";
 import Replicate from "replicate";
 import { provider2SoundEffectAgent } from "../types/provider2agent.js";
 import { apiKeyMissingError, agentGenerationError, imageAction, movieFileTarget, hasCause } from "../utils/error_cause.js";
+import { runReplicateWithMetrics } from "../utils/replicate_usage.js";
 export const soundEffectReplicateAgent = async ({ namedInputs, params, config }) => {
     const { prompt, movieFile } = namedInputs;
     const apiKey = config?.apiKey;
@@ -28,9 +29,7 @@ export const soundEffectReplicateAgent = async ({ namedInputs, params, config })
     };
     try {
         const model_identifier = provider2SoundEffectAgent.replicate.modelParams[model]?.identifier ?? model;
-        const output = await replicate.run(model_identifier, {
-            input,
-        });
+        const { output, predictSec } = await runReplicateWithMetrics(replicate, model_identifier, input);
         if (output && typeof output === "object" && "url" in output) {
             const videoUrl = output.url();
             const videoResponse = await fetch(videoUrl);
@@ -40,7 +39,8 @@ export const soundEffectReplicateAgent = async ({ namedInputs, params, config })
                 });
             }
             const arrayBuffer = await videoResponse.arrayBuffer();
-            return { buffer: Buffer.from(arrayBuffer) };
+            const usage = predictSec !== undefined ? { provider: "replicate", model, predictSec } : undefined;
+            return { buffer: Buffer.from(arrayBuffer), usage };
         }
         return undefined;
     }

package/lib/agents/tts_elevenlabs_agent.js

@@ -68,9 +68,20 @@ export const ttsElevenlabsAgent = async ({ namedInputs, params, config, }) => {
             cause: agentGenerationError("ttsElevenlabsAgent", audioAction, audioFileTarget),
         });
     }
+    // ElevenLabs returns the exact billed character count in the `character-cost`
+    // header (already discounted for V2 Flash/Turbo etc. — see #1420 research).
+    // We read it BEFORE consuming the body so the header is still accessible.
+    const billedHeader = response.headers.get("character-cost");
+    const billedChars = billedHeader ? Number(billedHeader) : undefined;
     const arrayBuffer = await response.arrayBuffer();
     const buffer = Buffer.from(arrayBuffer);
-    return { buffer };
+    const usage = {
+        provider: "elevenlabs",
+        model: requestBody.model_id,
+        // Use upstream-billed amount when available (exact); fall back to raw text length.
+        inputChars: Number.isFinite(billedChars) ? billedChars : text.length,
+    };
+    return { buffer, usage };
 };
 const ttsElevenlabsAgentInfo = {
     name: "ttsElevenlabsAgent",

package/lib/agents/tts_gemini_agent.js

@@ -19,45 +19,72 @@ export const ttsGeminiAgent = async ({ namedInputs, params, config, }) => {
             cause: apiKeyMissingError("ttsGeminiAgent", audioAction, "GEMINI_API_KEY"),
         });
     }
-    try {
-        const ai = new GoogleGenAI({ apiKey });
-        const response = await ai.models.generateContent({
-            model: model ?? provider2TTSAgent.gemini.defaultModel,
-            contents: [{ parts: [{ text: getPrompt(text, instructions) }] }],
-            config: {
-                responseModalities: ["AUDIO"],
-                speechConfig: {
-                    voiceConfig: {
-                        prebuiltVoiceConfig: { voiceName: voice ?? provider2TTSAgent.gemini.defaultVoice },
+    const geminiResult = await (async () => {
+        try {
+            const ai = new GoogleGenAI({ apiKey });
+            const response = await ai.models.generateContent({
+                model: model ?? provider2TTSAgent.gemini.defaultModel,
+                contents: [{ parts: [{ text: getPrompt(text, instructions) }] }],
+                config: {
+                    responseModalities: ["AUDIO"],
+                    speechConfig: {
+                        voiceConfig: {
+                            prebuiltVoiceConfig: { voiceName: voice ?? provider2TTSAgent.gemini.defaultVoice },
+                        },
                     },
                 },
-            },
-        });
-        const inlineData = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
-        const pcmBase64 = inlineData?.data;
-        const mimeType = inlineData?.mimeType;
-        if (!pcmBase64 || typeof pcmBase64 !== "string")
-            throw new Error("No audio data returned");
-        // Extract sample rate from mimeType (e.g., "audio/L16;codec=pcm;rate=24000")
-        const rateMatch = mimeType?.match(/rate=(\d+)/);
-        const sampleRate = rateMatch ? parseInt(rateMatch[1]) : 24000;
-        const rawPcm = Buffer.from(pcmBase64, "base64");
-        return { buffer: await pcmToMp3(rawPcm, sampleRate) };
+            });
+            const inlineData = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
+            const pcmBase64 = inlineData?.data;
+            const mimeType = inlineData?.mimeType;
+            if (!pcmBase64 || typeof pcmBase64 !== "string")
+                throw new Error("No audio data returned");
+            // Extract sample rate from mimeType (e.g., "audio/L16;codec=pcm;rate=24000")
+            const rateMatch = mimeType?.match(/rate=(\d+)/);
+            const sampleRate = rateMatch ? parseInt(rateMatch[1]) : 24000;
+            const rawPcm = Buffer.from(pcmBase64, "base64");
+            const usedModel = model ?? provider2TTSAgent.gemini.defaultModel;
+            const usage = response.usageMetadata
+                ? {
+                    provider: "gemini",
+                    model: usedModel,
+                    inputTokens: response.usageMetadata.promptTokenCount,
+                    outputTokens: response.usageMetadata.candidatesTokenCount,
+                    totalTokens: response.usageMetadata.totalTokenCount,
+                }
+                : undefined;
+            return { rawPcm, sampleRate, usage };
+        }
+        catch (e) {
+            if (suppressError) {
+                return { error: e };
+            }
+            GraphAILogger.info(e);
+            const reasonDetail = getGenAIErrorReason(e);
+            if (reasonDetail && reasonDetail.reason && reasonDetail.reason === "API_KEY_INVALID") {
+                throw new Error("Failed to generate tts: 400 Incorrect API key provided with gemini", {
+                    cause: agentIncorrectAPIKeyError("ttsGeminiAgent", audioAction, audioFileTarget),
+                });
+            }
+            const detail = e instanceof Error ? e.message : String(e);
+            throw new Error(`TTS Gemini Error: ${detail}`, {
+                cause: agentGenerationError("ttsGeminiAgent", audioAction, audioFileTarget),
+            });
+        }
+    })();
+    if ("error" in geminiResult) {
+        return geminiResult;
+    }
+    try {
+        return { buffer: await pcmToMp3(geminiResult.rawPcm, geminiResult.sampleRate), usage: geminiResult.usage };
     }
     catch (e) {
         if (suppressError) {
-            return {
-                error: e,
-            };
+            return { error: e };
         }
         GraphAILogger.info(e);
-        const reasonDetail = getGenAIErrorReason(e);
-        if (reasonDetail && reasonDetail.reason && reasonDetail.reason === "API_KEY_INVALID") {
-            throw new Error("Failed to generate tts: 400 Incorrect API key provided with gemini", {
-                cause: agentIncorrectAPIKeyError("ttsGeminiAgent", audioAction, audioFileTarget),
-            });
-        }
-        throw new Error("TTS Gemini Error", {
+        const detail = e instanceof Error ? e.message : String(e);
+        throw new Error(`Audio encoding (ffmpeg) failed: ${detail}`, {
             cause: agentGenerationError("ttsGeminiAgent", audioAction, audioFileTarget),
         });
     }

package/lib/agents/tts_google_agent.js

@@ -42,7 +42,15 @@ export const ttsGoogleAgent = async ({ namedInputs, params }) => {
     try {
         // Call the Text-to-Speech API
         const [response] = await client.synthesizeSpeech(request, { timeout: SYNTHESIZE_TIMEOUT_MS });
-        return { buffer: response.audioContent };
+        // Google Cloud TTS bills per character; SynthesizeSpeechResponse exposes
+        // no usage info. Record inputChars so the billing layer can compute cost
+        // from `model` (voice tier) + char count.
+        const usage = {
+            provider: "google",
+            model: model ?? voice ?? "default",
+            inputChars: text.length,
+        };
+        return { buffer: response.audioContent, usage };
     }
     catch (e) {
         if (suppressError) {