mulmocast 2.6.21 → 2.6.22

package/README.md

@@ -391,6 +391,34 @@ mulmo movie script.json
 mulmo translate script.json
 ```
 
+### Usage tracking (token / character / predict-second per provider:model)
+
+Any of the commands above can dump a JSON breakdown of upstream AI consumption when `MULMOCAST_DUMP_USAGE` is set:
+
+```bash
+# print JSON to stdout (pipe-friendly: `... | jq .`)
+MULMOCAST_DUMP_USAGE=1 mulmo audio script.json
+
+# ...or write it to a file
+MULMOCAST_DUMP_USAGE=/tmp/usage.json mulmo movie script.json
+```
+
+Payload shape:
+
+```json
+{
+  "records": 3,
+  "byModel": [
+    { "provider": "openai",     "model": "gpt-4o-mini-tts", "records": 1, "inputChars": 20 },
+    { "provider": "gemini",     "model": "gemini-2.5-flash-preview-tts", "records": 1, "inputTokens": 9, "outputTokens": 59, "totalTokens": 68 },
+    { "provider": "elevenlabs", "model": "eleven_multilingual_v2", "records": 1, "inputChars": 18 }
+  ],
+  "snapshot": [ /* per-call UsageRecord[] */ ]
+}
+```
+
+`byModel` groups by `provider:model` (different models have different rate cards, so summing across them isn't meaningful). The full per-call snapshot is also included so a billing layer can apply its own grouping. See [docs/api.md § Usage tracking](./docs/api.md#usage-tracking) for the per-agent field-population matrix and the programmatic (library) API.
+
 ## Cache and Re-run
 When running the same `mulmo` command multiple times, previously generated files are treated as cache. For example, audio or image files will not be regenerated if they already exist.
 

package/lib/agents/image_openai_agent.js

@@ -95,17 +95,22 @@ export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
             }
             if (error instanceof APIError) {
                 if (error.code && error.type) {
-                    throw new Error("Failed to generate image with OpenAI", {
+                    throw new Error(`Failed to generate image with OpenAI: ${error.message}`, {
                         cause: openAIAgentGenerationError("imageOpenaiAgent", imageAction, error.code, error.type),
                     });
                 }
                 if (error.type === "invalid_request_error" && error?.error?.message?.includes("Your organization must be verified")) {
-                    throw new Error("Failed to generate image with OpenAI", {
+                    throw new Error(`Failed to generate image with OpenAI: ${error.message}`, {
                         cause: openAIAgentGenerationError("imageOpenaiAgent", imageAction, "need_verified_organization", error.type),
                     });
                 }
             }
-            throw new Error("Failed to generate image with OpenAI", {
+            // Catch-all: include the underlying message so unknown errors
+            // (OpenAI SDK exceptions w/o APIError shape, fetch resets) are
+            // diagnosable from the thrown error alone — not just the log
+            // line above. Same template as #1452 / #1453 / #1454 / #1455 / #1456.
+            const detail = error instanceof Error ? error.message : String(error);
+            throw new Error(`Failed to generate image with OpenAI: ${detail}`, {
                 cause: agentGenerationError("imageOpenaiAgent", imageAction, imageFileTarget),
             });
         }

package/lib/agents/image_replicate_agent.js

@@ -76,7 +76,11 @@ export const imageReplicateAgent = async ({ namedInputs, params, config, }) => {
                 });
             }
         }
-        throw new Error("Failed to generate image with Replicate", {
+        // Include the underlying message so the catch-all path doesn't
+        // mask Replicate SDK / fetch / arrayBuffer failures behind a
+        // generic label (same fix as tts_gemini #1452, whisper #1453).
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to generate image with Replicate: ${detail}`, {
             cause: agentGenerationError("imageReplicateAgent", imageAction, imageFileTarget),
         });
     }

package/lib/agents/lipsync_replicate_agent.js

@@ -81,7 +81,12 @@ export const lipSyncReplicateAgent = async ({ namedInputs, params, config, }) =>
         if (hasCause(error) && error.cause) {
             throw error;
         }
-        throw new Error("Failed to lipSync with Replicate", {
+        // Include the underlying message so the catch-all path doesn't
+        // mask Replicate SDK / fetch / arrayBuffer failures behind a
+        // generic label (same fix as tts_gemini #1452, whisper #1453,
+        // image_replicate #1454).
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to lipSync with Replicate: ${detail}`, {
             cause: agentGenerationError("lipSyncReplicateAgent", imageAction, movieFileTarget),
         });
     }

package/lib/agents/movie_replicate_agent.js

@@ -166,6 +166,15 @@ export const movieReplicateAgent = async ({ namedInputs, params, config, }) => {
             throw error;
         }
         GraphAILogger.info("Failed to generate movie:", error.message);
+        // Throw a properly-labeled error with the underlying message
+        // preserved rather than falling through to the "returned
+        // undefined" line below — that message is only meaningful when
+        // `result` actually IS undefined, not when generation threw.
+        // (Same template as #1452 / #1453 / #1454 / #1455.)
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to generate movie with Replicate: ${detail}`, {
+            cause: agentGenerationError("movieReplicateAgent", imageAction, movieFileTarget),
+        });
     }
     throw new Error("ERROR: generateMovie returned undefined", {
         cause: agentInvalidResponseError("movieReplicateAgent", imageAction, movieFileTarget),

package/lib/agents/tts_gemini_agent.js

@@ -19,55 +19,72 @@ export const ttsGeminiAgent = async ({ namedInputs, params, config, }) => {
             cause: apiKeyMissingError("ttsGeminiAgent", audioAction, "GEMINI_API_KEY"),
         });
     }
-    try {
-        const ai = new GoogleGenAI({ apiKey });
-        const response = await ai.models.generateContent({
-            model: model ?? provider2TTSAgent.gemini.defaultModel,
-            contents: [{ parts: [{ text: getPrompt(text, instructions) }] }],
-            config: {
-                responseModalities: ["AUDIO"],
-                speechConfig: {
-                    voiceConfig: {
-                        prebuiltVoiceConfig: { voiceName: voice ?? provider2TTSAgent.gemini.defaultVoice },
+    const geminiResult = await (async () => {
+        try {
+            const ai = new GoogleGenAI({ apiKey });
+            const response = await ai.models.generateContent({
+                model: model ?? provider2TTSAgent.gemini.defaultModel,
+                contents: [{ parts: [{ text: getPrompt(text, instructions) }] }],
+                config: {
+                    responseModalities: ["AUDIO"],
+                    speechConfig: {
+                        voiceConfig: {
+                            prebuiltVoiceConfig: { voiceName: voice ?? provider2TTSAgent.gemini.defaultVoice },
+                        },
                     },
                 },
-            },
-        });
-        const inlineData = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
-        const pcmBase64 = inlineData?.data;
-        const mimeType = inlineData?.mimeType;
-        if (!pcmBase64 || typeof pcmBase64 !== "string")
-            throw new Error("No audio data returned");
-        // Extract sample rate from mimeType (e.g., "audio/L16;codec=pcm;rate=24000")
-        const rateMatch = mimeType?.match(/rate=(\d+)/);
-        const sampleRate = rateMatch ? parseInt(rateMatch[1]) : 24000;
-        const rawPcm = Buffer.from(pcmBase64, "base64");
-        const usedModel = model ?? provider2TTSAgent.gemini.defaultModel;
-        const usage = response.usageMetadata
-            ? {
-                provider: "gemini",
-                model: usedModel,
-                inputTokens: response.usageMetadata.promptTokenCount,
-                outputTokens: response.usageMetadata.candidatesTokenCount,
-                totalTokens: response.usageMetadata.totalTokenCount,
+            });
+            const inlineData = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
+            const pcmBase64 = inlineData?.data;
+            const mimeType = inlineData?.mimeType;
+            if (!pcmBase64 || typeof pcmBase64 !== "string")
+                throw new Error("No audio data returned");
+            // Extract sample rate from mimeType (e.g., "audio/L16;codec=pcm;rate=24000")
+            const rateMatch = mimeType?.match(/rate=(\d+)/);
+            const sampleRate = rateMatch ? parseInt(rateMatch[1]) : 24000;
+            const rawPcm = Buffer.from(pcmBase64, "base64");
+            const usedModel = model ?? provider2TTSAgent.gemini.defaultModel;
+            const usage = response.usageMetadata
+                ? {
+                    provider: "gemini",
+                    model: usedModel,
+                    inputTokens: response.usageMetadata.promptTokenCount,
+                    outputTokens: response.usageMetadata.candidatesTokenCount,
+                    totalTokens: response.usageMetadata.totalTokenCount,
+                }
+                : undefined;
+            return { rawPcm, sampleRate, usage };
+        }
+        catch (e) {
+            if (suppressError) {
+                return { error: e };
+            }
+            GraphAILogger.info(e);
+            const reasonDetail = getGenAIErrorReason(e);
+            if (reasonDetail && reasonDetail.reason && reasonDetail.reason === "API_KEY_INVALID") {
+                throw new Error("Failed to generate tts: 400 Incorrect API key provided with gemini", {
+                    cause: agentIncorrectAPIKeyError("ttsGeminiAgent", audioAction, audioFileTarget),
+                });
             }
-            : undefined;
-        return { buffer: await pcmToMp3(rawPcm, sampleRate), usage };
+            const detail = e instanceof Error ? e.message : String(e);
+            throw new Error(`TTS Gemini Error: ${detail}`, {
+                cause: agentGenerationError("ttsGeminiAgent", audioAction, audioFileTarget),
+            });
+        }
+    })();
+    if ("error" in geminiResult) {
+        return geminiResult;
+    }
+    try {
+        return { buffer: await pcmToMp3(geminiResult.rawPcm, geminiResult.sampleRate), usage: geminiResult.usage };
     }
     catch (e) {
         if (suppressError) {
-            return {
-                error: e,
-            };
+            return { error: e };
         }
         GraphAILogger.info(e);
-        const reasonDetail = getGenAIErrorReason(e);
-        if (reasonDetail && reasonDetail.reason && reasonDetail.reason === "API_KEY_INVALID") {
-            throw new Error("Failed to generate tts: 400 Incorrect API key provided with gemini", {
-                cause: agentIncorrectAPIKeyError("ttsGeminiAgent", audioAction, audioFileTarget),
-            });
-        }
-        throw new Error("TTS Gemini Error", {
+        const detail = e instanceof Error ? e.message : String(e);
+        throw new Error(`Audio encoding (ffmpeg) failed: ${detail}`, {
             cause: agentGenerationError("ttsGeminiAgent", audioAction, audioFileTarget),
         });
     }

package/lib/cli/commands/tool/whisper/handler.js

@@ -51,40 +51,63 @@ export const handler = async (argv) => {
         GraphAILogger.error("Error: OPENAI_API_KEY environment variable is required");
         process.exit(1);
     }
-    try {
-        // Get audio duration using FFmpeg
-        const { duration: audioDuration } = await ffmpegGetMediaDuration(fullPath);
-        GraphAILogger.info(`Audio duration: ${audioDuration.toFixed(2)} seconds`);
-        const openai = new OpenAI({ apiKey });
-        const transcription = await openai.audio.transcriptions.create({
-            file: createReadStream(fullPath),
-            model: "whisper-1",
-            response_format: "verbose_json",
-            timestamp_granularities: ["word", "segment"],
-        });
-        if (transcription.segments) {
-            const starts = transcription.segments.map((segment) => segment.start);
-            starts[0] = 0;
-            starts.push(audioDuration);
-            // Create beats from transcription segments
-            const beats = transcription.segments.map((segment, index) => {
-                const duration = Math.round((starts[index + 1] - starts[index]) * 100) / 100;
-                return {
-                    text: segment.text,
-                    duration,
-                    /*
-                    image: {
-                      type: "textSlide",
-                      slide: {
-                        title: "Place Holder",
-                      },
-                    },
-                    */
-                };
+    // Each phase has its own try/catch so a failure points at the right
+    // subsystem instead of being collapsed into "Error transcribing audio"
+    // (same anti-pattern as #1451 in tts_gemini_agent.ts — an ffmpeg
+    // SIGABRT or a permission-denied write would otherwise read as an
+    // OpenAI-side failure).
+    // Phase 1 — ffmpeg: read the audio file's duration.
+    const audioDuration = await (async () => {
+        try {
+            const { duration } = await ffmpegGetMediaDuration(fullPath);
+            return duration;
+        }
+        catch (error) {
+            GraphAILogger.error("Error reading audio duration with ffmpeg:", error);
+            process.exit(1);
+        }
+    })();
+    GraphAILogger.info(`Audio duration: ${audioDuration.toFixed(2)} seconds`);
+    // Phase 2 — OpenAI: Whisper transcription API call.
+    const transcription = await (async () => {
+        try {
+            const openai = new OpenAI({ apiKey });
+            return await openai.audio.transcriptions.create({
+                file: createReadStream(fullPath),
+                model: "whisper-1",
+                response_format: "verbose_json",
+                timestamp_granularities: ["word", "segment"],
             });
-            // Create the script with the processed beats
-            const script = createMulmoScript(fullPath, beats);
-            // Save script to output directory
+        }
+        catch (error) {
+            GraphAILogger.error("Error calling OpenAI transcription API:", error);
+            process.exit(1);
+        }
+    })();
+    if (transcription.segments) {
+        const starts = transcription.segments.map((segment) => segment.start);
+        starts[0] = 0;
+        starts.push(audioDuration);
+        // Create beats from transcription segments
+        const beats = transcription.segments.map((segment, index) => {
+            const duration = Math.round((starts[index + 1] - starts[index]) * 100) / 100;
+            return {
+                text: segment.text,
+                duration,
+                /*
+                image: {
+                  type: "textSlide",
+                  slide: {
+                    title: "Place Holder",
+                  },
+                },
+                */
+            };
+        });
+        // Create the script with the processed beats
+        const script = createMulmoScript(fullPath, beats);
+        // Phase 3 — fs: write the generated script to disk.
+        try {
             const outputDir = "output";
             if (!existsSync(outputDir)) {
                 mkdirSync(outputDir, { recursive: true });
@@ -93,9 +116,9 @@ export const handler = async (argv) => {
             writeFileSync(outputPath, JSON.stringify(script, null, 2));
             GraphAILogger.info(`Script saved to: ${outputPath}`);
         }
-    }
-    catch (error) {
-        GraphAILogger.error("Error transcribing audio:", error);
-        process.exit(1);
+        catch (error) {
+            GraphAILogger.error("Error writing transcription output file:", error);
+            process.exit(1);
+        }
     }
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.6.21",
+  "version": "2.6.22",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",