mulmocast 2.16.0 → 2.17.0

package/README.md

@@ -404,7 +404,7 @@ Payload shape:
   "records": 3,
   "byModel": [
     { "provider": "openai",     "model": "gpt-4o-mini-tts", "records": 1, "inputChars": 20 },
-    { "provider": "gemini",     "model": "gemini-2.5-flash-preview-tts", "records": 1, "inputTokens": 9, "outputTokens": 59, "totalTokens": 68 },
+    { "provider": "gemini",     "model": "gemini-3.8-flash-tts", "records": 1, "inputTokens": 9, "outputTokens": 59, "totalTokens": 68 },
     { "provider": "elevenlabs", "model": "eleven_multilingual_v2", "records": 1, "inputChars": 18 }
   ],
   "snapshot": [ /* per-call UsageRecord[] */ ]

package/lib/actions/image_references.d.ts

@@ -6,10 +6,12 @@ type ImageAgentInfo = ReturnType<typeof MulmoPresentationStyleMethods.getImageAg
 type MovieAgentInfo = ReturnType<typeof MulmoPresentationStyleMethods.getMovieAgentInfo>;
 export declare const referenceImageGraphParams: (imageAgentInfo: ImageAgentInfo, canvasSize: MulmoCanvasDimension) => {
     model: string | undefined;
+    moderation: string | undefined;
     canvasSize: {
         width: number;
         height: number;
     };
+    quality: string | undefined;
     vertexai_project: string | undefined;
     vertexai_location: string | undefined;
 };

package/lib/actions/image_references.js

@@ -6,10 +6,12 @@ import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods, MulmoImagePromp
 import { ImageMediaType, } from "../types/index.js";
 import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent } from "../agents/index.js";
 import { agentGenerationError, imageReferenceAction, imageFileTarget, movieFileTarget } from "../utils/error_cause.js";
-// Reference media are generated outside the beat graphs, so they must carry the same Vertex AI target the beats do.
+// Reference media are generated outside the beat graphs, so they must carry the same params the beats do.
 export const referenceImageGraphParams = (imageAgentInfo, canvasSize) => ({
     model: imageAgentInfo.imageParams.model,
+    moderation: imageAgentInfo.imageParams.moderation,
     canvasSize,
+    quality: imageAgentInfo.imageParams.quality,
     vertexai_project: imageAgentInfo.imageParams.vertexai_project,
     vertexai_location: imageAgentInfo.imageParams.vertexai_location,
 });

package/lib/agents/image_openai_agent.js

@@ -4,7 +4,7 @@ import { GraphAILogger } from "graphai";
 import { toFile, AuthenticationError, RateLimitError, APIError } from "openai";
 import { createOpenAIClient } from "../utils/openai_client.js";
 import { safeFetch, FETCH_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
-import { provider2ImageAgent, gptImages, deprecatedOpenAIImageModelHints } from "../types/provider2agent.js";
+import { provider2ImageAgent, gptImages, gptImageQuality, deprecatedOpenAIImageModelHints } from "../types/provider2agent.js";
 import { apiKeyMissingError, agentGenerationError, openAIAgentGenerationError, agentIncorrectAPIKeyError, agentAPIRateLimitError, agentInvalidResponseError, imageAction, imageFileTarget, unsupportedModelTarget, } from "../utils/error_cause.js";
 const isDeprecatedOpenAIImageModel = (model) => model in deprecatedOpenAIImageModelHints;
 export const buildDeprecatedModelMessage = (model) => {
@@ -63,8 +63,9 @@ export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
     if (gptImages.includes(model)) {
         imageOptions.moderation = moderation || "auto";
         imageOptions.background = "opaque";
-        if (quality) {
-            imageOptions.quality = quality;
+        const resolvedQuality = gptImageQuality(model, quality);
+        if (resolvedQuality) {
+            imageOptions.quality = resolvedQuality;
         }
     }
     const response = await (async () => {

package/lib/agents/tts_gemini_agent.d.ts

@@ -1,5 +1,61 @@
 import type { AgentFunction, AgentFunctionInfo } from "graphai";
+import { type GenerateContentConfig } from "@google/genai";
+import { buildGeminiInteractionsTtsRequest, type GeminiInteractionsTtsInput } from "../utils/gemini_tts.js";
 import type { GoogleTTSAgentParams, AgentBufferResult, AgentTextInputs, AgentErrorResult } from "../types/agent.js";
+import type { AgentUsage } from "../types/usage.js";
+type GeminiAudio = {
+    rawPcm: Buffer;
+    sampleRate: number;
+    usage: AgentUsage | undefined;
+};
+type InlineAudioResponse = {
+    candidates?: {
+        content?: {
+            parts?: {
+                inlineData?: {
+                    data?: string;
+                    mimeType?: string;
+                };
+            }[];
+        };
+    }[];
+    usageMetadata?: {
+        promptTokenCount?: number;
+        candidatesTokenCount?: number;
+        totalTokenCount?: number;
+    };
+};
+type InteractionAudioResponse = {
+    output_audio?: {
+        data?: string;
+        mime_type?: string;
+        sample_rate?: number;
+    };
+    usage?: {
+        total_input_tokens?: number;
+        total_output_tokens?: number;
+        total_tokens?: number;
+    };
+};
+export type GeminiTtsClient = {
+    models: {
+        generateContent: (params: {
+            model: string;
+            contents: {
+                parts: {
+                    text: string;
+                }[];
+            }[];
+            config: GenerateContentConfig;
+        }) => Promise<InlineAudioResponse>;
+    };
+    interactions: {
+        create: (params: ReturnType<typeof buildGeminiInteractionsTtsRequest>, options?: {
+            timeout?: number;
+        }) => Promise<InteractionAudioResponse>;
+    };
+};
+export declare const generateGeminiTts: (ai: GeminiTtsClient, input: GeminiInteractionsTtsInput) => Promise<GeminiAudio>;
 export declare const ttsGeminiAgent: AgentFunction<GoogleTTSAgentParams, AgentBufferResult | AgentErrorResult, AgentTextInputs>;
 declare const ttsGeminiAgentInfo: AgentFunctionInfo;
 export default ttsGeminiAgentInfo;

package/lib/agents/tts_gemini_agent.js

@@ -3,16 +3,51 @@ import { GoogleGenAI } from "@google/genai";
 import { provider2TTSAgent } from "../types/provider2agent.js";
 import { agentIncorrectAPIKeyError, apiKeyMissingError, agentGenerationError, audioAction, audioFileTarget, getGenAIErrorReason, } from "../utils/error_cause.js";
 import { pcmToMp3 } from "../utils/ffmpeg_utils.js";
+import { GEMINI_TTS_SAMPLE_RATE, buildGeminiInteractionsTtsRequest, geminiDirectorsNotesPrompt, pcmFromInteractionAudio, usesGeminiInteractionsTts, } from "../utils/gemini_tts.js";
 // Per-request timeout so a stalled GenAI TTS call rejects (and GraphAI retry can
 // recover) instead of hanging. Generous vs. normal generation time.
 const GENAI_REQUEST_TIMEOUT_MS = 120_000;
-const getPrompt = (text, instructions) => {
-    // https://ai.google.dev/gemini-api/docs/speech-generation?hl=ja#controllable
-    if (instructions) {
-        return `### DIRECTOR'S NOTES\n${instructions}\n\n#### TRANSCRIPT\n${text}`;
-    }
-    return text;
+const geminiUsage = (model, inputTokens, outputTokens, totalTokens) => ({
+    provider: "gemini",
+    model,
+    inputTokens,
+    outputTokens,
+    totalTokens,
+});
+// 2.5 TTS: generateContent, direction embedded in the prompt, headerless PCM with the rate in the mimeType.
+const generateWithContent = async (ai, { model, text, voice, instructions }) => {
+    const response = await ai.models.generateContent({
+        model,
+        contents: [{ parts: [{ text: geminiDirectorsNotesPrompt(text, instructions) }] }],
+        config: {
+            responseModalities: ["AUDIO"],
+            speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } } },
+        },
+    });
+    const inlineData = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
+    const pcmBase64 = inlineData?.data;
+    if (!pcmBase64 || typeof pcmBase64 !== "string")
+        throw new Error("No audio data returned");
+    // Extract sample rate from mimeType (e.g., "audio/L16;codec=pcm;rate=24000")
+    const rateMatch = inlineData?.mimeType?.match(/rate=(\d+)/);
+    const sampleRate = rateMatch ? parseInt(rateMatch[1]) : GEMINI_TTS_SAMPLE_RATE;
+    const metadata = response.usageMetadata;
+    return {
+        rawPcm: Buffer.from(pcmBase64, "base64"),
+        sampleRate,
+        usage: metadata ? geminiUsage(model, metadata.promptTokenCount, metadata.candidatesTokenCount, metadata.totalTokenCount) : undefined,
+    };
+};
+const generateWithInteractions = async (ai, input) => {
+    // Passed per request too: the Interactions client is built apart from the models client.
+    const interaction = await ai.interactions.create(buildGeminiInteractionsTtsRequest(input), { timeout: GENAI_REQUEST_TIMEOUT_MS });
+    const usage = interaction.usage;
+    return {
+        ...pcmFromInteractionAudio(interaction.output_audio),
+        usage: usage ? geminiUsage(input.model, usage.total_input_tokens, usage.total_output_tokens, usage.total_tokens) : undefined,
+    };
 };
+export const generateGeminiTts = (ai, input) => usesGeminiInteractionsTts(input.model) ? generateWithInteractions(ai, input) : generateWithContent(ai, input);
 export const ttsGeminiAgent = async ({ namedInputs, params, config, }) => {
     const { text } = namedInputs;
     const { model, voice, suppressError, instructions } = params;
@@ -25,38 +60,8 @@ export const ttsGeminiAgent = async ({ namedInputs, params, config, }) => {
     const geminiResult = await (async () => {
         try {
             const ai = new GoogleGenAI({ apiKey, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
-            const response = await ai.models.generateContent({
-                model: model ?? provider2TTSAgent.gemini.defaultModel,
-                contents: [{ parts: [{ text: getPrompt(text, instructions) }] }],
-                config: {
-                    responseModalities: ["AUDIO"],
-                    speechConfig: {
-                        voiceConfig: {
-                            prebuiltVoiceConfig: { voiceName: voice ?? provider2TTSAgent.gemini.defaultVoice },
-                        },
-                    },
-                },
-            });
-            const inlineData = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
-            const pcmBase64 = inlineData?.data;
-            const mimeType = inlineData?.mimeType;
-            if (!pcmBase64 || typeof pcmBase64 !== "string")
-                throw new Error("No audio data returned");
-            // Extract sample rate from mimeType (e.g., "audio/L16;codec=pcm;rate=24000")
-            const rateMatch = mimeType?.match(/rate=(\d+)/);
-            const sampleRate = rateMatch ? parseInt(rateMatch[1]) : 24000;
-            const rawPcm = Buffer.from(pcmBase64, "base64");
-            const usedModel = model ?? provider2TTSAgent.gemini.defaultModel;
-            const usage = response.usageMetadata
-                ? {
-                    provider: "gemini",
-                    model: usedModel,
-                    inputTokens: response.usageMetadata.promptTokenCount,
-                    outputTokens: response.usageMetadata.candidatesTokenCount,
-                    totalTokens: response.usageMetadata.totalTokenCount,
-                }
-                : undefined;
-            return { rawPcm, sampleRate, usage };
+            const request = { model: model ?? provider2TTSAgent.gemini.defaultModel, text, voice: voice ?? provider2TTSAgent.gemini.defaultVoice, instructions };
+            return await generateGeminiTts(ai, request);
         }
         catch (e) {
             if (suppressError) {

package/lib/types/provider2agent.d.ts

@@ -42,6 +42,8 @@ export declare const provider2TTSAgent: {
     };
 };
 export declare const gptImages: string[];
+export declare const isGptImage25Model: (model: string) => boolean;
+export declare const gptImageQuality: <Quality extends string>(model: string, quality?: Quality) => Quality | "high" | undefined;
 export declare const deprecatedOpenAIImageModelHints: {
     readonly "dall-e-2": "Use 'gpt-image-2.5-sunburst' or another supported model.";
     readonly "dall-e-3": "Use 'gpt-image-2.5-sunburst' or another supported model.";

package/lib/types/provider2agent.js

@@ -16,9 +16,10 @@ export const provider2TTSAgent = {
     gemini: {
         agentName: "ttsGeminiAgent",
         hasLimitedConcurrency: false,
-        defaultModel: "gemini-2.5-flash-preview-tts",
+        // The 2.5 preview TTS models shut down on 2026-11-17; Google names gemini-3.8-flash-tts as their successor.
+        defaultModel: "gemini-3.8-flash-tts",
         defaultVoice: "Kore",
-        models: ["gemini-2.5-flash-preview-tts", "gemini-2.5-pro-preview-tts"],
+        models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts", "gemini-2.5-flash-preview-tts", "gemini-2.5-pro-preview-tts"],
         keyName: "GEMINI_API_KEY",
     },
     elevenlabs: {
@@ -45,6 +46,11 @@ export const provider2TTSAgent = {
     },
 };
 export const gptImages = ["gpt-image-2.5-sunburst", "gpt-image-2.5-flare", "gpt-image-2", "gpt-image-1.5", "gpt-image-1", "gpt-image-1-mini"];
+// The 2.5 models render "auto" (their default) at "low", where gpt-image-1 rendered it at "high".
+export const isGptImage25Model = (model) => model.startsWith("gpt-image-2.5");
+// An unset quality on 2.5 is sent as "high" to keep the look scripts had before the default model changed;
+// an explicit "auto" is the script's choice and is sent as is.
+export const gptImageQuality = (model, quality) => quality ?? (isGptImage25Model(model) ? "high" : undefined);
 const supportedOpenAIImageReplacementHint = "Use 'gpt-image-2.5-sunburst' or another supported model.";
 export const deprecatedOpenAIImageModelHints = {
     "dall-e-2": supportedOpenAIImageReplacementHint,
@@ -641,6 +647,9 @@ export const modelPricing = {
         // https://ai.google.dev/gemini-api/docs/pricing
         "gemini-2.5-flash-preview-tts": { unit: "tokens", inputPerMTokensUSD: 0.5, outputPerMTokensUSD: 10, asOf: "2026-07-03" },
         "gemini-2.5-pro-preview-tts": { unit: "tokens", inputPerMTokensUSD: 1, outputPerMTokensUSD: 20, asOf: "2026-07-03" },
+        // through 2026-12-31; $1 / $18 (flash) and $1 / $12 (lite) from 2027-01-01
+        "gemini-3.8-flash-tts": { unit: "tokens", inputPerMTokensUSD: 0.5, outputPerMTokensUSD: 9, asOf: "2026-10-07" },
+        "gemini-3.8-flash-lite-tts": { unit: "tokens", inputPerMTokensUSD: 0.5, outputPerMTokensUSD: 6, asOf: "2026-10-07" },
     },
     google: {
         // https://ai.google.dev/gemini-api/docs/pricing ($0.039/image ≒ 1290 output tokens at $30/1M)

package/lib/types/schema.d.ts

@@ -4466,8 +4466,8 @@ export declare const mulmoOpenAIImageModelSchema: z.ZodObject<{
         [x: string]: string;
     }>>;
     quality: z.ZodOptional<z.ZodEnum<{
-        low: "low";
         high: "high";
+        low: "low";
         auto: "auto";
         medium: "medium";
     }>>;

package/lib/utils/estimate_usage.js

@@ -4,10 +4,11 @@
 // assume every asset is generated fresh.
 import { Tiktoken } from "js-tiktoken/lite";
 import o200k_base from "js-tiktoken/ranks/o200k_base";
-import { provider2TTSAgent, provider2MovieAgent, gptImageOutputTokens, getModelDuration, defaultMovieModel, veoExtendedSeconds, VEO_EXTENSION_MODEL, VEO_SEGMENT_SEC, getModelPricing, defaultProviders, } from "../types/provider2agent.js";
+import { provider2TTSAgent, provider2MovieAgent, gptImageOutputTokens, isGptImage25Model, getModelDuration, defaultMovieModel, veoExtendedSeconds, VEO_EXTENSION_MODEL, VEO_SEGMENT_SEC, getModelPricing, defaultProviders, } from "../types/provider2agent.js";
 import { text2SpeechProviderSchema, text2MovieProviderSchema } from "../types/schema.js";
 import { MulmoPresentationStyleMethods } from "../methods/mulmo_presentation_style.js";
 import { MulmoBeatMethods } from "../methods/mulmo_beat.js";
+import { geminiTtsInputText } from "./gemini_tts.js";
 import { imagePrompt, htmlImageSystemPrompt, translateSystemPrompt, translatePrompts } from "./prompt.js";
 const MILLION = 1_000_000;
 const LATIN_CHARS_PER_TOKEN = 4;
@@ -92,15 +93,14 @@ const buildOpenAITts = ({ speaker, text, textIsFinal, beatIndex, lang }) => {
     return { process: "tts", beatIndex, lang, provider: "openai", model, inputChars: metric(text.length, textIsFinal), ...tokenMetrics };
 };
 const buildGeminiTts = ({ speaker, speechOptions, text, beatIndex, lang }) => {
-    // The gemini TTS agent wraps the transcript in a "Director's Notes" prompt when an instruction is set.
-    const prompt = speechOptions?.instruction ? `### DIRECTOR'S NOTES\n${speechOptions.instruction}\n\n#### TRANSCRIPT\n${text}` : text;
+    const model = speaker.model ?? provider2TTSAgent.gemini.defaultModel;
     return {
         process: "tts",
         beatIndex,
         lang,
         provider: "gemini",
-        model: speaker.model ?? provider2TTSAgent.gemini.defaultModel,
-        inputTokens: estimated(countHeuristicTokens(prompt)),
+        model,
+        inputTokens: estimated(countHeuristicTokens(geminiTtsInputText(model, text, speechOptions?.instruction))),
         outputTokens: estimated(estimateSpeechSec(text) * GEMINI_TTS_AUDIO_TOKENS_PER_SEC),
     };
 };
@@ -162,8 +162,8 @@ const gptImageSize = (canvasSize) => {
 const gptImageOutputMetric = (model, canvasSize, quality) => {
     const table = gptImageOutputTokens[gptImageSize(canvasSize)];
     const knownQuality = quality === "low" || quality === "medium" || quality === "high" ? quality : undefined;
-    // Unspecified quality means API "auto"; assume "high" as the conservative upper bound.
-    const tokens = table[knownQuality ?? "high"];
+    // Anything else renders at "high" (the upper bound), except an explicit "auto" on 2.5, which renders at "low".
+    const tokens = table[knownQuality ?? (quality === "auto" && isGptImage25Model(model) ? "low" : "high")];
     return metric(tokens, GPT_IMAGE_FIXED_TOKEN_MODELS.includes(model) && knownQuality !== undefined);
 };
 const buildImageRecord = ({ process, provider, model, prompt, canvasSize, quality, beatIndex, refKey }) => {

package/lib/utils/gemini_tts.d.ts

@@ -0,0 +1,45 @@
+export declare const GEMINI_INTERACTIONS_TTS_MODELS: ReadonlySet<string>;
+export declare const GEMINI_TTS_SAMPLE_RATE = 24000;
+export declare const geminiDirectorsNotesPrompt: (text: string, instructions?: string) => string;
+export declare const geminiTtsInputText: (model: string, text: string, instructions?: string) => string;
+export declare const usesGeminiInteractionsTts: (model: string) => boolean;
+export type GeminiInteractionsTtsInput = {
+    model: string;
+    text: string;
+    voice: string;
+    instructions?: string;
+};
+export declare const buildGeminiInteractionsTtsRequest: ({ model, text, voice, instructions }: GeminiInteractionsTtsInput) => {
+    model: string;
+    input: {
+        type: "user_input";
+        content: {
+            annotations?: {
+                type: "speech_metadata";
+                style: string;
+            }[] | undefined;
+            type: "text";
+            text: string;
+        }[];
+    }[];
+    response_format: {
+        type: "audio";
+        mime_type: string;
+        sample_rate: number;
+    };
+    generation_config: {
+        speech_config: {
+            voice: string;
+        }[];
+    };
+};
+type InteractionAudio = {
+    data?: string;
+    mime_type?: string;
+    sample_rate?: number;
+};
+export declare const pcmFromInteractionAudio: (audio: InteractionAudio | undefined) => {
+    rawPcm: Buffer;
+    sampleRate: number;
+};
+export {};

package/lib/utils/gemini_tts.js

@@ -0,0 +1,35 @@
+// Gemini 3.8 TTS takes the Interactions API: the text is read verbatim, so speaker direction goes in a
+// speech_metadata annotation instead of the prompt, and raw PCM must be asked for (the unary default is WAV).
+export const GEMINI_INTERACTIONS_TTS_MODELS = new Set(["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"]);
+export const GEMINI_TTS_SAMPLE_RATE = 24000;
+// 2.5 TTS has no field for direction, so it goes in the prompt: https://ai.google.dev/gemini-api/docs/speech-generation#controllable
+export const geminiDirectorsNotesPrompt = (text, instructions) => instructions ? ["### DIRECTOR'S NOTES", instructions, "", "#### TRANSCRIPT", text].join("\n") : text;
+// The text the model is given for one line: 3.8 gets the transcript plus the style annotation, 2.5 the wrapped prompt.
+export const geminiTtsInputText = (model, text, instructions) => usesGeminiInteractionsTts(model) ? [text, instructions].filter(Boolean).join("\n") : geminiDirectorsNotesPrompt(text, instructions);
+export const usesGeminiInteractionsTts = (model) => GEMINI_INTERACTIONS_TTS_MODELS.has(model);
+export const buildGeminiInteractionsTtsRequest = ({ model, text, voice, instructions }) => ({
+    model,
+    input: [
+        {
+            type: "user_input",
+            content: [{ type: "text", text, ...(instructions ? { annotations: [{ type: "speech_metadata", style: instructions }] } : {}) }],
+        },
+    ],
+    response_format: { type: "audio", mime_type: "audio/l16", sample_rate: GEMINI_TTS_SAMPLE_RATE },
+    generation_config: { speech_config: [{ voice }] },
+});
+const sampleRateOf = (audio) => {
+    if (typeof audio.sample_rate === "number" && audio.sample_rate > 0)
+        return audio.sample_rate;
+    const match = audio.mime_type?.match(/rate=(\d+)/);
+    return match ? Number(match[1]) : GEMINI_TTS_SAMPLE_RATE;
+};
+// Anything but headerless 16-bit PCM would be fed to the PCM encoder as noise, so it is an error, not a guess.
+export const pcmFromInteractionAudio = (audio) => {
+    if (!audio?.data)
+        throw new Error("No audio data returned");
+    if (!audio.mime_type?.toLowerCase().startsWith("audio/l16")) {
+        throw new Error(`Gemini TTS returned ${audio.mime_type ?? "audio of an unknown type"}, expected audio/l16`);
+    }
+    return { rawPcm: Buffer.from(audio.data, "base64"), sampleRate: sampleRateOf(audio) };
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mulmocast",
-  "version": "2.16.0",
+  "version": "2.17.0",
   "description": "",
   "type": "module",
   "main": "lib/index.node.js",
@@ -99,7 +99,7 @@
   "homepage": "https://github.com/receptron/mulmocast-cli#readme",
   "dependencies": {
     "@google-cloud/text-to-speech": "^6.4.1",
-    "@google/genai": "^2.17.1",
+    "@google/genai": "^2.24.0",
     "@graphai/anthropic_agent": "^2.1.0",
     "@graphai/browserless_agent": "^2.0.4",
     "@graphai/gemini_agent": "^2.1.0",