mulmocast 2.18.0 → 2.19.0

package/assets/schemas/mulmo_script.json

@@ -15465,6 +15465,106 @@
           "soundEffectPrompt": {
             "type": "string"
           },
+          "soundEffects": {
+            "description": "Sound effects mixed into the audio track during this beat. Multiple effects may overlap.",
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "source": {
+                  "oneOf": [
+                    {
+                      "type": "object",
+                      "properties": {
+                        "kind": {
+                          "type": "string",
+                          "const": "url"
+                        },
+                        "url": {
+                          "type": "string",
+                          "format": "uri"
+                        }
+                      },
+                      "required": [
+                        "kind",
+                        "url"
+                      ],
+                      "additionalProperties": false
+                    },
+                    {
+                      "type": "object",
+                      "properties": {
+                        "kind": {
+                          "type": "string",
+                          "const": "path"
+                        },
+                        "path": {
+                          "type": "string",
+                          "minLength": 1
+                        }
+                      },
+                      "required": [
+                        "kind",
+                        "path"
+                      ],
+                      "additionalProperties": false
+                    },
+                    {
+                      "type": "object",
+                      "properties": {
+                        "kind": {
+                          "type": "string",
+                          "const": "jinglescript"
+                        },
+                        "score": {
+                          "type": "object",
+                          "propertyNames": {
+                            "type": "string"
+                          },
+                          "additionalProperties": {},
+                          "description": "JingleScript score (format jinglescript/1), rendered to audio when the audio is generated"
+                        }
+                      },
+                      "required": [
+                        "kind",
+                        "score"
+                      ],
+                      "additionalProperties": false
+                    }
+                  ],
+                  "description": "Audio file of the sound effect (url or path), or a JingleScript score to synthesize"
+                },
+                "startAt": {
+                  "default": 0,
+                  "description": "Seconds from when the beat appears on screen (the same clock as its html animation) at which the sound effect plays. The first beat's narration starts audioParams.introPadding later",
+                  "type": "number",
+                  "minimum": 0
+                },
+                "volume": {
+                  "default": 1,
+                  "description": "Volume of the sound effect. 1.0 is the gain of the narration at its default volume; audioParams.audioVolume and ttsVolume do not affect sound effects",
+                  "type": "number",
+                  "minimum": 0,
+                  "maximum": 4
+                },
+                "duration": {
+                  "description": "Maximum play length in seconds; the effect is cut (with a short fade-out) after it. Default: the whole file, even past the end of the beat",
+                  "type": "number",
+                  "exclusiveMinimum": 0
+                },
+                "loop": {
+                  "description": "Repeat the effect until duration, or until the beat leaves the screen when duration is not set (e.g. a ticking clock)",
+                  "type": "boolean"
+                }
+              },
+              "required": [
+                "source",
+                "startAt",
+                "volume"
+              ],
+              "additionalProperties": false
+            }
+          },
           "htmlPrompt": {
             "type": "object",
             "properties": {

package/lib/actions/audio.js

@@ -5,6 +5,7 @@ import { fileWriteAgent } from "@graphai/vanilla_node_agents";
 import { ttsOpenaiAgent, ttsGoogleAgent, ttsGeminiAgent, ttsElevenlabsAgent, ttsKotodamaAgent, addBGMAgent, combineAudioFilesAgent, mediaMockAgent, } from "../agents/index.js";
 import { audioGraphOption } from "./graph_option.js";
 import { createUsageCallback } from "../utils/usage_callback.js";
+import { renderJingleScores } from "../utils/jinglescript.js";
 import { getAudioArtifactFilePath, getAudioFilePath, getGroupedAudioFilePath, getOutputStudioFilePath, resolveDirPath, defaultBGMPath, mkdir, writingMessage, } from "../utils/file.js";
 import { localizedText } from "../utils/utils.js";
 import { text2hash } from "../utils/utils_node.js";
@@ -202,7 +203,7 @@ export const audio_graph_data = {
                 wait: ":combineFiles",
                 voiceFile: ":audioCombinedFilePath",
                 outputFile: ":audioArtifactFilePath",
-                context: ":context",
+                context: ":combineFiles", // has studio.beats[].startAt, needed to place sound effects
                 params: {
                     musicFile: ":musicFile",
                 },
@@ -268,6 +269,8 @@ export const audio = async (context, args) => {
         const outputStudioFilePath = getOutputStudioFilePath(outDirPath, fileName);
         mkdir(outDirPath);
         mkdir(audioSegmentDirPath);
+        // Before TTS, so an invalid score fails fast.
+        await renderJingleScores(context);
         const graph = new GraphAI(audio_graph_data, audioAgents, await audioGraphOption(context, settings));
         callbacks?.forEach((callback) => graph.registerCallback(callback));
         graph.registerCallback(createUsageCallback(context));

package/lib/actions/avatar.js

@@ -168,7 +168,10 @@ const renderAvatarTrack = async (plan, context, getAvatarScript) => {
     // rendered once, at the largest size it is shown; smaller placements scale it down
     const height = even(Math.max(...plan.placements.map((p) => percent(p.position.scale, canvas.height))));
     const segments = plan.segments.map((segment) => ({ ...segment, audioStamp: fileStamp(segment.audio) }));
-    const identity = { version: 4, segments, source: plan.source, sourceStamp: await sourceStamp(plan.source), duration: plan.duration, height, padTop: PAD_TOP };
+    // bump `version` when the track would come out differently, including an avatarscript upgrade that
+    // changes its output (5: avatarscript 0.4.0 holds a phrase's last sound): a reused track needs no
+    // avatarscript, so its version cannot be part of this
+    const identity = { version: 5, segments, source: plan.source, sourceStamp: await sourceStamp(plan.source), duration: plan.duration, height, padTop: PAD_TOP };
     const hash = createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, 12);
     const dir = MulmoStudioContextMethods.getImageProjectDirPath(context);
     const name = `avatar_${plan.speakerId.replace(/[^\w-]/g, "_")}_${hash}`;

package/lib/agents/add_bgm_agent.d.ts

@@ -1,13 +1,25 @@
 import type { AgentFunctionInfo } from "graphai";
 import { MulmoStudioContext } from "../types/index.js";
+import { FfmpegContext } from "../utils/ffmpeg_utils.js";
 export declare const resolveAddBgmMixParams: (audioParams: MulmoStudioContext["presentationStyle"]["audioParams"]) => {
     useExplicitMix: boolean;
     voiceVolume: number;
 };
-export declare const resolveAddBgmFilterConfig: (useExplicitMix: boolean) => {
+export declare const resolveAddBgmFilterConfig: (useExplicitMix: boolean, mixedInputId?: string) => {
     amixNormalize: string;
     mixedOutputId: string;
     limiterFilter: string | undefined;
 };
+export type SoundEffectPlacement = {
+    file: string;
+    startAt: number;
+    volume: number;
+    duration?: number;
+    loop: boolean;
+};
+export declare const getSoundEffectPlacements: (context: MulmoStudioContext) => SoundEffectPlacement[];
+export declare const soundEffectGain: (useExplicitMix: boolean) => 0.5 | 1;
+export declare const buildSoundEffectFilters: (inputIds: number[], placements: SoundEffectPlacement[], gain: number, outputId: string) => string[];
+export declare const addSoundEffects: (ffmpegContext: FfmpegContext, context: MulmoStudioContext) => "mixed" | "mixed_sfx";
 declare const addBGMAgentInfo: AgentFunctionInfo;
 export default addBGMAgentInfo;

package/lib/agents/add_bgm_agent.js

@@ -1,7 +1,10 @@
 import { GraphAILogger } from "graphai";
 import { FfmpegContextAddInput, FfmpegContextInit, FfmpegContextGenerateOutput, ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
 import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
+import { MulmoMediaSourceMethods } from "../methods/mulmo_media_source.js";
 import { isFile } from "../utils/file.js";
+import { userAssert } from "../utils/utils.js";
+import { getJingleFilePath } from "../utils/jinglescript.js";
 import { agentGenerationError, agentFileNotExistError, audioAction, audioFileTarget } from "../utils/error_cause.js";
 export const resolveAddBgmMixParams = (audioParams) => {
     const useExplicitMix = audioParams.ttsVolume !== undefined;
@@ -11,14 +14,82 @@ export const resolveAddBgmMixParams = (audioParams) => {
         voiceVolume: audioParams.audioVolume * ttsVolume,
     };
 };
-export const resolveAddBgmFilterConfig = (useExplicitMix) => {
+export const resolveAddBgmFilterConfig = (useExplicitMix, mixedInputId = "mixed") => {
     const amixNormalize = useExplicitMix ? ":normalize=0" : "";
     return {
         amixNormalize,
-        mixedOutputId: useExplicitMix ? "mixed_limited" : "mixed",
-        limiterFilter: useExplicitMix ? "[mixed]alimiter=limit=0.95:attack=5:release=50[mixed_limited]" : undefined,
+        mixedOutputId: useExplicitMix ? "mixed_limited" : mixedInputId,
+        limiterFilter: useExplicitMix ? `[${mixedInputId}]alimiter=limit=0.95:attack=5:release=50[mixed_limited]` : undefined,
     };
 };
+// Absolute placement of every beat's sound effects on the final audio timeline. startAt counts from
+// when the beat appears on screen, the same clock as its html animation (for the first beat, that is
+// introPadding before its narration starts).
+// The context must come from combineAudioFilesAgent, which sets studio.beats[].startAt.
+export const getSoundEffectPlacements = (context) => {
+    return context.studio.script.beats.flatMap((beat, index) => (beat.soundEffects ?? []).map((soundEffect, seIndex) => {
+        const beatStartAt = context.studio.beats[index]?.startAt;
+        userAssert(beatStartAt !== undefined, `soundEffects: startAt of beat ${index} is not computed yet`);
+        const { source } = soundEffect;
+        const file = source.kind === "jinglescript" ? getJingleFilePath(source.score, context) : MulmoMediaSourceMethods.resolve(source, context);
+        userAssert(!!file, `soundEffects: unsupported source at beat ${index}, effect ${seIndex} (use url or path)`);
+        const startAt = soundEffect.startAt ?? 0;
+        const loop = soundEffect.loop ?? false;
+        // A looping effect without duration plays until the beat leaves the screen.
+        const duration = soundEffect.duration ?? (loop ? MulmoStudioContextMethods.getBeatScreenDuration(context, index) - startAt : undefined);
+        userAssert(duration === undefined || duration > 0, `soundEffects: beat ${index}, effect ${seIndex} starts after the end of the beat`);
+        return {
+            file,
+            startAt: MulmoStudioContextMethods.getBeatScreenStartAt(context, index) + startAt,
+            volume: soundEffect.volume ?? 1.0,
+            ...(duration !== undefined ? { duration } : {}),
+            loop,
+        };
+    }));
+};
+// In legacy mode, amix (normalize=1) halves the voice and the music. Sound effects are mixed in
+// afterwards with normalize=0, so apply the same factor to keep volume 1.0 at the gain of the narration at its
+// default volume. audioVolume / ttsVolume deliberately do not scale sound effects.
+export const soundEffectGain = (useExplicitMix) => (useExplicitMix ? 1.0 : 0.5);
+const soundEffectFadeOut = 0.05; // seconds, avoids a click where the effect is cut
+const getSoundEffectTrim = (duration) => {
+    const fade = Math.min(soundEffectFadeOut, duration);
+    return `atrim=duration=${duration}, afade=t=out:st=${duration - fade}:d=${fade}, `;
+};
+export const buildSoundEffectFilters = (inputIds, placements, gain, outputId) => {
+    const ids = placements.map((_, index) => `se_${index}`);
+    const filters = placements.map((placement, index) => {
+        const delayMs = Math.round(placement.startAt * 1000);
+        const trim = placement.duration !== undefined ? getSoundEffectTrim(placement.duration) : "";
+        return `[${inputIds[index]}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, ${trim}volume=${placement.volume * gain}, adelay=${delayMs}|${delayMs}[${ids[index]}]`;
+    });
+    if (ids.length === 1) {
+        return [...filters, `[${ids[0]}]anull[${outputId}]`];
+    }
+    const inputs = ids.map((id) => "[" + id + "]").join("");
+    return [...filters, `${inputs}amix=inputs=${ids.length}:duration=longest:normalize=0[${outputId}]`];
+};
+// Adds the beats' sound effects to ffmpegContext and mixes them onto [mixed].
+// Returns the id of the resulting stream ("mixed" when there are none).
+export const addSoundEffects = (ffmpegContext, context) => {
+    const { useExplicitMix } = resolveAddBgmMixParams(context.presentationStyle.audioParams);
+    const soundEffects = getSoundEffectPlacements(context);
+    if (soundEffects.length === 0) {
+        return "mixed";
+    }
+    soundEffects.forEach(({ file }) => {
+        if (!/^https?:\/\//.test(file) && !isFile(file)) {
+            throw new Error(`AddBGMAgent soundEffect file not exist: ${file}`, {
+                cause: agentFileNotExistError("addBGMAgent", audioAction, audioFileTarget, file),
+            });
+        }
+    });
+    GraphAILogger.log("soundEffects:", soundEffects);
+    const inputIds = soundEffects.map(({ file, loop }) => FfmpegContextAddInput(ffmpegContext, file, loop ? ["-stream_loop", "-1"] : undefined));
+    ffmpegContext.filterComplex.push(...buildSoundEffectFilters(inputIds, soundEffects, soundEffectGain(useExplicitMix), "sfx"));
+    ffmpegContext.filterComplex.push("[mixed][sfx]amix=inputs=2:duration=first:normalize=0[mixed_sfx]");
+    return "mixed_sfx";
+};
 const addBGMAgent = async ({ namedInputs, params, }) => {
     const { voiceFile, outputFile, context } = namedInputs;
     const { musicFile } = params;
@@ -44,8 +115,10 @@ const addBGMAgent = async ({ namedInputs, params, }) => {
     const { useExplicitMix, voiceVolume } = resolveAddBgmMixParams(audioParams);
     ffmpegContext.filterComplex.push(`[${musicInputIndex}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, volume=${audioParams.bgmVolume}[music]`);
     ffmpegContext.filterComplex.push(`[${voiceInputIndex}:a]aformat=sample_fmts=fltp:sample_rates=44100:channel_layouts=stereo, volume=${voiceVolume}, adelay=${introPadding * 1000}|${introPadding * 1000}[voice]`);
-    const { amixNormalize, mixedOutputId, limiterFilter } = resolveAddBgmFilterConfig(useExplicitMix);
+    const { amixNormalize } = resolveAddBgmFilterConfig(useExplicitMix);
     ffmpegContext.filterComplex.push(`[music][voice]amix=inputs=2:duration=longest${amixNormalize}[mixed]`);
+    const mixedInputId = addSoundEffects(ffmpegContext, context);
+    const { mixedOutputId, limiterFilter } = resolveAddBgmFilterConfig(useExplicitMix, mixedInputId);
     if (limiterFilter) {
         ffmpegContext.filterComplex.push(limiterFilter);
     }

package/lib/methods/mulmo_studio_context.d.ts

@@ -23,6 +23,8 @@ export declare const MulmoStudioContextMethods: {
     needTranslate(context: MulmoStudioContext, includeCaption?: boolean): boolean | "" | undefined;
     getIntroPadding(context: MulmoStudioContext): number;
     getExtraPadding(context: MulmoStudioContext, index: number): number;
+    getBeatScreenStartAt(context: MulmoStudioContext, index: number): number;
+    getBeatScreenDuration(context: MulmoStudioContext, index: number): number;
     getBeatDuration(context: MulmoStudioContext, index: number): number;
     getVoiceOverGroupDuration(context: MulmoStudioContext, index: number): number;
     isMovieBeat(context: MulmoStudioContext, studioBeat: MulmoStudioBeat, beat: MulmoBeat): boolean;

package/lib/methods/mulmo_studio_context.js

@@ -110,6 +110,25 @@ export const MulmoStudioContextMethods = {
         }
         return 0;
     },
+    // When this beat appears on screen, in seconds from the start of the movie. The first beat's segment
+    // starts at 0 and absorbs the intro padding; every later beat starts when its narration does.
+    getBeatScreenStartAt(context, index) {
+        if (index === 0) {
+            return 0;
+        }
+        return MulmoStudioContextMethods.getIntroPadding(context) + (context.studio.beats[index]?.startAt ?? 0);
+    },
+    // How long this beat stays on screen from getBeatScreenStartAt: its whole segment, including the
+    // voice_over beats that share its shot. A voice_over beat itself is counted for its own slice only.
+    // The audio ends outroPadding after the last beat; getExtraPadding gives a single-beat script only
+    // the intro padding, so its outro is added here.
+    getBeatScreenDuration(context, index) {
+        if (MulmoBeatMethods.isVoiceOver(context.studio.script.beats[index])) {
+            return MulmoStudioContextMethods.getBeatDuration(context, index);
+        }
+        const singleBeatOutro = context.studio.beats.length === 1 ? context.presentationStyle.audioParams.outroPadding : 0;
+        return MulmoStudioContextMethods.getSegmentDuration(context, index) + singleBeatOutro;
+    },
     // The duration this beat occupies on the audio timeline.
     getBeatDuration(context, index) {
         return (context.studio.beats[index]?.duration ?? 0) + MulmoStudioContextMethods.getExtraPadding(context, index);

package/lib/types/schema.d.ts

@@ -4985,6 +4985,22 @@ export declare const mulmoMovieParamsSchema: z.ZodObject<{
     concurrency: z.ZodOptional<z.ZodNumber>;
     generateAudio: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
+export declare const mulmoBeatSoundEffectSchema: z.ZodObject<{
+    source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodURL;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"jinglescript">;
+        score: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    }, z.core.$strict>], "kind">;
+    startAt: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    volume: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    duration: z.ZodOptional<z.ZodNumber>;
+    loop: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strict>;
 export declare const mulmoBeatSchema: z.ZodObject<{
     speaker: z.ZodOptional<z.ZodString>;
     text: z.ZodDefault<z.ZodOptional<z.ZodString>>;
@@ -9086,6 +9102,22 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     imagePrompt: z.ZodOptional<z.ZodString>;
     moviePrompt: z.ZodOptional<z.ZodString>;
     soundEffectPrompt: z.ZodOptional<z.ZodString>;
+    soundEffects: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodURL;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"jinglescript">;
+            score: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strict>], "kind">;
+        startAt: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        volume: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        duration: z.ZodOptional<z.ZodNumber>;
+        loop: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>>>;
     htmlPrompt: z.ZodOptional<z.ZodObject<{
         systemPrompt: z.ZodDefault<z.ZodOptional<z.ZodString>>;
         prompt: z.ZodString;
@@ -14353,6 +14385,22 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imagePrompt: z.ZodOptional<z.ZodString>;
         moviePrompt: z.ZodOptional<z.ZodString>;
         soundEffectPrompt: z.ZodOptional<z.ZodString>;
+        soundEffects: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"url">;
+                url: z.ZodURL;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"path">;
+                path: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"jinglescript">;
+                score: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            }, z.core.$strict>], "kind">;
+            startAt: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+            volume: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+            duration: z.ZodOptional<z.ZodNumber>;
+            loop: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>>>;
         htmlPrompt: z.ZodOptional<z.ZodObject<{
             systemPrompt: z.ZodDefault<z.ZodOptional<z.ZodString>>;
             prompt: z.ZodString;
@@ -19144,6 +19192,22 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imagePrompt: z.ZodOptional<z.ZodString>;
             moviePrompt: z.ZodOptional<z.ZodString>;
             soundEffectPrompt: z.ZodOptional<z.ZodString>;
+            soundEffects: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"jinglescript">;
+                    score: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+                }, z.core.$strict>], "kind">;
+                startAt: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+                volume: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+                duration: z.ZodOptional<z.ZodNumber>;
+                loop: z.ZodOptional<z.ZodBoolean>;
+            }, z.core.$strict>>>;
             htmlPrompt: z.ZodOptional<z.ZodObject<{
                 systemPrompt: z.ZodDefault<z.ZodOptional<z.ZodString>>;
                 prompt: z.ZodString;

package/lib/types/schema.js

@@ -612,6 +612,44 @@ export const mulmoMovieParamsSchema = z.object({
     concurrency: z.number().int().positive().optional().describe("Max concurrent movie generation requests"),
     generateAudio: z.boolean().optional().describe("Request audio generation in the video (model-dependent)"),
 });
+export const mulmoBeatSoundEffectSchema = z
+    .object({
+    source: z
+        .discriminatedUnion("kind", [
+        z.object({ kind: z.literal("url"), url: URLStringSchema }).strict(),
+        z.object({ kind: z.literal("path"), path: z.string().min(1) }).strict(),
+        z
+            .object({
+            kind: z.literal("jinglescript"),
+            score: z.record(z.string(), z.unknown()).describe("JingleScript score (format jinglescript/1), rendered to audio when the audio is generated"),
+        })
+            .strict(),
+    ])
+        .describe("Audio file of the sound effect (url or path), or a JingleScript score to synthesize"),
+    startAt: z
+        .number()
+        .min(0)
+        .optional()
+        .default(0)
+        .describe("Seconds from when the beat appears on screen (the same clock as its html animation) at which the sound effect plays. The first beat's narration starts audioParams.introPadding later"),
+    volume: z
+        .number()
+        .min(0)
+        .max(4)
+        .optional()
+        .default(1.0)
+        .describe("Volume of the sound effect. 1.0 is the gain of the narration at its default volume; audioParams.audioVolume and ttsVolume do not affect sound effects"),
+    duration: z
+        .number()
+        .positive()
+        .optional()
+        .describe("Maximum play length in seconds; the effect is cut (with a short fade-out) after it. Default: the whole file, even past the end of the beat"),
+    loop: z
+        .boolean()
+        .optional()
+        .describe("Repeat the effect until duration, or until the beat leaves the screen when duration is not set (e.g. a ticking clock)"),
+})
+    .strict();
 export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.optional(),
@@ -643,6 +681,10 @@ export const mulmoBeatSchema = z
     imagePrompt: z.string().optional(),
     moviePrompt: z.string().optional(),
     soundEffectPrompt: z.string().optional(),
+    soundEffects: z
+        .array(mulmoBeatSoundEffectSchema)
+        .optional()
+        .describe("Sound effects mixed into the audio track during this beat. Multiple effects may overlap."),
     htmlPrompt: htmlPromptParamsSchema.optional(),
     enableLipSync: z.boolean().optional().describe("Enable lip sync generation for this beat"),
     hidden: z.boolean().optional().describe("Hide this beat from the presentation"),

package/lib/types/type.d.ts

@@ -1,6 +1,6 @@
 import { type CallbackFunction } from "graphai";
 import type { UsageCollectorAPI } from "./usage.js";
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoRemotionMediaSchema, remotionCodeSourceSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema, mulmoAvatarParamsSchema, avatarPositionSchema, speakerAvatarSchema, mulmoAvatarTrackSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoBeatSoundEffectSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoRemotionMediaSchema, remotionCodeSourceSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema, mulmoAvatarParamsSchema, avatarPositionSchema, speakerAvatarSchema, mulmoAvatarTrackSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "./const.js";
 import type { LLM } from "./provider2agent.js";
 import { z } from "zod";
@@ -43,6 +43,7 @@ export type MulmoStudioMultiLingualFile = z.infer<typeof mulmoStudioMultiLingual
 export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
 export type MulmoMovieParams = z.infer<typeof mulmoMovieParamsSchema>;
 export type MulmoSoundEffectParams = z.infer<typeof mulmoSoundEffectParamsSchema>;
+export type MulmoBeatSoundEffect = z.infer<typeof mulmoBeatSoundEffectSchema>;
 export type MulmoLipSyncParams = z.infer<typeof mulmoLipSyncParamsSchema>;
 export type MulmoAvatarParams = z.infer<typeof mulmoAvatarParamsSchema>;
 export type MulmoAvatarPosition = z.infer<typeof avatarPositionSchema>;

package/lib/utils/context.d.ts

@@ -2504,6 +2504,22 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
             soundEffectPrompt?: string | undefined;
+            soundEffects?: {
+                source: {
+                    kind: "url";
+                    url: string;
+                } | {
+                    kind: "path";
+                    path: string;
+                } | {
+                    kind: "jinglescript";
+                    score: Record<string, unknown>;
+                };
+                startAt: number;
+                volume: number;
+                duration?: number | undefined;
+                loop?: boolean | undefined;
+            }[] | undefined;
             htmlPrompt?: {
                 systemPrompt: string;
                 prompt: string;
@@ -5143,6 +5159,22 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 imagePrompt?: string | undefined;
                 moviePrompt?: string | undefined;
                 soundEffectPrompt?: string | undefined;
+                soundEffects?: {
+                    source: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    } | {
+                        kind: "jinglescript";
+                        score: Record<string, unknown>;
+                    };
+                    startAt: number;
+                    volume: number;
+                    duration?: number | undefined;
+                    loop?: boolean | undefined;
+                }[] | undefined;
                 htmlPrompt?: {
                     systemPrompt: string;
                     prompt: string;

package/lib/utils/jinglescript.d.ts

@@ -0,0 +1,12 @@
+import type { MulmoStudioContext } from "../types/index.js";
+type JingleScore = Record<string, unknown>;
+export declare const getJingleFilePath: (score: JingleScore, context: MulmoStudioContext) => string;
+export declare const getJingleScores: (context: MulmoStudioContext) => {
+    score: Record<string, unknown>;
+    beatIndex: number;
+    effectIndex: number;
+}[];
+export declare const validateJingleScores: (context: MulmoStudioContext) => void;
+export declare const renderJingleScores: (context: MulmoStudioContext) => Promise<void>;
+export declare const writeFileAtomically: (filePath: string, data: Uint8Array) => void;
+export {};

package/lib/utils/jinglescript.js

@@ -0,0 +1,63 @@
+import fs from "fs";
+import path from "path";
+import { GraphAILogger } from "graphai";
+import { checkScore, formatProblem, loadSamples, parseScore, render, toWav } from "jinglescript";
+import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
+import { hashSHA256, mkdir } from "./file.js";
+import { userAssert } from "./utils.js";
+// The audio mix (addBGMAgent) runs at 44.1 kHz.
+const JINGLE_SAMPLE_RATE = 44100;
+// Per download of a recorded sample (grandpiano: about 1.2 MB per note).
+const SAMPLE_FETCH_TIMEOUT_MS = 60000;
+// Rendered jingles are cached by the hash of their score, so identical scores share one file.
+export const getJingleFilePath = (score, context) => {
+    return path.join(MulmoStudioContextMethods.getAudioDirPath(context), `jingle_${hashSHA256(JSON.stringify(score))}.wav`);
+};
+export const getJingleScores = (context) => {
+    return context.studio.script.beats.flatMap((beat, beatIndex) => (beat.soundEffects ?? []).flatMap((soundEffect, effectIndex) => soundEffect.source.kind === "jinglescript" ? [{ score: soundEffect.source.score, beatIndex, effectIndex }] : []));
+};
+// Checks every JingleScript score in the script; throws listing every problem with its location.
+export const validateJingleScores = (context) => {
+    const problems = getJingleScores(context).flatMap(({ score, beatIndex, effectIndex }) => checkScore(score).errors.map((problem) => `beats[${beatIndex}].soundEffects[${effectIndex}].source.score: ${formatProblem(problem).trim()}`));
+    userAssert(problems.length === 0, `Invalid JingleScript score:\n${problems.join("\n")}`);
+};
+const fetchWithTimeout = (input, init) => fetch(input, { ...init, signal: AbortSignal.timeout(SAMPLE_FETCH_TIMEOUT_MS) });
+// Sampled instruments (grandpiano) play recordings that jinglescript downloads on first use and
+// caches; a score without them loads nothing.
+const loadJingleSamples = async (score, location) => {
+    try {
+        return await loadSamples(score, { fetch: fetchWithTimeout });
+    }
+    catch (error) {
+        throw new Error(`${location}: failed to load JingleScript samples: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
+    }
+};
+// Renders the JingleScript scores of the script to WAV files (skipping cached ones unless forced).
+export const renderJingleScores = async (context) => {
+    validateJingleScores(context);
+    for (const { score, beatIndex, effectIndex } of getJingleScores(context)) {
+        const filePath = getJingleFilePath(score, context);
+        if (!context.force && fs.existsSync(filePath)) {
+            continue;
+        }
+        const parsed = parseScore(score);
+        const samples = await loadJingleSamples(parsed, `beats[${beatIndex}].soundEffects[${effectIndex}].source.score`);
+        const { audio, sampleRate } = render(parsed, { sampleRate: JINGLE_SAMPLE_RATE, samples });
+        mkdir(path.dirname(filePath));
+        writeFileAtomically(filePath, toWav(audio, sampleRate));
+        GraphAILogger.info(`jinglescript: rendered ${filePath}`);
+    }
+};
+// Writes to a temporary file next to the target, then renames it, so an interrupted write never
+// leaves a partial WAV that a later run would take for a cached one.
+export const writeFileAtomically = (filePath, data) => {
+    const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+    try {
+        fs.writeFileSync(tmpPath, data);
+        fs.renameSync(tmpPath, filePath);
+    }
+    catch (error) {
+        fs.rmSync(tmpPath, { force: true });
+        throw error;
+    }
+};