mulmocast 2.9.0 → 2.9.2

package/lib/cli/tool-args.js

@@ -0,0 +1,64 @@
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { commonOptions } from "./common.js";
+import { getAvailableTemplates } from "../utils/file.js";
+import { llmAgents } from "../utils/utils.js";
+const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+export const args = commonOptions(yargs(hideBin(process.argv)))
+    .scriptName("mulmo-tool")
+    .option("u", {
+    alias: "url",
+    description: "URLs to reference (required when not in interactive mode)",
+    demandOption: false,
+    default: [],
+    type: "array",
+    string: true,
+})
+    .option("i", {
+    alias: "interactive",
+    description: "Generate script in interactive mode with user prompts",
+    demandOption: false,
+    type: "boolean",
+})
+    .option("t", {
+    alias: "template",
+    description: "Template name to use",
+    demandOption: false,
+    choices: availableTemplateNames,
+    type: "string",
+})
+    .option("c", {
+    alias: "cache",
+    description: "cache dir",
+    demandOption: false,
+    type: "string",
+})
+    .option("s", {
+    alias: "script",
+    description: "script filename",
+    demandOption: false,
+    default: "script",
+    type: "string",
+})
+    .option("llm_agent", {
+    description: "llm agent",
+    demandOption: false,
+    choices: llmAgents,
+    type: "string",
+})
+    .option("llm_model", {
+    description: "llm model",
+    demandOption: false,
+    type: "string",
+})
+    .command("$0 <action>", "Run mulmocast tool", (yargs) => {
+    return yargs.positional("action", {
+        describe: "action to perform",
+        choices: ["scripting", "prompt", "schema"],
+        type: "string",
+    });
+})
+    .strict()
+    .help()
+    .alias("help", "h")
+    .parseSync();

package/lib/cli/tool-cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import "dotenv/config";

package/lib/cli/tool-cli.js

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+import "dotenv/config";
+import yargs from "yargs/yargs";
+import { hideBin } from "yargs/helpers";
+import { GraphAILogger } from "graphai";
+import * as scriptingCmd from "./tool_commands/scripting/index.js";
+import * as promptCmd from "./tool_commands/prompt/index.js";
+import * as schemaCmd from "./tool_commands/schema/index.js";
+const cli = yargs(hideBin(process.argv))
+    .scriptName("mulmo-tool")
+    .usage("$0 <command> [args]")
+    .option("v", {
+    alias: "verbose",
+    describe: "verbose log",
+    demandOption: true,
+    default: false,
+    type: "boolean",
+})
+    .command(scriptingCmd)
+    .command(promptCmd)
+    .command(schemaCmd)
+    .demandCommand()
+    .strict()
+    .help()
+    .alias("help", "h");
+const main = async () => {
+    await cli.parseAsync();
+};
+main().catch((error) => {
+    GraphAILogger.info("An unexpected error occurred:", error);
+    process.exit(1);
+});

package/lib/cli/tool_commands/prompt/builder.d.ts

@@ -0,0 +1,4 @@
+import { Argv } from "yargs";
+export declare const builder: (yargs: Argv) => Argv<{
+    t: string | undefined;
+}>;

package/lib/cli/tool_commands/prompt/builder.js

@@ -0,0 +1,11 @@
+import { getAvailableTemplates } from "@/src/utils/file.js";
+const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+export const builder = (yargs) => {
+    return yargs.option("t", {
+        alias: "template",
+        describe: "Template name to use",
+        demandOption: false,
+        choices: availableTemplateNames,
+        type: "string",
+    });
+};

package/lib/cli/tool_commands/prompt/handler.d.ts

@@ -0,0 +1,4 @@
+import { ToolCliArgs } from "@/src/types/cli_types.js";
+export declare const handler: (argv: ToolCliArgs<{
+    template?: string;
+}>) => Promise<void>;

package/lib/cli/tool_commands/prompt/handler.js

@@ -0,0 +1,14 @@
+import { selectTemplate } from "@/src/utils/inquirer.js";
+import { dumpPromptFromTemplate } from "../../../tools/dump_prompt.js";
+import { setGraphAILogger } from "@/src/utils/cli.js";
+export const handler = async (argv) => {
+    let { template } = argv;
+    const { v: verbose } = argv;
+    if (!template) {
+        template = await selectTemplate();
+    }
+    setGraphAILogger(verbose, {
+        template,
+    });
+    await dumpPromptFromTemplate({ templateName: template });
+};

package/lib/cli/tool_commands/prompt/index.d.ts

@@ -0,0 +1,4 @@
+export declare const command = "prompt";
+export declare const desc = "Dump prompt from template";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/tool_commands/prompt/index.js

@@ -0,0 +1,4 @@
+export const command = "prompt";
+export const desc = "Dump prompt from template";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/tool_commands/schema/builder.d.ts

@@ -0,0 +1,2 @@
+import { Argv } from "yargs";
+export declare const builder: (yargs: Argv) => Argv<{}>;

package/lib/cli/tool_commands/schema/builder.js

@@ -0,0 +1,3 @@
+export const builder = (yargs) => {
+    return yargs;
+};

package/lib/cli/tool_commands/schema/handler.d.ts

@@ -0,0 +1,2 @@
+import { ToolCliArgs } from "@/src/types/cli_types.js";
+export declare const handler: (argv: ToolCliArgs) => Promise<void>;

package/lib/cli/tool_commands/schema/handler.js

@@ -0,0 +1,12 @@
+import { mulmoScriptSchema } from "@/src/types/schema.js";
+import { zodToJsonSchema } from "zod-to-json-schema";
+import { GraphAILogger } from "graphai";
+import { setGraphAILogger } from "@/src/utils/cli.js";
+export const handler = async (argv) => {
+    const { v: verbose } = argv;
+    setGraphAILogger(verbose);
+    const defaultSchema = zodToJsonSchema(mulmoScriptSchema, {
+        strictUnions: true,
+    });
+    GraphAILogger.info(JSON.stringify(defaultSchema, null, 2));
+};

package/lib/cli/tool_commands/schema/index.d.ts

@@ -0,0 +1,4 @@
+export declare const command = "schema";
+export declare const desc = "Dump mulmocast schema";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/tool_commands/schema/index.js

@@ -0,0 +1,4 @@
+export const command = "schema";
+export const desc = "Dump mulmocast schema";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/tool_commands/scripting/builder.d.ts

@@ -0,0 +1,20 @@
+import { Argv } from "yargs";
+export declare const builder: (yargs: Argv) => Argv<{
+    o: string | undefined;
+} & {
+    b: string | undefined;
+} & {
+    u: string[] | never[];
+} & {
+    i: boolean | undefined;
+} & {
+    t: string | undefined;
+} & {
+    c: string | undefined;
+} & {
+    s: string;
+} & {
+    llm_agent: string | undefined;
+} & {
+    llm_model: string | undefined;
+}>;

package/lib/cli/tool_commands/scripting/builder.js

@@ -0,0 +1,63 @@
+import { llmAgents } from "../../../utils/utils.js";
+import { getAvailableTemplates } from "../../../utils/file.js";
+const availableTemplateNames = getAvailableTemplates().map((template) => template.filename);
+export const builder = (yargs) => {
+    return yargs
+        .option("o", {
+        alias: "outdir",
+        description: "output dir",
+        demandOption: false,
+        type: "string",
+    })
+        .option("b", {
+        alias: "basedir",
+        description: "base dir",
+        demandOption: false,
+        type: "string",
+    })
+        .option("u", {
+        alias: "url",
+        description: "URLs to reference (required when not in interactive mode)",
+        demandOption: false,
+        default: [],
+        type: "array",
+        string: true,
+    })
+        .option("i", {
+        alias: "interactive",
+        description: "Generate script in interactive mode with user prompts",
+        demandOption: false,
+        type: "boolean",
+    })
+        .option("t", {
+        alias: "template",
+        description: "Template name to use",
+        demandOption: false,
+        choices: availableTemplateNames,
+        type: "string",
+    })
+        .option("c", {
+        alias: "cache",
+        description: "cache dir",
+        demandOption: false,
+        type: "string",
+    })
+        .option("s", {
+        alias: "script",
+        description: "script filename",
+        demandOption: false,
+        default: "script",
+        type: "string",
+    })
+        .option("llm_agent", {
+        description: "llm agent",
+        demandOption: false,
+        choices: llmAgents,
+        type: "string",
+    })
+        .option("llm_model", {
+        description: "llm model",
+        demandOption: false,
+        type: "string",
+    });
+};

package/lib/cli/tool_commands/scripting/handler.d.ts

@@ -0,0 +1,12 @@
+import { ToolCliArgs } from "@/src/types/cli_types.js";
+export declare const handler: (argv: ToolCliArgs<{
+    o?: string;
+    b?: string;
+    u?: string[];
+    i?: boolean;
+    t?: string;
+    c?: string;
+    s?: string;
+    llm_agent?: string;
+    llm_model?: string;
+}>) => Promise<void>;

package/lib/cli/tool_commands/scripting/handler.js

@@ -0,0 +1,36 @@
+import { getBaseDirPath, getFullPath } from "@/src/utils/file.js";
+import { outDirName, cacheDirName } from "../../../utils/const.js";
+import { getUrlsIfNeeded, selectTemplate } from "@/src/utils/inquirer.js";
+import { createMulmoScriptFromUrl } from "../../../tools/create_mulmo_script_from_url.js";
+import { createMulmoScriptInteractively } from "../../../tools/create_mulmo_script_interactively.js";
+import { setGraphAILogger } from "@/src/utils/cli.js";
+export const handler = async (argv) => {
+    const { o: outdir, b: basedir, v: verbose, i: interactive, c: cache, s: filename, llm_agent, llm_model } = argv;
+    let { t: template } = argv;
+    const urls = argv.u || [];
+    const baseDirPath = getBaseDirPath(basedir);
+    const outDirPath = getFullPath(baseDirPath, outdir ?? outDirName);
+    const cacheDirPath = getFullPath(outDirPath, cache ?? cacheDirName);
+    if (!template) {
+        template = await selectTemplate();
+    }
+    setGraphAILogger(verbose, {
+        baseDirPath,
+        outDirPath,
+        cacheDirPath,
+        template,
+        urls,
+        interactive,
+        filename,
+        llm_agent,
+        llm_model,
+    });
+    const context = { outDirPath, templateName: template, urls, filename: filename, cacheDirPath, llm_model, llm_agent };
+    if (interactive) {
+        await createMulmoScriptInteractively(context);
+    }
+    else {
+        context.urls = await getUrlsIfNeeded(urls);
+        await createMulmoScriptFromUrl(context);
+    }
+};

package/lib/cli/tool_commands/scripting/index.d.ts

@@ -0,0 +1,4 @@
+export declare const command = "scripting";
+export declare const desc = "Generate mulmocast script";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/cli/tool_commands/scripting/index.js

@@ -0,0 +1,4 @@
+export const command = "scripting";
+export const desc = "Generate mulmocast script";
+export { builder } from "./builder.js";
+export { handler } from "./handler.js";

package/lib/index.d.ts

@@ -0,0 +1,8 @@
+export * from "./actions/index.js";
+export * from "./cli/helpers.js";
+export * from "./utils/file.js";
+export * from "./utils/ffmpeg_utils.js";
+export * from "./utils/provider2agent.js";
+export * from "./methods/index.js";
+export * from "./agents/index.js";
+export * from "./types/index.js";

package/lib/index.js

@@ -0,0 +1,8 @@
+export * from "./actions/index.js";
+export * from "./cli/helpers.js";
+export * from "./utils/file.js";
+export * from "./utils/ffmpeg_utils.js";
+export * from "./utils/provider2agent.js";
+export * from "./methods/index.js";
+export * from "./agents/index.js";
+export * from "./types/index.js";

package/lib/methods/mulmo_beat.d.ts

@@ -8,6 +8,9 @@ export declare const MulmoBeatMethods: {
     isAnimationObject: (animation: unknown) => animation is AnimationConfig;
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
+    isVoiceOver: (beat?: MulmoBeat) => boolean;
+    getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
+    getNextRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
     getHtmlPrompt(beat: MulmoBeat): string | undefined;
     getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
 };

package/lib/methods/mulmo_beat.js

@@ -1,3 +1,4 @@
+import { ImageMediaType } from "../types/index.js";
 /** Type guard: checks if animation value is an object config like { fps: 30 } */
 const isAnimationObject = (animation) => {
     return typeof animation === "object" && animation !== null && !Array.isArray(animation);
@@ -17,11 +18,27 @@ const isAnimatedHtmlTailwind = (beat) => {
     const animation = beat.image.animation;
     return isAnimationEnabled(animation);
 };
+// voice_over beats share the preceding beat's shot, and contribute no video segment of their own.
+const isVoiceOver = (beat) => beat?.image?.type === ImageMediaType.VoiceOver;
 export const MulmoBeatMethods = {
     isAnimationEnabled,
     isAnimationObject,
     isAnimatedHtmlTailwind,
     isMovieMode,
+    isVoiceOver,
+    // Index of the closest preceding beat which is actually rendered as a video segment (-1 if none).
+    getPrevRenderedBeatIndex(beats, index) {
+        const offset = beats
+            .slice(0, Math.max(index, 0))
+            .reverse()
+            .findIndex((beat) => !isVoiceOver(beat));
+        return offset < 0 ? -1 : index - 1 - offset;
+    },
+    // Index of the closest following beat which is actually rendered as a video segment (-1 if none).
+    getNextRenderedBeatIndex(beats, index) {
+        const offset = beats.slice(index + 1).findIndex((beat) => !isVoiceOver(beat));
+        return offset < 0 ? -1 : index + 1 + offset;
+    },
     getHtmlPrompt(beat) {
         if (beat?.htmlPrompt?.data) {
             return beat.htmlPrompt.prompt + "\n\n[data]\n" + JSON.stringify(beat.htmlPrompt.data, null, 2);

package/lib/methods/mulmo_beat_node.d.ts

@@ -17,6 +17,9 @@ export declare const MulmoBeatMethods: {
     };
     isAnimatedHtmlTailwind: (beat: MulmoBeat) => boolean;
     isMovieMode: (animation: unknown) => boolean;
+    isVoiceOver: (beat?: MulmoBeat) => boolean;
+    getPrevRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
+    getNextRenderedBeatIndex(beats: MulmoBeat[], index: number): number;
     getHtmlPrompt(beat: MulmoBeat): string | undefined;
     getImageReferenceForImageGenerator(beat: MulmoBeat, imageRefs: Record<string, string>): string[];
 };

package/lib/methods/mulmo_presentation_style.d.ts

@@ -4,7 +4,7 @@
  * Works in both Node.js and modern browsers.
  */
 import type { SlideTheme } from "@mulmocast/deck";
-import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext, MulmoTransition } from "../types/index.js";
+import { MulmoCanvasDimension, MulmoBeat, Text2SpeechProvider, Text2ImageAgentInfo, Text2HtmlAgentInfo, BeatMediaType, MulmoPresentationStyle, SpeakerData, Text2ImageProvider, MulmoStudioContext, MulmoTransition, MulmoFillOption } from "../types/index.js";
 export declare const MulmoPresentationStyleMethods: {
     getCanvasSize(presentationStyle: MulmoPresentationStyle): MulmoCanvasDimension;
     getAllSpeechProviders(presentationStyle: MulmoPresentationStyle): Set<Text2SpeechProvider>;
@@ -30,6 +30,10 @@ export declare const MulmoPresentationStyleMethods: {
     getSpeakerData(presentationStyle: MulmoPresentationStyle, beat: MulmoBeat, lang: string | undefined): SpeakerData;
     getSpeaker(context: MulmoStudioContext, beat: MulmoBeat, targetLang: string | undefined): SpeakerData;
     getMovieTransition(context: MulmoStudioContext, beat: MulmoBeat): MulmoTransition | null;
+    getNeedFirstFrame(context: MulmoStudioContext): boolean[];
+    getNeedLastFrame(context: MulmoStudioContext): boolean[];
+    getFillOption(context: MulmoStudioContext, beat: MulmoBeat): MulmoFillOption;
+    getMovieVolume(context: MulmoStudioContext, beat: MulmoBeat): number;
     getText2ImageProvider(provider: Text2ImageProvider | undefined): Text2ImageProvider;
     getImageAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): Text2ImageAgentInfo;
     getMovieAgentInfo(presentationStyle: MulmoPresentationStyle, beat?: MulmoBeat): {

package/lib/methods/mulmo_presentation_style.js

@@ -6,7 +6,8 @@
 import { isNull } from "graphai";
 import { userAssert } from "../utils/utils.js";
 import { slideThemes } from "../data/slideThemes.js";
-import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, } from "../types/schema.js";
+import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, mulmoFillOptionSchema, } from "../types/schema.js";
+import { MulmoBeatMethods } from "./mulmo_beat.js";
 import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
 const defaultTextSlideStyles = [
     '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
@@ -22,6 +23,13 @@ const defaultTextSlideStyles = [
     "td, th { padding: 8px }",
     "tr:nth-child(even) { background-color: #eee }",
 ];
+const DEFAULT_DUCKING_RATIO = 0.3;
+// Both frame requirements answer the same question for each *rendered* beat (voice_over beats have
+// no video segment, so they never supply a frame): does a transition need a still from this beat?
+const mapRenderedBeats = (context, needsFrame) => {
+    const beats = context.studio.script.beats;
+    return beats.map((beat, index) => (MulmoBeatMethods.isVoiceOver(beat) ? false : needsFrame(beats, index)));
+};
 export const MulmoPresentationStyleMethods = {
     getCanvasSize(presentationStyle) {
         return mulmoCanvasDimensionSchema.parse(presentationStyle.canvasSize);
@@ -95,6 +103,40 @@ export const MulmoPresentationStyleMethods = {
             return null;
         return mulmoTransitionSchema.parse(transitionData);
     },
+    getNeedFirstFrame(context) {
+        // This beat slides/wipes in over the previous shot, so it supplies its own first frame.
+        return mapRenderedBeats(context, (beats, index) => {
+            if (MulmoBeatMethods.getPrevRenderedBeatIndex(beats, index) < 0)
+                return false; // No previous shot: the transition is skipped
+            const transition = MulmoPresentationStyleMethods.getMovieTransition(context, beats[index]);
+            return (transition?.type.startsWith("slidein_") || transition?.type.startsWith("wipe")) ?? false;
+        });
+    },
+    getNeedLastFrame(context) {
+        // Any transition consumes the last frame of the previous rendered beat, skipping voice_over beats.
+        return mapRenderedBeats(context, (beats, index) => {
+            const nextIndex = MulmoBeatMethods.getNextRenderedBeatIndex(beats, index);
+            if (nextIndex < 0)
+                return false; // Last rendered beat doesn't need _last
+            return MulmoPresentationStyleMethods.getMovieTransition(context, beats[nextIndex]) !== null;
+        });
+    },
+    getFillOption(context, beat) {
+        const globalFillOption = context.presentationStyle.movieParams?.fillOption;
+        const beatFillOption = beat.movieParams?.fillOption;
+        const defaultFillOption = mulmoFillOptionSchema.parse({}); // let the schema infer the default value
+        return { ...defaultFillOption, ...globalFillOption, ...beatFillOption };
+    },
+    getMovieVolume(context, beat) {
+        const baseMovieVolume = beat.audioParams?.movieVolume ?? context.presentationStyle.audioParams.movieVolume ?? 1.0;
+        const ducking = context.presentationStyle.audioParams.ducking;
+        const hasSpeech = !!beat.text && !context.presentationStyle.audioParams.suppressSpeech;
+        if (ducking && hasSpeech) {
+            const ratio = ducking.ratio ?? DEFAULT_DUCKING_RATIO;
+            return baseMovieVolume * ratio;
+        }
+        return baseMovieVolume;
+    },
     /* NOTE: This method is not used.
     getTTSModel(context: MulmoStudioContext, beat: MulmoBeat): string | undefined {
       const speaker = MulmoPresentationStyleMethods.getSpeaker(context, beat);

package/lib/methods/mulmo_script_template.d.ts

@@ -0,0 +1,4 @@
+import { MulmoScript, MulmoScriptTemplate } from "../types/index.js";
+export declare const MulmoScriptTemplateMethods: {
+    getSystemPrompt(template: MulmoScriptTemplate, script?: MulmoScript): string;
+};

package/lib/methods/mulmo_script_template.js

@@ -0,0 +1,6 @@
+import { getMulmoScriptTemplateSystemPrompt } from "../utils/prompt.js";
+export const MulmoScriptTemplateMethods = {
+    getSystemPrompt(template, script) {
+        return getMulmoScriptTemplateSystemPrompt(template, script);
+    },
+};

package/lib/methods/mulmo_studio.d.ts

@@ -0,0 +1,8 @@
+import { MulmoStudio } from "../types/index.js";
+type SessionType = "audio" | "image" | "video" | "multiLingual" | "caption" | "pdf";
+type BeatSessionType = "audio" | "image" | "multiLingual" | "caption" | "movie";
+export declare const MulmoStudioMethods: {
+    setSessionState(studio: MulmoStudio, sessionType: SessionType, value: boolean): void;
+    setBeatSessionState(studio: MulmoStudio, sessionType: BeatSessionType, index: number, value: boolean): void;
+};
+export {};

package/lib/methods/mulmo_studio.js

@@ -0,0 +1,25 @@
+import { GraphAILogger } from "graphai";
+const notifyStateChange = (studio, sessionType) => {
+    const prefix = studio.state.inSession[sessionType] ? "<" : " >";
+    GraphAILogger.info(`${prefix} ${sessionType}`);
+};
+const notifyBeatStateChange = (studio, sessionType, index) => {
+    const prefix = studio.state.inBeatSession[sessionType][index] ? "{" : " }";
+    GraphAILogger.info(`${prefix} ${sessionType} ${index}`);
+};
+export const MulmoStudioMethods = {
+    setSessionState(studio, sessionType, value) {
+        studio.state.inSession[sessionType] = value;
+        notifyStateChange(studio, sessionType);
+    },
+    setBeatSessionState(studio, sessionType, index, value) {
+        if (value) {
+            studio.state.inBeatSession[sessionType][index] = true;
+        }
+        else {
+            // NOTE: Setting to false causes the parse error in rebuildStudio in preprocess.ts
+            delete studio.state.inBeatSession[sessionType][index];
+        }
+        notifyBeatStateChange(studio, sessionType, index);
+    },
+};

package/lib/methods/mulmo_studio_context.d.ts

@@ -3,7 +3,7 @@
  * (No Node.js built-ins like fs, path, dotenv, etc.)
  * Works in both Node.js and modern browsers.
  */
-import { BeatSessionType, MulmoStudioContext, SessionProgressCallback, SessionType, MulmoBeat, SpeechOptions } from "../types/index.js";
+import { BeatSessionType, MulmoStudioContext, SessionProgressCallback, SessionType, MulmoBeat, MulmoStudioBeat, SpeechOptions } from "../types/index.js";
 import { provider2TTSAgent } from "../types/provider2agent.js";
 export declare const addSessionProgressCallback: (cb: SessionProgressCallback) => void;
 export declare const removeSessionProgressCallback: (cb: SessionProgressCallback) => void;
@@ -18,6 +18,11 @@ export declare const MulmoStudioContextMethods: {
     setBeatSessionState(context: MulmoStudioContext, sessionType: BeatSessionType | undefined, index: number, id: string | undefined, value: boolean): void;
     needTranslate(context: MulmoStudioContext, includeCaption?: boolean): boolean | "" | undefined;
     getIntroPadding(context: MulmoStudioContext): number;
+    getExtraPadding(context: MulmoStudioContext, index: number): number;
+    getBeatDuration(context: MulmoStudioContext, index: number): number;
+    getVoiceOverGroupDuration(context: MulmoStudioContext, index: number): number;
+    isMovieBeat(context: MulmoStudioContext, studioBeat: MulmoStudioBeat, beat: MulmoBeat): boolean;
+    getSegmentDuration(context: MulmoStudioContext, index: number): number;
     getAudioParam(context: MulmoStudioContext, beat: MulmoBeat, lang?: string): {
         provider: keyof typeof provider2TTSAgent;
         voiceId: string;

package/lib/methods/mulmo_studio_context.js

@@ -3,10 +3,11 @@
  * (No Node.js built-ins like fs, path, dotenv, etc.)
  * Works in both Node.js and modern browsers.
  */
-import { text2SpeechProviderSchema, } from "../types/index.js";
+import { ImageMediaType, text2SpeechProviderSchema, } from "../types/index.js";
 import { beatId } from "../utils/utils.js";
 import { GraphAILogger } from "graphai";
 import { MulmoPresentationStyleMethods } from "./mulmo_presentation_style.js";
+import { MulmoBeatMethods } from "./mulmo_beat.js";
 const sessionProgressCallbacks = new Set();
 export const addSessionProgressCallback = (cb) => {
     sessionProgressCallbacks.add(cb);
@@ -92,6 +93,40 @@ export const MulmoStudioContextMethods = {
         }
         return context.presentationStyle.audioParams.introPadding;
     },
+    // Intro/outro padding is the only padding not already folded into studio.beats[].duration.
+    getExtraPadding(context, index) {
+        if (index === 0) {
+            return MulmoStudioContextMethods.getIntroPadding(context);
+        }
+        else if (index === context.studio.beats.length - 1) {
+            return context.presentationStyle.audioParams.outroPadding;
+        }
+        return 0;
+    },
+    // The duration this beat occupies on the audio timeline.
+    getBeatDuration(context, index) {
+        return (context.studio.beats[index]?.duration ?? 0) + MulmoStudioContextMethods.getExtraPadding(context, index);
+    },
+    // The total duration of the voice_over beats which follow (and share the shot of) this beat.
+    getVoiceOverGroupDuration(context, index) {
+        const trailingBeats = context.studio.script.beats.slice(index + 1);
+        const groupEnd = trailingBeats.findIndex((beat) => !MulmoBeatMethods.isVoiceOver(beat));
+        const groupSize = groupEnd < 0 ? trailingBeats.length : groupEnd;
+        return trailingBeats.slice(0, groupSize).reduce((total, _, offset) => total + MulmoStudioContextMethods.getBeatDuration(context, index + 1 + offset), 0);
+    },
+    // Whether this beat's source is a movie (as opposed to a still which has to be looped).
+    isMovieBeat(context, studioBeat, beat) {
+        return !!(studioBeat.lipSyncFile ||
+            studioBeat.movieFile ||
+            MulmoPresentationStyleMethods.getImageType(context.presentationStyle, beat) === ImageMediaType.Movie);
+    },
+    // How long this beat's video segment is: its own duration plus the trailing voice_over beats which
+    // share its shot (they have no video segment of their own), never shorter than its movie. This is
+    // the single rule for a segment's length -- createVideo builds from it and transitions clamp against it.
+    getSegmentDuration(context, index) {
+        const ownDuration = MulmoStudioContextMethods.getBeatDuration(context, index) + MulmoStudioContextMethods.getVoiceOverGroupDuration(context, index);
+        return Math.max(ownDuration, context.studio.beats[index]?.movieDuration ?? 0);
+    },
     getAudioParam(context, beat, lang) {
         const speaker = MulmoPresentationStyleMethods.getSpeaker(context, beat, lang);
         const speechOptions = { ...speaker.speechOptions, ...beat.speechOptions };

package/lib/slide/blocks.d.ts

@@ -0,0 +1,13 @@
+import type { ContentBlock, TableCellValue } from "./schema.js";
+export declare const resolveCellColor: (cellObj: {
+    color?: string;
+}, isRowHeader: boolean) => string;
+export declare const renderBadge: (text: string, color: string) => string;
+export declare const renderCellValue: (cell: TableCellValue, isRowHeader: boolean) => string;
+export declare const renderTableCore: (headers: string[] | undefined, rows: TableCellValue[][], rowHeaders?: boolean, striped?: boolean) => string;
+/** Render a single content block to HTML */
+export declare const renderContentBlock: (block: ContentBlock) => string;
+/** Render an array of content blocks to HTML */
+export declare const renderContentBlocks: (blocks: ContentBlock[]) => string;
+/** Render content blocks with fixed aspect-ratio container for image blocks (used in card layouts) */
+export declare const renderCardContentBlocks: (blocks: ContentBlock[]) => string;