mulmocast 2.17.0 → 2.19.0

package/lib/types/schema.d.ts

@@ -4985,6 +4985,22 @@ export declare const mulmoMovieParamsSchema: z.ZodObject<{
     concurrency: z.ZodOptional<z.ZodNumber>;
     generateAudio: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
+export declare const mulmoBeatSoundEffectSchema: z.ZodObject<{
+    source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodURL;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"jinglescript">;
+        score: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    }, z.core.$strict>], "kind">;
+    startAt: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    volume: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    duration: z.ZodOptional<z.ZodNumber>;
+    loop: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strict>;
 export declare const mulmoBeatSchema: z.ZodObject<{
     speaker: z.ZodOptional<z.ZodString>;
     text: z.ZodDefault<z.ZodOptional<z.ZodString>>;
@@ -9086,6 +9102,22 @@ export declare const mulmoBeatSchema: z.ZodObject<{
     imagePrompt: z.ZodOptional<z.ZodString>;
     moviePrompt: z.ZodOptional<z.ZodString>;
     soundEffectPrompt: z.ZodOptional<z.ZodString>;
+    soundEffects: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodURL;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"jinglescript">;
+            score: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strict>], "kind">;
+        startAt: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        volume: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        duration: z.ZodOptional<z.ZodNumber>;
+        loop: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>>>;
     htmlPrompt: z.ZodOptional<z.ZodObject<{
         systemPrompt: z.ZodDefault<z.ZodOptional<z.ZodString>>;
         prompt: z.ZodString;
@@ -14353,6 +14385,22 @@ export declare const mulmoScriptSchema: z.ZodObject<{
         imagePrompt: z.ZodOptional<z.ZodString>;
         moviePrompt: z.ZodOptional<z.ZodString>;
         soundEffectPrompt: z.ZodOptional<z.ZodString>;
+        soundEffects: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"url">;
+                url: z.ZodURL;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"path">;
+                path: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"jinglescript">;
+                score: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+            }, z.core.$strict>], "kind">;
+            startAt: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+            volume: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+            duration: z.ZodOptional<z.ZodNumber>;
+            loop: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>>>;
         htmlPrompt: z.ZodOptional<z.ZodObject<{
             systemPrompt: z.ZodDefault<z.ZodOptional<z.ZodString>>;
             prompt: z.ZodString;
@@ -19144,6 +19192,22 @@ export declare const mulmoStudioSchema: z.ZodObject<{
             imagePrompt: z.ZodOptional<z.ZodString>;
             moviePrompt: z.ZodOptional<z.ZodString>;
             soundEffectPrompt: z.ZodOptional<z.ZodString>;
+            soundEffects: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"url">;
+                    url: z.ZodURL;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"path">;
+                    path: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"jinglescript">;
+                    score: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+                }, z.core.$strict>], "kind">;
+                startAt: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+                volume: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+                duration: z.ZodOptional<z.ZodNumber>;
+                loop: z.ZodOptional<z.ZodBoolean>;
+            }, z.core.$strict>>>;
             htmlPrompt: z.ZodOptional<z.ZodObject<{
                 systemPrompt: z.ZodDefault<z.ZodOptional<z.ZodString>>;
                 prompt: z.ZodString;

package/lib/types/schema.js

@@ -612,6 +612,44 @@ export const mulmoMovieParamsSchema = z.object({
     concurrency: z.number().int().positive().optional().describe("Max concurrent movie generation requests"),
     generateAudio: z.boolean().optional().describe("Request audio generation in the video (model-dependent)"),
 });
+export const mulmoBeatSoundEffectSchema = z
+    .object({
+    source: z
+        .discriminatedUnion("kind", [
+        z.object({ kind: z.literal("url"), url: URLStringSchema }).strict(),
+        z.object({ kind: z.literal("path"), path: z.string().min(1) }).strict(),
+        z
+            .object({
+            kind: z.literal("jinglescript"),
+            score: z.record(z.string(), z.unknown()).describe("JingleScript score (format jinglescript/1), rendered to audio when the audio is generated"),
+        })
+            .strict(),
+    ])
+        .describe("Audio file of the sound effect (url or path), or a JingleScript score to synthesize"),
+    startAt: z
+        .number()
+        .min(0)
+        .optional()
+        .default(0)
+        .describe("Seconds from when the beat appears on screen (the same clock as its html animation) at which the sound effect plays. The first beat's narration starts audioParams.introPadding later"),
+    volume: z
+        .number()
+        .min(0)
+        .max(4)
+        .optional()
+        .default(1.0)
+        .describe("Volume of the sound effect. 1.0 is the gain of the narration at its default volume; audioParams.audioVolume and ttsVolume do not affect sound effects"),
+    duration: z
+        .number()
+        .positive()
+        .optional()
+        .describe("Maximum play length in seconds; the effect is cut (with a short fade-out) after it. Default: the whole file, even past the end of the beat"),
+    loop: z
+        .boolean()
+        .optional()
+        .describe("Repeat the effect until duration, or until the beat leaves the screen when duration is not set (e.g. a ticking clock)"),
+})
+    .strict();
 export const mulmoBeatSchema = z
     .object({
     speaker: speakerIdSchema.optional(),
@@ -643,6 +681,10 @@ export const mulmoBeatSchema = z
     imagePrompt: z.string().optional(),
     moviePrompt: z.string().optional(),
     soundEffectPrompt: z.string().optional(),
+    soundEffects: z
+        .array(mulmoBeatSoundEffectSchema)
+        .optional()
+        .describe("Sound effects mixed into the audio track during this beat. Multiple effects may overlap."),
     htmlPrompt: htmlPromptParamsSchema.optional(),
     enableLipSync: z.boolean().optional().describe("Enable lip sync generation for this beat"),
     hidden: z.boolean().optional().describe("Hide this beat from the presentation"),

package/lib/types/type.d.ts

@@ -1,6 +1,6 @@
 import { type CallbackFunction } from "graphai";
 import type { UsageCollectorAPI } from "./usage.js";
-import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoRemotionMediaSchema, remotionCodeSourceSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema, mulmoAvatarParamsSchema, avatarPositionSchema, speakerAvatarSchema, mulmoAvatarTrackSchema } from "./schema.js";
+import { langSchema, localizedTextSchema, mulmoBeatSchema, mulmoScriptSchema, mulmoStudioSchema, mulmoStudioBeatSchema, mulmoStoryboardSchema, mulmoStoryboardSceneSchema, mulmoStudioMultiLingualSchema, mulmoStudioMultiLingualArraySchema, mulmoStudioMultiLingualDataSchema, mulmoStudioMultiLingualFileSchema, speakerDictionarySchema, speakerSchema, mulmoSpeechParamsSchema, mulmoImageParamsSchema, mulmoImageParamsImagesValueSchema, mulmoImageParamsImagesSchema, mulmoFillOptionSchema, mulmoTransitionSchema, mulmoVideoFilterSchema, mulmoMovieParamsSchema, mulmoSoundEffectParamsSchema, mulmoBeatSoundEffectSchema, mulmoLipSyncParamsSchema, textSlideParamsSchema, speechOptionsSchema, speakerDataSchema, mulmoCanvasDimensionSchema, mulmoPromptTemplateSchema, mulmoPromptTemplateFileSchema, text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoPresentationStyleSchema, multiLingualTextsSchema, mulmoImageAssetSchema, mulmoMermaidMediaSchema, mulmoRemotionMediaSchema, remotionCodeSourceSchema, mulmoTextSlideMediaSchema, mulmoMarkdownMediaSchema, mulmoImageMediaSchema, mulmoChartMediaSchema, mediaSourceSchema, mediaSourceMermaidSchema, backgroundImageSchema, backgroundImageSourceSchema, mulmoSessionStateSchema, mulmoOpenAIImageModelSchema, mulmoGoogleImageModelSchema, mulmoGoogleMovieModelSchema, mulmoReplicateMovieModelSchema, mulmoImagePromptMediaSchema, mulmoImageReferenceSchema, mulmoMovieMediaSchema, mulmoMoviePromptMediaSchema, markdownLayoutSchema, row2Schema, grid2x2Schema, mulmoAvatarParamsSchema, avatarPositionSchema, speakerAvatarSchema, mulmoAvatarTrackSchema } from "./schema.js";
 import { pdf_modes, pdf_sizes, storyToScriptGenerateMode } from "./const.js";
 import type { LLM } from "./provider2agent.js";
 import { z } from "zod";
@@ -43,6 +43,7 @@ export type MulmoStudioMultiLingualFile = z.infer<typeof mulmoStudioMultiLingual
 export type MultiLingualTexts = z.infer<typeof multiLingualTextsSchema>;
 export type MulmoMovieParams = z.infer<typeof mulmoMovieParamsSchema>;
 export type MulmoSoundEffectParams = z.infer<typeof mulmoSoundEffectParamsSchema>;
+export type MulmoBeatSoundEffect = z.infer<typeof mulmoBeatSoundEffectSchema>;
 export type MulmoLipSyncParams = z.infer<typeof mulmoLipSyncParamsSchema>;
 export type MulmoAvatarParams = z.infer<typeof mulmoAvatarParamsSchema>;
 export type MulmoAvatarPosition = z.infer<typeof avatarPositionSchema>;
@@ -77,6 +78,7 @@ export type MulmoStudioContext = {
     presentationStyle: MulmoPresentationStyle;
     multiLingual: MulmoStudioMultiLingualArray;
     usageCollector?: UsageCollectorAPI;
+    strictNetwork?: boolean;
 };
 export type ScriptingParams = {
     urls: string[];
@@ -162,6 +164,7 @@ export type InitOptions = {
     c?: string;
     p?: string;
     g?: boolean;
+    strictNetwork?: boolean;
 };
 export type PublicAPIArgs = {
     settings?: Record<string, string>;

package/lib/utils/context.d.ts

@@ -2504,6 +2504,22 @@ export declare const createStudioData: (_mulmoScript: MulmoScript, fileName: str
             imagePrompt?: string | undefined;
             moviePrompt?: string | undefined;
             soundEffectPrompt?: string | undefined;
+            soundEffects?: {
+                source: {
+                    kind: "url";
+                    url: string;
+                } | {
+                    kind: "path";
+                    path: string;
+                } | {
+                    kind: "jinglescript";
+                    score: Record<string, unknown>;
+                };
+                startAt: number;
+                volume: number;
+                duration?: number | undefined;
+                loop?: boolean | undefined;
+            }[] | undefined;
             htmlPrompt?: {
                 systemPrompt: string;
                 prompt: string;
@@ -5143,6 +5159,22 @@ export declare const initializeContextFromFiles: (files: FileObject, raiseError:
                 imagePrompt?: string | undefined;
                 moviePrompt?: string | undefined;
                 soundEffectPrompt?: string | undefined;
+                soundEffects?: {
+                    source: {
+                        kind: "url";
+                        url: string;
+                    } | {
+                        kind: "path";
+                        path: string;
+                    } | {
+                        kind: "jinglescript";
+                        score: Record<string, unknown>;
+                    };
+                    startAt: number;
+                    volume: number;
+                    duration?: number | undefined;
+                    loop?: boolean | undefined;
+                }[] | undefined;
                 htmlPrompt?: {
                     systemPrompt: string;
                     prompt: string;

package/lib/utils/estimate_usage.js

@@ -4,11 +4,12 @@
 // assume every asset is generated fresh.
 import { Tiktoken } from "js-tiktoken/lite";
 import o200k_base from "js-tiktoken/ranks/o200k_base";
-import { provider2TTSAgent, provider2MovieAgent, gptImageOutputTokens, isGptImage25Model, getModelDuration, defaultMovieModel, veoExtendedSeconds, VEO_EXTENSION_MODEL, VEO_SEGMENT_SEC, getModelPricing, defaultProviders, } from "../types/provider2agent.js";
+import { provider2TTSAgent, provider2MovieAgent, gptImageOutputTokens, isGptImage25Model, getModelDuration, defaultMovieModel, getModelPricing, defaultProviders, } from "../types/provider2agent.js";
 import { text2SpeechProviderSchema, text2MovieProviderSchema } from "../types/schema.js";
 import { MulmoPresentationStyleMethods } from "../methods/mulmo_presentation_style.js";
 import { MulmoBeatMethods } from "../methods/mulmo_beat.js";
 import { geminiTtsInputText } from "./gemini_tts.js";
+import { geminiOmniTotalSec, isGeminiOmniVideoModel } from "./gemini_omni_video.js";
 import { imagePrompt, htmlImageSystemPrompt, translateSystemPrompt, translatePrompts } from "./prompt.js";
 const MILLION = 1_000_000;
 const LATIN_CHARS_PER_TOKEN = 4;
@@ -217,8 +218,8 @@ const beatDurationMetric = (beat) => {
     }
     return estimated(beat.text ? estimateSpeechSec(beat.text) : DEFAULT_MOVIE_DURATION_SEC);
 };
-// The movie agent extends these clips past one segment instead of capping them, and Google bills every generated second.
-const isVeoExtension = (provider, model, duration) => provider === "google" && model === VEO_EXTENSION_MODEL && duration.value > VEO_SEGMENT_SEC;
+// Gemini Omni beats past one segment are extended rather than capped, and Google bills every generated second.
+const isGeminiOmniMovie = (provider, model) => provider === "google" && isGeminiOmniVideoModel(model);
 const snapMovieDuration = (provider, model, duration) => {
     if (!isKeyOf(provider2MovieAgent[provider].modelParams, model)) {
         return duration;
@@ -234,8 +235,8 @@ const estimateMovieGeneration = (style, beat, beatIndex, refKey) => {
     }
     const model = info.movieParams?.model ?? defaultMovieModel(provider, info.movieParams);
     const duration = beat ? beatDurationMetric(beat) : estimated(DEFAULT_MOVIE_DURATION_SEC);
-    const predictSec = isVeoExtension(provider, model, duration)
-        ? metric(veoExtendedSeconds(duration.value), duration.precision === "exact")
+    const predictSec = isGeminiOmniMovie(provider, model)
+        ? metric(geminiOmniTotalSec(duration.value), duration.precision === "exact")
         : snapMovieDuration(provider, model, duration);
     return attachCost({ process: refKey === undefined ? "movie" : "movieReference", beatIndex, refKey, provider, model, predictSec });
 };

package/lib/utils/gemini_omni_video.d.ts

@@ -0,0 +1,49 @@
+export declare const GEMINI_OMNI_MIN_SEGMENT_SEC = 3;
+export declare const GEMINI_OMNI_MAX_SEGMENT_SEC = 10;
+export declare const GEMINI_OMNI_MAX_TOTAL_SEC = 40;
+export declare const GEMINI_OMNI_RESOLUTION = "720p";
+export declare const isGeminiOmniVideoModel: (model: string) => boolean;
+export declare const geminiOmniTotalSec: (requestedSec: number) => number;
+export declare const geminiOmniSegments: (requestedSec: number) => number[];
+export declare const geminiOmniAspectRatio: (canvasSize: {
+    width: number;
+    height: number;
+}) => string;
+export type GeminiOmniImage = {
+    data: string;
+    mime_type: string;
+};
+export type GeminiOmniVideoRequestInput = {
+    model: string;
+    prompt: string;
+    aspectRatio: string;
+    durationSec: number;
+    frames?: GeminiOmniImage[];
+    previousInteractionId?: string;
+};
+export declare const geminiOmniExtensionPrompt: (prompt: string) => string;
+export declare const buildGeminiOmniVideoRequest: ({ model, prompt, aspectRatio, durationSec, frames, previousInteractionId }: GeminiOmniVideoRequestInput) => {
+    response_format: {
+        type: "video";
+        aspect_ratio: string;
+        resolution: string;
+        duration: string;
+    };
+    previous_interaction_id?: string | undefined;
+    model: string;
+    input: string | ({
+        data: string;
+        mime_type: string;
+        type: "image";
+    } | {
+        type: "text";
+        text: string;
+    })[];
+};
+export type GeminiOmniVideoRequest = ReturnType<typeof buildGeminiOmniVideoRequest>;
+type InteractionVideo = {
+    data?: string;
+    mime_type?: string;
+};
+export declare const videoFromInteraction: (video: InteractionVideo | undefined) => Buffer;
+export {};

package/lib/utils/gemini_omni_video.js

@@ -0,0 +1,36 @@
+import { getAspectRatio } from "./utils.js";
+// Gemini Omni video goes through the Interactions API: one call makes 3 to 10 whole seconds, and a call that
+// names the previous interaction extends that video, up to 40 seconds in all.
+export const GEMINI_OMNI_MIN_SEGMENT_SEC = 3;
+export const GEMINI_OMNI_MAX_SEGMENT_SEC = 10;
+export const GEMINI_OMNI_MAX_TOTAL_SEC = 40;
+export const GEMINI_OMNI_RESOLUTION = "720p";
+const GEMINI_OMNI_ASPECT_RATIOS = ["16:9", "9:16"];
+export const isGeminiOmniVideoModel = (model) => model.startsWith("gemini-omni-");
+export const geminiOmniTotalSec = (requestedSec) => {
+    if (!Number.isFinite(requestedSec))
+        throw new RangeError(`Invalid video duration: ${requestedSec}`);
+    return Math.min(Math.max(Math.ceil(requestedSec), GEMINI_OMNI_MIN_SEGMENT_SEC), GEMINI_OMNI_MAX_TOTAL_SEC);
+};
+// Split evenly so no segment falls under the 3-second minimum (11s is 6 + 5, not 10 + 1).
+export const geminiOmniSegments = (requestedSec) => {
+    const totalSec = geminiOmniTotalSec(requestedSec);
+    const count = Math.ceil(totalSec / GEMINI_OMNI_MAX_SEGMENT_SEC);
+    const baseSec = Math.floor(totalSec / count);
+    return Array.from({ length: count }, (_, index) => baseSec + (index < totalSec % count ? 1 : 0));
+};
+export const geminiOmniAspectRatio = (canvasSize) => getAspectRatio(canvasSize, GEMINI_OMNI_ASPECT_RATIOS);
+export const geminiOmniExtensionPrompt = (prompt) => `The scene continues: ${prompt}`;
+export const buildGeminiOmniVideoRequest = ({ model, prompt, aspectRatio, durationSec, frames = [], previousInteractionId }) => ({
+    model,
+    input: frames.length > 0 ? [...frames.map((frame) => ({ type: "image", ...frame })), { type: "text", text: prompt }] : prompt,
+    ...(previousInteractionId ? { previous_interaction_id: previousInteractionId } : {}),
+    response_format: { type: "video", aspect_ratio: aspectRatio, resolution: GEMINI_OMNI_RESOLUTION, duration: `${durationSec}s` },
+});
+export const videoFromInteraction = (video) => {
+    if (!video?.data)
+        throw new Error("No video data returned");
+    if (video.mime_type !== "video/mp4")
+        throw new Error(`Gemini Omni returned ${video.mime_type ?? "video of an unknown type"}, expected video/mp4`);
+    return Buffer.from(video.data, "base64");
+};

package/lib/utils/html_render.d.ts

@@ -1,4 +1,5 @@
-export declare const renderHTMLToImage: (html: string, outputPath: string, width: number, height: number, isMermaid?: boolean, omitBackground?: boolean) => Promise<void>;
+import { type RenderNetworkOptions } from "./render_network_policy.js";
+export declare const renderHTMLToImage: (html: string, outputPath: string, width: number, height: number, isMermaid?: boolean, omitBackground?: boolean, network?: RenderNetworkOptions) => Promise<void>;
 /**
  * Render an animated HTML page frame-by-frame using Puppeteer.
  *
@@ -9,18 +10,18 @@ export declare const renderHTMLToImage: (html: string, outputPath: string, width
  *
  * The user-defined render() function may be sync or async.
  */
-export declare const renderHTMLToFrames: (html: string, outputDir: string, width: number, height: number, totalFrames: number, fps: number) => Promise<string[]>;
+export declare const renderHTMLToFrames: (html: string, outputDir: string, width: number, height: number, totalFrames: number, fps: number, network?: RenderNetworkOptions) => Promise<string[]>;
 /**
  * Record an animated HTML page as video using Puppeteer's screencast API.
  * The animation plays in real-time via requestAnimationFrame, and
  * page.screencast() captures frames directly to an mp4 file.
  */
-export declare const renderHTMLToVideo: (html: string, videoPath: string, width: number, height: number, totalFrames: number, fps: number) => Promise<void>;
+export declare const renderHTMLToVideo: (html: string, videoPath: string, width: number, height: number, totalFrames: number, fps: number, network?: RenderNetworkOptions) => Promise<void>;
 /**
  * Render the final frame of an animated HTML page as a static image.
  * Loads the animated HTML, calls window.renderFinal() to set all animations
  * to their end state, then takes a screenshot. Used for PDF/thumbnail generation.
  */
-export declare const renderHTMLToFinalFrame: (html: string, outputPath: string, width: number, height: number) => Promise<void>;
-export declare const renderMarkdownToImage: (markdown: string, style: string, outputPath: string, width: number, height: number) => Promise<void>;
+export declare const renderHTMLToFinalFrame: (html: string, outputPath: string, width: number, height: number, network?: RenderNetworkOptions) => Promise<void>;
+export declare const renderMarkdownToImage: (markdown: string, style: string, outputPath: string, width: number, height: number, network?: RenderNetworkOptions) => Promise<void>;
 export declare const interpolate: (template: string, data: Record<string, string>) => string;

package/lib/utils/html_render.js

@@ -4,6 +4,8 @@ import nodePath from "node:path";
 import crypto from "node:crypto";
 import { marked } from "marked";
 import puppeteer from "puppeteer";
+import { guardRenderPage } from "./render_network_guard.js";
+import { strictNetworkLaunchArgs, withRenderContentSecurityPolicy } from "./render_network_policy.js";
 import { GraphAILogger } from "graphai";
 const isCI = process.env.CI === "true";
 const VIDEO_LOAD_TIMEOUT_MS = 15000;
@@ -83,15 +85,16 @@ const resolveWaitUntil = (html) => {
  * and navigate via page.goto (setContent uses about:blank origin
  * which blocks file:// loading).
  */
-const loadHtmlIntoPage = async (page, html, timeout_ms) => {
+const renderTempPagePath = () => nodePath.join(os.tmpdir(), `mulmocast_render_${crypto.randomUUID()}.html`);
+const loadHtmlIntoPage = async (page, html, timeout_ms, forcedFilePath) => {
     const waitUntil = resolveWaitUntil(html);
     const hasFileUrls = /file:\/\//.test(html);
     const hasExternalScripts = /script src=["']https?:\/\//.test(html);
     // puppeteer's setContent does not accept "networkidle0" (no navigation, so network-idle states aren't meaningful in types).
     // Route any HTML that needs network-idle semantics through the page.goto path to preserve the original wait behavior.
     const needsNetworkIdle = waitUntil === "networkidle0";
-    if (hasFileUrls || hasExternalScripts || needsNetworkIdle) {
-        const tmpFile = nodePath.join(os.tmpdir(), `mulmocast_render_${crypto.randomUUID()}.html`);
+    if (forcedFilePath || hasFileUrls || hasExternalScripts || needsNetworkIdle) {
+        const tmpFile = forcedFilePath ?? renderTempPagePath();
         fs.writeFileSync(tmpFile, html);
         try {
             await page.goto(`file://${tmpFile}`, { waitUntil, timeout: timeout_ms });
@@ -109,14 +112,28 @@ const loadHtmlIntoPage = async (page, html, timeout_ms) => {
         await page.setContent(html, { waitUntil, timeout: timeout_ms });
     }
 };
-export const renderHTMLToImage = async (html, outputPath, width, height, isMermaid = false, omitBackground = false) => {
+const RENDER_LOAD_TIMEOUT_MS = 30000;
+const renderLaunchArgs = ({ strictNetwork = false }) => [
+    ...(isCI ? ["--no-sandbox"] : []),
+    "--allow-file-access-from-files",
+    ...(strictNetwork ? strictNetworkLaunchArgs() : []),
+];
+// Strict mode always loads through a file: the guard's WebRTC removal never reaches a setContent document.
+const loadGuardedPage = async (page, html, allowedFileRoots) => {
+    const pagePath = renderTempPagePath();
+    await guardRenderPage(page, [pagePath, ...allowedFileRoots]);
+    await loadHtmlIntoPage(page, withRenderContentSecurityPolicy(html), RENDER_LOAD_TIMEOUT_MS, pagePath);
+};
+const openRenderPage = async (browser, html, { strictNetwork = false, allowedFileRoots = [] }) => {
+    const page = await browser.newPage();
+    await (strictNetwork ? loadGuardedPage(page, html, allowedFileRoots) : loadHtmlIntoPage(page, html, RENDER_LOAD_TIMEOUT_MS));
+    return page;
+};
+export const renderHTMLToImage = async (html, outputPath, width, height, isMermaid = false, omitBackground = false, network = {}) => {
     // Use Puppeteer to render HTML to an image
-    const browser = await puppeteer.launch({
-        args: isCI ? ["--no-sandbox", "--allow-file-access-from-files"] : ["--allow-file-access-from-files"],
-    });
+    const browser = await puppeteer.launch({ args: renderLaunchArgs(network) });
     try {
-        const page = await browser.newPage();
-        await loadHtmlIntoPage(page, html, 30000);
+        const page = await openRenderPage(browser, html, network);
         // Adjust page settings if needed (like width, height, etc.)
         await page.setViewport({ width, height });
         // height:100% ensures background fills viewport; only reset html, let body styles come from custom CSS
@@ -158,14 +175,10 @@ export const renderHTMLToImage = async (html, outputPath, width, height, isMerma
  *
  * The user-defined render() function may be sync or async.
  */
-export const renderHTMLToFrames = async (html, outputDir, width, height, totalFrames, fps) => {
-    const browser = await puppeteer.launch({
-        args: isCI ? ["--no-sandbox", "--allow-file-access-from-files"] : ["--allow-file-access-from-files"],
-    });
+export const renderHTMLToFrames = async (html, outputDir, width, height, totalFrames, fps, network = {}) => {
+    const browser = await puppeteer.launch({ args: renderLaunchArgs(network) });
     try {
-        const page = await browser.newPage();
-        // Wait for Tailwind CSS CDN to load
-        await loadHtmlIntoPage(page, html, 30000);
+        const page = await openRenderPage(browser, html, network);
         await page.setViewport({ width, height });
         await page.addStyleTag({ content: "html{height:100%;margin:0;padding:0;overflow:hidden}" });
         // Scale content to fit viewport (same logic as renderHTMLToImage)
@@ -199,14 +212,11 @@ export const renderHTMLToFrames = async (html, outputDir, width, height, totalFr
  * The animation plays in real-time via requestAnimationFrame, and
  * page.screencast() captures frames directly to an mp4 file.
  */
-export const renderHTMLToVideo = async (html, videoPath, width, height, totalFrames, fps) => {
+export const renderHTMLToVideo = async (html, videoPath, width, height, totalFrames, fps, network = {}) => {
     const duration_ms = (totalFrames / fps) * 1000;
-    const browser = await puppeteer.launch({
-        args: isCI ? ["--no-sandbox", "--allow-file-access-from-files"] : ["--allow-file-access-from-files"],
-    });
+    const browser = await puppeteer.launch({ args: renderLaunchArgs(network) });
     try {
-        const page = await browser.newPage();
-        await loadHtmlIntoPage(page, html, 30000);
+        const page = await openRenderPage(browser, html, network);
         await page.setViewport({ width, height });
         await page.addStyleTag({ content: "html{height:100%;margin:0;padding:0;overflow:hidden}" });
         await scaleContentToFit(page, width, height);
@@ -242,13 +252,10 @@ export const renderHTMLToVideo = async (html, videoPath, width, height, totalFra
  * Loads the animated HTML, calls window.renderFinal() to set all animations
  * to their end state, then takes a screenshot. Used for PDF/thumbnail generation.
  */
-export const renderHTMLToFinalFrame = async (html, outputPath, width, height) => {
-    const browser = await puppeteer.launch({
-        args: isCI ? ["--no-sandbox", "--allow-file-access-from-files"] : ["--allow-file-access-from-files"],
-    });
+export const renderHTMLToFinalFrame = async (html, outputPath, width, height, network = {}) => {
+    const browser = await puppeteer.launch({ args: renderLaunchArgs(network) });
     try {
-        const page = await browser.newPage();
-        await loadHtmlIntoPage(page, html, 30000);
+        const page = await openRenderPage(browser, html, network);
         await page.setViewport({ width, height });
         await page.addStyleTag({ content: "html{height:100%;margin:0;padding:0;overflow:hidden}" });
         await scaleContentToFit(page, width, height);
@@ -266,11 +273,11 @@ export const renderHTMLToFinalFrame = async (html, outputPath, width, height) =>
         await browser.close();
     }
 };
-export const renderMarkdownToImage = async (markdown, style, outputPath, width, height) => {
+export const renderMarkdownToImage = async (markdown, style, outputPath, width, height, network = {}) => {
     const header = `<head><style>${style}</style></head>`;
     const body = await marked(markdown);
     const html = `<html>${header}<body>${body}</body></html>`;
-    await renderHTMLToImage(html, outputPath, width, height);
+    await renderHTMLToImage(html, outputPath, width, height, false, false, network);
 };
 export const interpolate = (template, data) => {
     return template.replace(/\$\{(.*?)\}/g, (_, key) => data[key.trim()] ?? "");

package/lib/utils/image_plugins/chart.js

@@ -3,6 +3,7 @@ import { renderHTMLToImage, interpolate } from "../html_render.js";
 import { parrotingImagePath, generateUniqueId } from "./utils.js";
 import { resolveCombinedStyle } from "./bg_image_util.js";
 import { chartHtml, escapedChartTemplateValues, resolveChartPlugins, stringifyChartData } from "./chart_html.js";
+import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
 export const imageType = "chart";
 const processChart = async (params) => {
     const { beat, imagePath, canvasSize } = params;
@@ -19,7 +20,7 @@ const processChart = async (params) => {
         chart_width: chart_width.toString(),
         chart_plugins: resolveChartPlugins(chartType),
     });
-    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, false, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
     return imagePath;
 };
 const dumpHtml = async (params) => {

package/lib/utils/image_plugins/html_tailwind.js

@@ -7,6 +7,7 @@ import { framesToVideo } from "../ffmpeg_utils.js";
 import { parrotingImagePath } from "./utils.js";
 import { swipeElementsToHtml, swipeElementsToScript } from "../swipe_to_html.js";
 import { htmlTailwindMarkup } from "./html_tailwind_markup.js";
+import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
 export const imageType = "html_tailwind";
 /**
  * Resolve image:name references to file:// absolute paths using imageRefs.
@@ -178,13 +179,13 @@ const processHtmlTailwindAnimated = async (params) => {
         // imagePath is set to the .mp4 path by imagePluginAgent for animated beats
         const videoPath = imagePath;
         if (animConfig.movie) {
-            await renderHTMLToVideo(htmlData, videoPath, canvasSize.width, canvasSize.height, totalFrames, fps);
+            await renderHTMLToVideo(htmlData, videoPath, canvasSize.width, canvasSize.height, totalFrames, fps, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
         }
         else {
             const framesDir = videoPath.replace(/\.[^/.]+$/, "_frames");
             fs.mkdirSync(framesDir, { recursive: true });
             try {
-                await renderHTMLToFrames(htmlData, framesDir, canvasSize.width, canvasSize.height, totalFrames, fps);
+                await renderHTMLToFrames(htmlData, framesDir, canvasSize.width, canvasSize.height, totalFrames, fps, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
                 await framesToVideo(framesDir, videoPath, fps, canvasSize.width, canvasSize.height);
             }
             finally {
@@ -197,7 +198,7 @@ const processHtmlTailwindAnimated = async (params) => {
     // even when exact duration is unknown (e.g., PDF generation without audio).
     const finalFramePath = imagePath.replace(/_animated\.mp4$/, ".png");
     const finalHtml = buildAnimatedHtml(params, FINAL_FRAME_TOTAL, fps);
-    await renderHTMLToFinalFrame(finalHtml, finalFramePath, canvasSize.width, canvasSize.height);
+    await renderHTMLToFinalFrame(finalHtml, finalFramePath, canvasSize.width, canvasSize.height, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
     // Return video path when video was generated, otherwise return the static PNG path
     return duration !== undefined ? imagePath : finalFramePath;
 };
@@ -216,7 +217,7 @@ const processHtmlTailwindStatic = async (params) => {
     const resolvedAllRefs = resolveMovieRefs(resolvedImageRefs, params.movieRefs ?? {});
     const resolvedImages = resolveRelativeImagePaths(resolvedAllRefs, context.fileDirs.mulmoFileDirPath);
     const htmlData = resolveRelativeModelPathsInScript(resolvedImages, context.fileDirs.mulmoFileDirPath);
-    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
+    await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, false, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
     return imagePath;
 };
 const processHtmlTailwind = async (params) => {

package/lib/utils/image_plugins/markdown.js

@@ -5,6 +5,7 @@ import { resolveCombinedStyle } from "./bg_image_util.js";
 import { renderMarkdownContent, renderMarkdownLayout, layoutToMarkdown, toMarkdownString } from "./markdown_layout.js";
 import { resolveImageRefs, resolveMovieRefs } from "./html_tailwind.js";
 import { isObject } from "graphai";
+import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
 /**
  * This path renders once to a PNG, so a random element id is fine. The browser path needs
  * a stable one and supplies its own — that generator is the only thing the two callers of
@@ -77,7 +78,7 @@ const processMarkdown = async (params) => {
     const { html: rawHtml, hasMermaid } = await generateHtml(params);
     const resolvedImages = resolveImageRefs(rawHtml, params.imageRefs ?? {});
     const html = resolveMovieRefs(resolvedImages, params.movieRefs ?? {});
-    await renderHTMLToImage(html, imagePath, canvasSize.width, canvasSize.height, hasMermaid);
+    await renderHTMLToImage(html, imagePath, canvasSize.width, canvasSize.height, hasMermaid, false, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
     return imagePath;
 };
 const dumpHtml = async (params) => {

package/lib/utils/image_plugins/mermaid.js

@@ -5,6 +5,7 @@ import { parrotingImagePath, generateUniqueId } from "./utils.js";
 import { escapedMermaidTemplateValues, mermaidHtml } from "./mermaid_html.js";
 import { resolveCombinedStyle } from "./bg_image_util.js";
 import { resolveImageRefs, resolveMovieRefs, resolveRelativeImagePaths } from "./html_tailwind.js";
+import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
 export const imageType = "mermaid";
 /** Mermaid markup for this path. The markup itself is shared with the browser path. */
 export const generateMermaidHtml = (code, title) => mermaidHtml(code, generateUniqueId("mermaid"), title);
@@ -23,7 +24,7 @@ const processMermaid = async (params) => {
         const resolvedImageRefs = resolveImageRefs(rawHtml, params.imageRefs ?? {});
         const resolvedAllRefs = resolveMovieRefs(resolvedImageRefs, params.movieRefs ?? {});
         const htmlData = resolveRelativeImagePaths(resolvedAllRefs, context.fileDirs.mulmoFileDirPath);
-        await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, true);
+        await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, true, false, MulmoStudioContextMethods.getRenderNetworkOptions(context));
     }
     return imagePath;
 };

package/lib/utils/image_plugins/slide.js

@@ -7,6 +7,7 @@ import { pathToDataUrl } from "../../methods/mulmo_media_source.js";
 import { MulmoMediaSourceMethods } from "../../methods/mulmo_media_source.js";
 import { MulmoPresentationStyleMethods } from "../../methods/mulmo_presentation_style.js";
 import { imageAction, imageFileTarget, unknownMediaType } from "../error_cause.js";
+import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
 export const imageType = "slide";
 const slideImageRefError = (refKey) => ({
     type: unknownMediaType,
@@ -167,7 +168,7 @@ const processSlide = async (params) => {
     const reference = beat.image.reference;
     const resolvedBranding = await resolveAndConvertBranding(params);
     const html = generateSlideHTML(theme, slide, reference, resolvedBranding);
-    await renderHTMLToImage(html, imagePath, canvasSize.width, canvasSize.height);
+    await renderHTMLToImage(html, imagePath, canvasSize.width, canvasSize.height, false, false, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
     return imagePath;
 };
 /**