mulmocast 2.14.0 → 2.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +1 -7
  2. package/assets/schemas/mulmo_script.json +2 -2
  3. package/lib/actions/avatar.d.ts +2 -1
  4. package/lib/actions/avatar.js +59 -3
  5. package/lib/actions/image_references.d.ts +23 -1
  6. package/lib/actions/image_references.js +16 -9
  7. package/lib/agents/image_genai_agent.js +3 -3
  8. package/lib/agents/movie_genai_agent.js +5 -7
  9. package/lib/data/scriptTemplates.js +2 -2
  10. package/lib/data/templateDataSet.js +1 -1
  11. package/lib/index.remotion.d.ts +3 -0
  12. package/lib/index.remotion.js +2 -0
  13. package/lib/index.remotion_guide.d.ts +1 -0
  14. package/lib/index.remotion_guide.js +3 -0
  15. package/lib/methods/mulmo_presentation_style.js +3 -3
  16. package/lib/types/provider2agent.d.ts +25 -16
  17. package/lib/types/provider2agent.js +99 -63
  18. package/lib/types/schema.js +3 -1
  19. package/lib/utils/estimate_usage.js +7 -3
  20. package/lib/utils/image_plugins/remotion.js +3 -10
  21. package/lib/utils/remotion/frames.d.ts +3 -0
  22. package/lib/utils/remotion/frames.js +11 -0
  23. package/lib/utils/remotion/network_guard.d.ts +5 -0
  24. package/lib/utils/remotion/network_guard.js +48 -0
  25. package/lib/utils/remotion/network_policy.d.ts +10 -0
  26. package/lib/utils/remotion/network_policy.js +51 -0
  27. package/lib/utils/remotion/render.d.ts +1 -1
  28. package/lib/utils/remotion/render.js +50 -6
  29. package/lib/utils/remotion/render_frames.d.ts +23 -0
  30. package/lib/utils/remotion/render_frames.js +61 -0
  31. package/package.json +7 -3
  32. package/scripts/templates/html.json +1 -1
  33. package/scripts/templates/presentation.json +1 -1
  34. package/scripts/test/test_avatar.json +97 -23
  35. package/scripts/test/test_genai.json +2 -2
  36. package/scripts/test/test_gpt_image.json +1 -1
  37. package/scripts/test/test_replicate.json +3 -3
  38. package/scripts/test/test_vertexai.json +3 -3
package/README.md CHANGED
@@ -103,11 +103,6 @@ Create a `.env` file in your project directory with the following API keys:
103
103
  OPENAI_API_KEY=your_openai_api_key
104
104
  ```
105
105
 
106
- #### (Optional) For the advanced image generation model
107
- ```bash
108
- DEFAULT_OPENAI_IMAGE_MODEL=gpt-image-1 # for the advanced image generation model
109
- ```
110
-
111
106
  #### (Optional) For Google's image and TTS.
112
107
  ```bash
113
108
  GEMINI_API_KEY=your_google_gemini_api_key
@@ -298,10 +293,9 @@ Optionally, you can specify *__clipboard* as the script file name to paste the s
298
293
  Verify your `.env` file contains:
299
294
  ```bash
300
295
  OPENAI_API_KEY=your_openai_api_key
301
- DEFAULT_OPENAI_IMAGE_MODEL=gpt-image-1 # required for high-quality Ghibli-style images
302
296
  ```
303
297
 
304
- > **Note:** Ensure your OpenAI organization is verified to access the gpt-image-1 model. Visit https://platform.openai.com/settings/organization/general and complete the "Verifications" section.
298
+ > **Note:** Ensure your OpenAI organization is verified to access the GPT Image models. Visit https://platform.openai.com/settings/organization/general and complete the "Verifications" section.
305
299
 
306
300
  ### Step 2: Generate a Ghibli-style MulmoScript
307
301
  ```bash
@@ -125,7 +125,7 @@
125
125
  "properties": {
126
126
  "source": {
127
127
  "type": "string",
128
- "description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project), relative to the script"
128
+ "description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project): a path relative to the script, or an http(s) URL"
129
129
  },
130
130
  "position": {
131
131
  "type": "object",
@@ -226,7 +226,7 @@
226
226
  "properties": {
227
227
  "source": {
228
228
  "type": "string",
229
- "description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project), relative to the script"
229
+ "description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project): a path relative to the script, or an http(s) URL"
230
230
  },
231
231
  "position": {
232
232
  "type": "object",
@@ -19,7 +19,7 @@ export type AvatarPlacementPlan = {
19
19
  };
20
20
  export type AvatarTrackPlan = {
21
21
  speakerId: string;
22
- /** absolute path of the avatar package */
22
+ /** the avatar package: an absolute path, or an http(s) URL */
23
23
  source: string;
24
24
  segments: AvatarSegmentPlan[];
25
25
  /** where the avatar is shown, beat by beat; it is not shown outside these */
@@ -33,5 +33,6 @@ export type AvatarTrackPlan = {
33
33
  * avatarParams, then the defaults; during other speakers' beats the avatar stays where it was.
34
34
  */
35
35
  export declare const planAvatarTracks: (context: MulmoStudioContext) => AvatarTrackPlan[];
36
+ export declare const avatarUrlStamp: (source: string) => Promise<(string | null)[]>;
36
37
  /** The `avatar` action: renders the avatar tracks into studio.avatarTracks (none when no speaker has an avatar). */
37
38
  export declare const avatar: (context: MulmoStudioContext) => Promise<MulmoStudioContext>;
@@ -7,7 +7,7 @@ import path from "node:path";
7
7
  import { GraphAILogger } from "graphai";
8
8
  import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
9
9
  import { getFullPath, getOutputStudioFilePath } from "../utils/file.js";
10
- import { localizedText } from "../utils/utils.js";
10
+ import { isHttp, localizedText } from "../utils/utils.js";
11
11
  const DEFAULT_POSITION = { x: "84%", y: "100%", scale: "62%" };
12
12
  // Show the whole avatar image. A rig may crop the top of its image (a flat cut edge, hidden when the
13
13
  // avatar fills the frame); over a slide the cut would run through the head.
@@ -43,7 +43,7 @@ export const planAvatarTracks = (context) => {
43
43
  tracks.set(found.speakerId, {
44
44
  speakerId: found.speakerId,
45
45
  avatar: found.avatar,
46
- source: getFullPath(context.fileDirs.mulmoFileDirPath, found.avatar.source),
46
+ source: isHttp(found.avatar.source) ? found.avatar.source : getFullPath(context.fileDirs.mulmoFileDirPath, found.avatar.source),
47
47
  segments: [],
48
48
  placements: [],
49
49
  duration,
@@ -105,6 +105,62 @@ const folderStamp = (dir) => {
105
105
  .sort()
106
106
  .map((file) => [path.relative(dir, file), fileStamp(file)]);
107
107
  };
108
+ // An avatar at a URL: its small JSON files (avatar.json, the rig, layers.json, sprites.json) change
109
+ // whenever the avatar is edited or rebuilt, so they stand in for the files' stamps. They are fetched
110
+ // on every run; avatarscript caches the images.
111
+ // Redirects are followed only within the same origin, so a public host cannot send these requests
112
+ // to another one (an internal service, say). GitHub's github.com/…/raw/… links redirect to
113
+ // raw.githubusercontent.com: use the latter.
114
+ const MAX_REDIRECTS = 5;
115
+ const fetchSameOrigin = async (url) => {
116
+ let location = url;
117
+ for (let hops = 0; hops <= MAX_REDIRECTS; hops++) {
118
+ const response = await fetch(location, { redirect: "manual", signal: AbortSignal.timeout(60_000) });
119
+ const next = response.status >= 300 && response.status < 400 ? response.headers.get("location") : null;
120
+ if (!next)
121
+ return response;
122
+ const target = new URL(next, location);
123
+ if (target.origin !== url.origin)
124
+ throw new Error(`avatar: ${url.href} redirects to another host (${target.origin}); use the final URL`);
125
+ location = target;
126
+ }
127
+ throw new Error(`avatar: ${url.href}: too many redirects`);
128
+ };
129
+ const fetchText = async (url) => {
130
+ const response = await fetchSameOrigin(url);
131
+ if (response.status === 404)
132
+ return null;
133
+ if (!response.ok)
134
+ throw new Error(`avatar: ${url.href}: HTTP ${response.status}`);
135
+ return response.text();
136
+ };
137
+ export const avatarUrlStamp = async (source) => {
138
+ const url = new URL(source);
139
+ const folder = new URL(url);
140
+ folder.pathname = url.pathname.endsWith("/avatar.json") ? url.pathname.slice(0, -"avatar.json".length) : url.pathname.replace(/\/?$/, "/");
141
+ // the source's query (a version, a signature) goes with every file that has no query of its own
142
+ // like avatarscript, fetch only files inside the package's folder (no "../", no other host)
143
+ const at = (file, base) => {
144
+ const location = new URL(file, base);
145
+ if (location.origin !== folder.origin || !location.pathname.startsWith(folder.pathname)) {
146
+ throw new Error(`avatar: ${file} is outside the avatar package ${folder.href}`);
147
+ }
148
+ if (!location.search)
149
+ location.search = url.search;
150
+ return location;
151
+ };
152
+ const manifestText = await fetchText(at("avatar.json", folder));
153
+ const manifest = manifestText ? JSON.parse(manifestText) : undefined;
154
+ const root = at((manifest?.root ?? ".").replace(/\/?$/, "/"), folder);
155
+ // as avatarscript: a manifest names its files (sprites are optional); without one, a
156
+ // mesh-avatar-studio project's default paths
157
+ const assets = manifest ? (manifest.assets ?? {}) : { rig: "rig.json", layers: "built/layers.json", sprites: "built/sprites/sprites.json" };
158
+ const files = [assets.rig, assets.layers, assets.sprites];
159
+ // every location is checked before anything is fetched
160
+ const locations = files.map((file) => (file === undefined ? undefined : at(file, root)));
161
+ return [manifestText, ...(await Promise.all(locations.map((location) => (location ? fetchText(location) : null))))];
162
+ };
163
+ const sourceStamp = (source) => (isHttp(source) ? avatarUrlStamp(source) : folderStamp(source));
108
164
  const even = (n) => Math.max(2, Math.round(n / 2) * 2); // video codecs need even sizes
109
165
  /** Renders one track (or reuses it), returning where it goes on the canvas. */
110
166
  const renderAvatarTrack = async (plan, context, getAvatarScript) => {
@@ -112,7 +168,7 @@ const renderAvatarTrack = async (plan, context, getAvatarScript) => {
112
168
  // rendered once, at the largest size it is shown; smaller placements scale it down
113
169
  const height = even(Math.max(...plan.placements.map((p) => percent(p.position.scale, canvas.height))));
114
170
  const segments = plan.segments.map((segment) => ({ ...segment, audioStamp: fileStamp(segment.audio) }));
115
- const identity = { version: 4, segments, source: plan.source, sourceStamp: folderStamp(plan.source), duration: plan.duration, height, padTop: PAD_TOP };
171
+ const identity = { version: 4, segments, source: plan.source, sourceStamp: await sourceStamp(plan.source), duration: plan.duration, height, padTop: PAD_TOP };
116
172
  const hash = createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, 12);
117
173
  const dir = MulmoStudioContextMethods.getImageProjectDirPath(context);
118
174
  const name = `avatar_${plan.speakerId.replace(/[^\w-]/g, "_")}_${hash}`;
@@ -1,6 +1,28 @@
1
- import { MulmoStudioContext, MulmoBeat, MulmoImagePromptMedia } from "../types/index.js";
1
+ import { MulmoPresentationStyleMethods } from "../methods/index.js";
2
+ import { MulmoStudioContext, MulmoBeat, MulmoImagePromptMedia, MulmoCanvasDimension } from "../types/index.js";
2
3
  import type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
3
4
  export type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
5
+ type ImageAgentInfo = ReturnType<typeof MulmoPresentationStyleMethods.getImageAgentInfo>;
6
+ type MovieAgentInfo = ReturnType<typeof MulmoPresentationStyleMethods.getMovieAgentInfo>;
7
+ export declare const referenceImageGraphParams: (imageAgentInfo: ImageAgentInfo, canvasSize: MulmoCanvasDimension) => {
8
+ model: string | undefined;
9
+ canvasSize: {
10
+ width: number;
11
+ height: number;
12
+ };
13
+ vertexai_project: string | undefined;
14
+ vertexai_location: string | undefined;
15
+ };
16
+ export declare const referenceMovieGraphParams: (movieAgentInfo: MovieAgentInfo, canvasSize: MulmoCanvasDimension) => {
17
+ model: string | undefined;
18
+ canvasSize: {
19
+ width: number;
20
+ height: number;
21
+ };
22
+ generateAudio: boolean | undefined;
23
+ vertexai_project: string | undefined;
24
+ vertexai_location: string | undefined;
25
+ };
4
26
  export declare const generateReferenceImage: (inputs: {
5
27
  context: MulmoStudioContext;
6
28
  key: string;
@@ -6,6 +6,20 @@ import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods, MulmoImagePromp
6
6
  import { ImageMediaType, } from "../types/index.js";
7
7
  import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent } from "../agents/index.js";
8
8
  import { agentGenerationError, imageReferenceAction, imageFileTarget, movieFileTarget } from "../utils/error_cause.js";
9
+ // Reference media are generated outside the beat graphs, so they must carry the same Vertex AI target the beats do.
10
+ export const referenceImageGraphParams = (imageAgentInfo, canvasSize) => ({
11
+ model: imageAgentInfo.imageParams.model,
12
+ canvasSize,
13
+ vertexai_project: imageAgentInfo.imageParams.vertexai_project,
14
+ vertexai_location: imageAgentInfo.imageParams.vertexai_location,
15
+ });
16
+ export const referenceMovieGraphParams = (movieAgentInfo, canvasSize) => ({
17
+ model: movieAgentInfo.movieParams.model,
18
+ canvasSize,
19
+ generateAudio: movieAgentInfo.movieParams.generateAudio,
20
+ vertexai_project: movieAgentInfo.movieParams.vertexai_project,
21
+ vertexai_location: movieAgentInfo.movieParams.vertexai_location,
22
+ });
9
23
  // Resolve the normalized reference list to local file paths (with labels), in reference order.
10
24
  // imageKey is the imageRefs key of the image being generated (used for warnings and url
11
25
  // download file naming). A named reference that is missing from imageRefs warns and is
@@ -62,10 +76,7 @@ export const generateReferenceImage = async (inputs) => {
62
76
  sessionType: "imageReference",
63
77
  },
64
78
  },
65
- params: {
66
- model: imageAgentInfo.imageParams.model,
67
- canvasSize: image.canvasSize ?? context.presentationStyle.canvasSize,
68
- },
79
+ params: referenceImageGraphParams(imageAgentInfo, image.canvasSize ?? context.presentationStyle.canvasSize),
69
80
  },
70
81
  },
71
82
  };
@@ -145,11 +156,7 @@ const generateReferenceMovie = async (inputs) => {
145
156
  sessionType: "imageReference",
146
157
  },
147
158
  },
148
- params: {
149
- model: movieAgentInfo.movieParams.model,
150
- canvasSize: context.presentationStyle.canvasSize,
151
- generateAudio: movieAgentInfo.movieParams.generateAudio,
152
- },
159
+ params: referenceMovieGraphParams(movieAgentInfo, context.presentationStyle.canvasSize),
153
160
  },
154
161
  },
155
162
  };
@@ -1,6 +1,6 @@
1
1
  import fs from "fs";
2
2
  import { GraphAILogger } from "graphai";
3
- import { provider2ImageAgent, deprecatedGoogleImageModelHints, vertexAIGlobalOnlyImageModels, } from "../types/provider2agent.js";
3
+ import { provider2ImageAgent, deprecatedGoogleImageModelHints, vertexAIGlobalOnlyImageModels, isGeminiImageModel, vertexImageLocation, } from "../types/provider2agent.js";
4
4
  import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, agentInvalidResponseError, imageAction, imageFileTarget, unsupportedModelTarget, hasCause, getGenAIErrorReason, resultify, } from "../utils/error_cause.js";
5
5
  import { getAspectRatio } from "../utils/utils.js";
6
6
  import { ASPECT_RATIOS, PRO_ASPECT_RATIOS } from "../types/const.js";
@@ -85,7 +85,7 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
85
85
  const apiKey = config?.apiKey;
86
86
  const ai = params.vertexai_project
87
87
  ? (() => {
88
- const location = params.vertexai_location ?? "us-central1";
88
+ const location = vertexImageLocation(model, params.vertexai_location);
89
89
  if (vertexAIGlobalOnlyImageModels.has(model) && location !== "global") {
90
90
  GraphAILogger.warn(`imageGenAIAgent: model "${model}" on Vertex AI is only available in location "global", but got "${location}". Set imageParams.vertexai_location to "global".`);
91
91
  }
@@ -99,7 +99,7 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
99
99
  }
100
100
  return new GoogleGenAI({ apiKey, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
101
101
  })();
102
- if (model === "gemini-2.5-flash-image" || model === "gemini-3.1-flash-image-preview" || model === "gemini-3-pro-image-preview") {
102
+ if (isGeminiImageModel(model)) {
103
103
  const contentParams = (() => {
104
104
  const contents = getGeminiContents(prompt, referenceImages);
105
105
  return {
@@ -5,7 +5,7 @@ import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, im
5
5
  import { getAspectRatio } from "../utils/utils.js";
6
6
  import { ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
7
7
  import { ASPECT_RATIOS } from "../types/const.js";
8
- import { getModelDuration, provider2MovieAgent, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS } from "../types/provider2agent.js";
8
+ import { getModelDuration, provider2MovieAgent, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS, defaultMovieModel, veoExtensionCount, VEO_EXTENSION_MODEL, VEO_SEGMENT_SEC, } from "../types/provider2agent.js";
9
9
  // Per-request timeout so a stalled GenAI video API call rejects instead of hanging.
10
10
  const GENAI_REQUEST_TIMEOUT_MS = 120_000;
11
11
  // Wall-clock cap on the long-running video operation poll loop (Veo runs minutes).
@@ -89,14 +89,12 @@ const createVeo31Payload = (model, prompt, aspectRatio, source) => ({
89
89
  ...source,
90
90
  });
91
91
  const generateExtendedVideo = async (ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile, isVertexAI) => {
92
- const initialDuration = 8;
93
- const maxExtensionDuration = 8;
94
- const extensionsNeeded = Math.ceil((requestedDuration - initialDuration) / maxExtensionDuration);
92
+ const extensionsNeeded = veoExtensionCount(requestedDuration);
95
93
  GraphAILogger.info(`Veo 3.1 video extension: ${extensionsNeeded} extensions needed for ${requestedDuration}s target`);
96
94
  const generateIteration = async (iteration, accumulatedDuration, previousVideo) => {
97
95
  const isInitial = iteration === 0;
98
96
  const remainingDuration = requestedDuration - accumulatedDuration;
99
- const extensionDuration = isInitial ? initialDuration : (getModelDuration("google", model, remainingDuration) ?? maxExtensionDuration);
97
+ const extensionDuration = isInitial ? VEO_SEGMENT_SEC : (getModelDuration("google", model, remainingDuration) ?? VEO_SEGMENT_SEC);
100
98
  const getSource = () => {
101
99
  if (isInitial)
102
100
  return imagePath ? { image: loadImageAsBase64(imagePath) } : undefined;
@@ -172,7 +170,7 @@ const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath,
172
170
  export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
173
171
  const { prompt, imagePath, lastFrameImagePath, referenceImages, movieFile } = namedInputs;
174
172
  const aspectRatio = getAspectRatio(params.canvasSize, ASPECT_RATIOS);
175
- const model = params.model ?? provider2MovieAgent.google.defaultModel;
173
+ const model = params.model ?? defaultMovieModel("google", params);
176
174
  const apiKey = config?.apiKey;
177
175
  try {
178
176
  const requestedDuration = params.duration ?? 8;
@@ -211,7 +209,7 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
211
209
  return new GoogleGenAI({ apiKey, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
212
210
  })();
213
211
  // Veo 3.1: Video extension mode for videos longer than 8s
214
- if (model === "veo-3.1-generate-preview" && requestedDuration > 8 && params.canvasSize) {
212
+ if (model === VEO_EXTENSION_MODEL && requestedDuration > VEO_SEGMENT_SEC && params.canvasSize) {
215
213
  return generateExtendedVideo(ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile, isVertexAI);
216
214
  }
217
215
  // Standard mode
@@ -484,7 +484,7 @@ export const scriptTemplates = [
484
484
  ],
485
485
  filename: "html",
486
486
  htmlImageParams: {
487
- model: "claude-sonnet-4-5-20250929",
487
+ model: "claude-sonnet-4-6",
488
488
  provider: "anthropic",
489
489
  },
490
490
  lang: "en",
@@ -987,7 +987,7 @@ export const scriptTemplates = [
987
987
  ],
988
988
  filename: "presentation",
989
989
  htmlImageParams: {
990
- model: "claude-sonnet-4-5-20250929",
990
+ model: "claude-sonnet-4-6",
991
991
  provider: "anthropic",
992
992
  },
993
993
  lang: "en",
@@ -49,7 +49,7 @@ export const templateDataSet = {
49
49
  "```",
50
50
  html: "Another LLM will generate actual slides from the prompt and data for each beat. Adding optional data would help it to generate more compelling slide. Mention the reference in one of beats, if it exists. The valid type of reference is 'article', 'paper', 'image', 'video', 'audio'. Use the JSON below as a template.\n" +
51
51
  "```JSON\n" +
52
- '{"$mulmocast":{"version":"1.1","credit":"closing"},"references":[{"url":"https://www.somegreatwebsite.com/article/123","title":"Title of the article we are referencing","type":"[TYPE OF ARTICLE: article, paper, image, video, audio]"}],"title":"[TITLE: Brief, engaging title for the topic]","htmlImageParams":{"provider":"anthropic","model":"claude-sonnet-4-5-20250929"},"lang":"en","beats":[{"text":"[NARRATION: Narration for the beat.]","htmlPrompt":{"prompt":"[PROMPT to create appropriate HTML page for the beat.]"}},{"text":"[NARRATION: Narration for the beat.]","htmlPrompt":{"prompt":"[PROMPT to create appropriate HTML page for the beat with the data.]","data":{"description":"DATA TO BE PRESENTED IN THIS BEAT (in any format)]","net_income":{"Q2 FY2024":320,"Q3 FY2024":333,"Q4 FY2024":350},"unit":"USD (Million)"}}}],"canvasSize":{"width":1536,"height":1024}}\n' +
52
+ '{"$mulmocast":{"version":"1.1","credit":"closing"},"references":[{"url":"https://www.somegreatwebsite.com/article/123","title":"Title of the article we are referencing","type":"[TYPE OF ARTICLE: article, paper, image, video, audio]"}],"title":"[TITLE: Brief, engaging title for the topic]","htmlImageParams":{"provider":"anthropic","model":"claude-sonnet-4-6"},"lang":"en","beats":[{"text":"[NARRATION: Narration for the beat.]","htmlPrompt":{"prompt":"[PROMPT to create appropriate HTML page for the beat.]"}},{"text":"[NARRATION: Narration for the beat.]","htmlPrompt":{"prompt":"[PROMPT to create appropriate HTML page for the beat with the data.]","data":{"description":"DATA TO BE PRESENTED IN THIS BEAT (in any format)]","net_income":{"Q2 FY2024":320,"Q3 FY2024":333,"Q4 FY2024":350},"unit":"USD (Million)"}}}],"canvasSize":{"width":1536,"height":1024}}\n' +
53
53
  "```",
54
54
  image_prompt: "Another AI will generate images for each beat based on the image prompt of that beat. Movie prompts must be written in English. Mention the reference in one of beats, if it exists. Use the JSON below as a template.\n" +
55
55
  "```JSON\n" +
@@ -1,3 +1,6 @@
1
1
  export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
2
2
  export { REMOTION_PACKAGES, REMOTION_RENDER_PACKAGES, REMOTION_SCENE_PACKAGES, remotionInstallCommand } from "./utils/remotion/packages.js";
3
3
  export { ensureRemotionPackages, missingInstalledRemotionPackages } from "./utils/remotion/render.js";
4
+ export { REMOTION_REVIEW_FRACTIONS } from "./utils/remotion/frames.js";
5
+ export { renderRemotionFrames } from "./utils/remotion/render_frames.js";
6
+ export type { RemotionFrame, RemotionFramesRequest } from "./utils/remotion/render_frames.js";
@@ -2,3 +2,5 @@
2
2
  export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
3
3
  export { REMOTION_PACKAGES, REMOTION_RENDER_PACKAGES, REMOTION_SCENE_PACKAGES, remotionInstallCommand } from "./utils/remotion/packages.js";
4
4
  export { ensureRemotionPackages, missingInstalledRemotionPackages } from "./utils/remotion/render.js";
5
+ export { REMOTION_REVIEW_FRACTIONS } from "./utils/remotion/frames.js";
6
+ export { renderRemotionFrames } from "./utils/remotion/render_frames.js";
@@ -0,0 +1 @@
1
+ export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
@@ -0,0 +1,3 @@
1
+ // Entry point for code that only needs the remotion component guide — a tool description built into
2
+ // a browser bundle, for example. `mulmocast/remotion` also carries the renderer, which needs Node.
3
+ export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
@@ -8,7 +8,7 @@ import { userAssert } from "../utils/utils.js";
8
8
  import { slideThemes } from "../data/slideThemes.js";
9
9
  import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, mulmoFillOptionSchema, } from "../types/schema.js";
10
10
  import { MulmoBeatMethods } from "./mulmo_beat.js";
11
- import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
11
+ import { provider2ImageAgent, provider2MovieAgent, defaultMovieModel, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
12
12
  const defaultTextSlideStyles = [
13
13
  '*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
14
14
  "body { margin: 60px; margin-top: 40px; color:#333; background-color:#fff; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
@@ -160,7 +160,7 @@ export const MulmoPresentationStyleMethods = {
160
160
  const imageParams = { ...presentationStyle.imageParams, ...beat?.imageParams };
161
161
  const provider = MulmoPresentationStyleMethods.getText2ImageProvider(imageParams?.provider) ?? defaultProviders.text2image;
162
162
  const agentInfo = provider2ImageAgent[provider];
163
- // The default text2image model is gpt-image-1 from OpenAI.
163
+ // The default text2image model is gpt-image-2.5-sunburst from OpenAI.
164
164
  const defaultImageParams = {
165
165
  provider,
166
166
  model: agentInfo.defaultModel,
@@ -189,7 +189,7 @@ export const MulmoPresentationStyleMethods = {
189
189
  return true;
190
190
  }
191
191
  const provider = text2MovieProviderSchema.parse(movieParams?.provider ?? defaultProviders.text2movie);
192
- const model = movieParams?.model ?? provider2MovieAgent[provider].defaultModel;
192
+ const model = movieParams?.model ?? defaultMovieModel(provider, movieParams);
193
193
  return getModelAudio(provider, model)?.mode === AUDIO_MODE_ALWAYS;
194
194
  },
195
195
  getSoundEffectAgentInfo(presentationStyle, beat) {
@@ -43,20 +43,22 @@ export declare const provider2TTSAgent: {
43
43
  };
44
44
  export declare const gptImages: string[];
45
45
  export declare const deprecatedOpenAIImageModelHints: {
46
- readonly "dall-e-2": "Use 'gpt-image-1' or another supported model.";
47
- readonly "dall-e-3": "Use 'gpt-image-1' or another supported model.";
46
+ readonly "dall-e-2": "Use 'gpt-image-2.5-sunburst' or another supported model.";
47
+ readonly "dall-e-3": "Use 'gpt-image-2.5-sunburst' or another supported model.";
48
48
  };
49
49
  export type DeprecatedOpenAIImageModel = keyof typeof deprecatedOpenAIImageModelHints;
50
50
  export declare const deprecatedGoogleImageModelHints: {
51
- readonly "imagen-3.0-generate-002": "Use 'gemini-2.5-flash-image' or 'gemini-3-pro-image-preview' instead.";
52
- readonly "imagen-4.0-generate-001": "Use 'gemini-2.5-flash-image' or 'gemini-3-pro-image-preview' instead.";
53
- readonly "imagen-4.0-ultra-generate-001": "Use 'gemini-2.5-flash-image' or 'gemini-3-pro-image-preview' instead.";
54
- readonly "imagen-4.0-fast-generate-001": "Use 'gemini-2.5-flash-image' or 'gemini-3-pro-image-preview' instead.";
55
- readonly "imagen-4.0-generate-preview-06-06": "Use 'gemini-2.5-flash-image' or 'gemini-3-pro-image-preview' instead.";
56
- readonly "imagen-4.0-ultra-generate-preview-06-06": "Use 'gemini-2.5-flash-image' or 'gemini-3-pro-image-preview' instead.";
51
+ readonly "imagen-3.0-generate-002": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
52
+ readonly "imagen-4.0-generate-001": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
53
+ readonly "imagen-4.0-ultra-generate-001": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
54
+ readonly "imagen-4.0-fast-generate-001": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
55
+ readonly "imagen-4.0-generate-preview-06-06": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
56
+ readonly "imagen-4.0-ultra-generate-preview-06-06": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
57
57
  };
58
58
  export type DeprecatedGoogleImageModel = keyof typeof deprecatedGoogleImageModelHints;
59
+ export declare const isGeminiImageModel: (model: string) => boolean;
59
60
  export declare const vertexAIGlobalOnlyImageModels: ReadonlySet<string>;
61
+ export declare const vertexImageLocation: (model: string, location?: string) => string;
60
62
  export declare const provider2ImageAgent: {
61
63
  openai: {
62
64
  agentName: string;
@@ -169,34 +171,34 @@ export declare const provider2LipSyncAgent: {
169
171
  export declare const provider2LLMAgent: {
170
172
  readonly openai: {
171
173
  readonly agentName: "openAIAgent";
172
- readonly defaultModel: "gpt-5";
174
+ readonly defaultModel: "gpt-5.6-sol";
173
175
  readonly keyName: "OPENAI_API_KEY";
174
176
  readonly baseURLKeyName: "OPENAI_BASE_URL";
175
177
  readonly apiVersionKeyName: "OPENAI_API_VERSION";
176
178
  readonly max_tokens: 8192;
177
- readonly models: readonly ["gpt-5", "gpt-5-nano", "gpt-5-mini", "gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano", "o3", "o3-mini", "o3-pro", "o1", "o1-pro", "gpt-4o", "gpt-4o-mini"];
179
+ readonly models: readonly ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5", "gpt-5-nano", "gpt-5-mini", "gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano", "o3", "o3-mini", "o3-pro", "o1", "o1-pro", "gpt-4o", "gpt-4o-mini"];
178
180
  };
179
181
  readonly anthropic: {
180
182
  readonly agentName: "anthropicAgent";
181
- readonly defaultModel: "claude-sonnet-4-5-20250929";
183
+ readonly defaultModel: "claude-sonnet-4-6";
182
184
  readonly max_tokens: 8192;
183
- readonly models: readonly ["claude-opus-4-1-20250805", "claude-opus-4-20250514", "claude-sonnet-4-20250514", "claude-sonnet-4-5-20250929", "claude-haiku-4-5-20251001"];
185
+ readonly models: readonly ["claude-sonnet-4-6", "claude-sonnet-4-5-20250929", "claude-haiku-4-5-20251001"];
184
186
  readonly keyName: "ANTHROPIC_API_KEY";
185
187
  readonly apiKeyNameOverride: "ANTHROPIC_API_TOKEN";
186
188
  };
187
189
  readonly gemini: {
188
190
  readonly agentName: "geminiAgent";
189
- readonly defaultModel: "gemini-2.5-flash";
191
+ readonly defaultModel: "gemini-3.8-flash";
190
192
  readonly max_tokens: 8192;
191
- readonly models: readonly ["gemini-3.1-pro-preview", "gemini-3-flash-preview", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite"];
193
+ readonly models: readonly ["gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3-flash-preview", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite"];
192
194
  readonly keyName: "GEMINI_API_KEY";
193
195
  };
194
196
  readonly groq: {
195
197
  readonly agentName: "groqAgent";
196
- readonly defaultModel: "llama-3.1-8b-instant";
198
+ readonly defaultModel: "openai/gpt-oss-20b";
197
199
  readonly keyName: "GROQ_API_KEY";
198
200
  readonly max_tokens: 4096;
199
- readonly models: readonly ["llama-3.1-8b-instant", "llama-3.3-70b-versatile", "deepseek-r1-distill-llama-70b", "openai/gpt-oss-120b", "openai/gpt-oss-20b"];
201
+ readonly models: readonly ["openai/gpt-oss-20b", "openai/gpt-oss-120b"];
200
202
  };
201
203
  readonly mock: {
202
204
  readonly agentName: "mediaMockAgent";
@@ -229,6 +231,13 @@ export declare const ProviderName: {
229
231
  export declare const isImageProvider: (provider: string | undefined) => provider is keyof typeof provider2ImageAgent;
230
232
  export declare const getMaxImageReferenceImages: (provider: string | undefined, model: string) => number | undefined;
231
233
  export declare const getModelAudio: (provider: keyof typeof provider2MovieAgent, model: string) => MovieAudioSpec | undefined;
234
+ export declare const defaultMovieModel: (provider: keyof typeof provider2MovieAgent, movieParams?: {
235
+ vertexai_project?: string;
236
+ }) => string;
237
+ export declare const VEO_EXTENSION_MODEL = "veo-3.1-generate-preview";
238
+ export declare const VEO_SEGMENT_SEC = 8;
239
+ export declare const veoExtensionCount: (requestedSec: number) => number;
240
+ export declare const veoExtendedSeconds: (requestedSec: number) => number;
232
241
  export declare const getModelDuration: (provider: keyof typeof provider2MovieAgent, model: string, movieDuration?: number) => number | undefined;
233
242
  export type ModelPricing = {
234
243
  unit: "tokens" | "chars" | "seconds" | "images";