mulmocast 2.14.0 → 2.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -7
- package/assets/schemas/mulmo_script.json +2 -2
- package/lib/actions/avatar.d.ts +2 -1
- package/lib/actions/avatar.js +59 -3
- package/lib/actions/image_references.d.ts +23 -1
- package/lib/actions/image_references.js +16 -9
- package/lib/agents/image_genai_agent.js +3 -3
- package/lib/agents/movie_genai_agent.js +5 -7
- package/lib/data/scriptTemplates.js +2 -2
- package/lib/data/templateDataSet.js +1 -1
- package/lib/index.remotion.d.ts +3 -0
- package/lib/index.remotion.js +2 -0
- package/lib/index.remotion_guide.d.ts +1 -0
- package/lib/index.remotion_guide.js +3 -0
- package/lib/methods/mulmo_presentation_style.js +3 -3
- package/lib/types/provider2agent.d.ts +25 -16
- package/lib/types/provider2agent.js +99 -63
- package/lib/types/schema.js +3 -1
- package/lib/utils/estimate_usage.js +7 -3
- package/lib/utils/image_plugins/remotion.js +3 -10
- package/lib/utils/remotion/frames.d.ts +3 -0
- package/lib/utils/remotion/frames.js +11 -0
- package/lib/utils/remotion/network_guard.d.ts +5 -0
- package/lib/utils/remotion/network_guard.js +48 -0
- package/lib/utils/remotion/network_policy.d.ts +10 -0
- package/lib/utils/remotion/network_policy.js +51 -0
- package/lib/utils/remotion/render.d.ts +1 -1
- package/lib/utils/remotion/render.js +50 -6
- package/lib/utils/remotion/render_frames.d.ts +23 -0
- package/lib/utils/remotion/render_frames.js +61 -0
- package/package.json +7 -3
- package/scripts/templates/html.json +1 -1
- package/scripts/templates/presentation.json +1 -1
- package/scripts/test/test_avatar.json +97 -23
- package/scripts/test/test_genai.json +2 -2
- package/scripts/test/test_gpt_image.json +1 -1
- package/scripts/test/test_replicate.json +3 -3
- package/scripts/test/test_vertexai.json +3 -3
package/README.md
CHANGED
|
@@ -103,11 +103,6 @@ Create a `.env` file in your project directory with the following API keys:
|
|
|
103
103
|
OPENAI_API_KEY=your_openai_api_key
|
|
104
104
|
```
|
|
105
105
|
|
|
106
|
-
#### (Optional) For the advanced image generation model
|
|
107
|
-
```bash
|
|
108
|
-
DEFAULT_OPENAI_IMAGE_MODEL=gpt-image-1 # for the advanced image generation model
|
|
109
|
-
```
|
|
110
|
-
|
|
111
106
|
#### (Optional) For Google's image and TTS.
|
|
112
107
|
```bash
|
|
113
108
|
GEMINI_API_KEY=your_google_gemini_api_key
|
|
@@ -298,10 +293,9 @@ Optionally, you can specify *__clipboard* as the script file name to paste the s
|
|
|
298
293
|
Verify your `.env` file contains:
|
|
299
294
|
```bash
|
|
300
295
|
OPENAI_API_KEY=your_openai_api_key
|
|
301
|
-
DEFAULT_OPENAI_IMAGE_MODEL=gpt-image-1 # required for high-quality Ghibli-style images
|
|
302
296
|
```
|
|
303
297
|
|
|
304
|
-
> **Note:** Ensure your OpenAI organization is verified to access the
|
|
298
|
+
> **Note:** Ensure your OpenAI organization is verified to access the GPT Image models. Visit https://platform.openai.com/settings/organization/general and complete the "Verifications" section.
|
|
305
299
|
|
|
306
300
|
### Step 2: Generate a Ghibli-style MulmoScript
|
|
307
301
|
```bash
|
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
"properties": {
|
|
126
126
|
"source": {
|
|
127
127
|
"type": "string",
|
|
128
|
-
"description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project)
|
|
128
|
+
"description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project): a path relative to the script, or an http(s) URL"
|
|
129
129
|
},
|
|
130
130
|
"position": {
|
|
131
131
|
"type": "object",
|
|
@@ -226,7 +226,7 @@
|
|
|
226
226
|
"properties": {
|
|
227
227
|
"source": {
|
|
228
228
|
"type": "string",
|
|
229
|
-
"description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project)
|
|
229
|
+
"description": "AvatarScript avatar package (a folder with avatar.json, or a mesh-avatar-studio project): a path relative to the script, or an http(s) URL"
|
|
230
230
|
},
|
|
231
231
|
"position": {
|
|
232
232
|
"type": "object",
|
package/lib/actions/avatar.d.ts
CHANGED
|
@@ -19,7 +19,7 @@ export type AvatarPlacementPlan = {
|
|
|
19
19
|
};
|
|
20
20
|
export type AvatarTrackPlan = {
|
|
21
21
|
speakerId: string;
|
|
22
|
-
/** absolute path
|
|
22
|
+
/** the avatar package: an absolute path, or an http(s) URL */
|
|
23
23
|
source: string;
|
|
24
24
|
segments: AvatarSegmentPlan[];
|
|
25
25
|
/** where the avatar is shown, beat by beat; it is not shown outside these */
|
|
@@ -33,5 +33,6 @@ export type AvatarTrackPlan = {
|
|
|
33
33
|
* avatarParams, then the defaults; during other speakers' beats the avatar stays where it was.
|
|
34
34
|
*/
|
|
35
35
|
export declare const planAvatarTracks: (context: MulmoStudioContext) => AvatarTrackPlan[];
|
|
36
|
+
export declare const avatarUrlStamp: (source: string) => Promise<(string | null)[]>;
|
|
36
37
|
/** The `avatar` action: renders the avatar tracks into studio.avatarTracks (none when no speaker has an avatar). */
|
|
37
38
|
export declare const avatar: (context: MulmoStudioContext) => Promise<MulmoStudioContext>;
|
package/lib/actions/avatar.js
CHANGED
|
@@ -7,7 +7,7 @@ import path from "node:path";
|
|
|
7
7
|
import { GraphAILogger } from "graphai";
|
|
8
8
|
import { MulmoPresentationStyleMethods, MulmoStudioContextMethods } from "../methods/index.js";
|
|
9
9
|
import { getFullPath, getOutputStudioFilePath } from "../utils/file.js";
|
|
10
|
-
import { localizedText } from "../utils/utils.js";
|
|
10
|
+
import { isHttp, localizedText } from "../utils/utils.js";
|
|
11
11
|
const DEFAULT_POSITION = { x: "84%", y: "100%", scale: "62%" };
|
|
12
12
|
// Show the whole avatar image. A rig may crop the top of its image (a flat cut edge, hidden when the
|
|
13
13
|
// avatar fills the frame); over a slide the cut would run through the head.
|
|
@@ -43,7 +43,7 @@ export const planAvatarTracks = (context) => {
|
|
|
43
43
|
tracks.set(found.speakerId, {
|
|
44
44
|
speakerId: found.speakerId,
|
|
45
45
|
avatar: found.avatar,
|
|
46
|
-
source: getFullPath(context.fileDirs.mulmoFileDirPath, found.avatar.source),
|
|
46
|
+
source: isHttp(found.avatar.source) ? found.avatar.source : getFullPath(context.fileDirs.mulmoFileDirPath, found.avatar.source),
|
|
47
47
|
segments: [],
|
|
48
48
|
placements: [],
|
|
49
49
|
duration,
|
|
@@ -105,6 +105,62 @@ const folderStamp = (dir) => {
|
|
|
105
105
|
.sort()
|
|
106
106
|
.map((file) => [path.relative(dir, file), fileStamp(file)]);
|
|
107
107
|
};
|
|
108
|
+
// An avatar at a URL: its small JSON files (avatar.json, the rig, layers.json, sprites.json) change
|
|
109
|
+
// whenever the avatar is edited or rebuilt, so they stand in for the files' stamps. They are fetched
|
|
110
|
+
// on every run; avatarscript caches the images.
|
|
111
|
+
// Redirects are followed only within the same origin, so a public host cannot send these requests
|
|
112
|
+
// to another one (an internal service, say). GitHub's github.com/…/raw/… links redirect to
|
|
113
|
+
// raw.githubusercontent.com: use the latter.
|
|
114
|
+
const MAX_REDIRECTS = 5;
|
|
115
|
+
const fetchSameOrigin = async (url) => {
|
|
116
|
+
let location = url;
|
|
117
|
+
for (let hops = 0; hops <= MAX_REDIRECTS; hops++) {
|
|
118
|
+
const response = await fetch(location, { redirect: "manual", signal: AbortSignal.timeout(60_000) });
|
|
119
|
+
const next = response.status >= 300 && response.status < 400 ? response.headers.get("location") : null;
|
|
120
|
+
if (!next)
|
|
121
|
+
return response;
|
|
122
|
+
const target = new URL(next, location);
|
|
123
|
+
if (target.origin !== url.origin)
|
|
124
|
+
throw new Error(`avatar: ${url.href} redirects to another host (${target.origin}); use the final URL`);
|
|
125
|
+
location = target;
|
|
126
|
+
}
|
|
127
|
+
throw new Error(`avatar: ${url.href}: too many redirects`);
|
|
128
|
+
};
|
|
129
|
+
const fetchText = async (url) => {
|
|
130
|
+
const response = await fetchSameOrigin(url);
|
|
131
|
+
if (response.status === 404)
|
|
132
|
+
return null;
|
|
133
|
+
if (!response.ok)
|
|
134
|
+
throw new Error(`avatar: ${url.href}: HTTP ${response.status}`);
|
|
135
|
+
return response.text();
|
|
136
|
+
};
|
|
137
|
+
export const avatarUrlStamp = async (source) => {
|
|
138
|
+
const url = new URL(source);
|
|
139
|
+
const folder = new URL(url);
|
|
140
|
+
folder.pathname = url.pathname.endsWith("/avatar.json") ? url.pathname.slice(0, -"avatar.json".length) : url.pathname.replace(/\/?$/, "/");
|
|
141
|
+
// the source's query (a version, a signature) goes with every file that has no query of its own
|
|
142
|
+
// like avatarscript, fetch only files inside the package's folder (no "../", no other host)
|
|
143
|
+
const at = (file, base) => {
|
|
144
|
+
const location = new URL(file, base);
|
|
145
|
+
if (location.origin !== folder.origin || !location.pathname.startsWith(folder.pathname)) {
|
|
146
|
+
throw new Error(`avatar: ${file} is outside the avatar package ${folder.href}`);
|
|
147
|
+
}
|
|
148
|
+
if (!location.search)
|
|
149
|
+
location.search = url.search;
|
|
150
|
+
return location;
|
|
151
|
+
};
|
|
152
|
+
const manifestText = await fetchText(at("avatar.json", folder));
|
|
153
|
+
const manifest = manifestText ? JSON.parse(manifestText) : undefined;
|
|
154
|
+
const root = at((manifest?.root ?? ".").replace(/\/?$/, "/"), folder);
|
|
155
|
+
// as avatarscript: a manifest names its files (sprites are optional); without one, a
|
|
156
|
+
// mesh-avatar-studio project's default paths
|
|
157
|
+
const assets = manifest ? (manifest.assets ?? {}) : { rig: "rig.json", layers: "built/layers.json", sprites: "built/sprites/sprites.json" };
|
|
158
|
+
const files = [assets.rig, assets.layers, assets.sprites];
|
|
159
|
+
// every location is checked before anything is fetched
|
|
160
|
+
const locations = files.map((file) => (file === undefined ? undefined : at(file, root)));
|
|
161
|
+
return [manifestText, ...(await Promise.all(locations.map((location) => (location ? fetchText(location) : null))))];
|
|
162
|
+
};
|
|
163
|
+
const sourceStamp = (source) => (isHttp(source) ? avatarUrlStamp(source) : folderStamp(source));
|
|
108
164
|
const even = (n) => Math.max(2, Math.round(n / 2) * 2); // video codecs need even sizes
|
|
109
165
|
/** Renders one track (or reuses it), returning where it goes on the canvas. */
|
|
110
166
|
const renderAvatarTrack = async (plan, context, getAvatarScript) => {
|
|
@@ -112,7 +168,7 @@ const renderAvatarTrack = async (plan, context, getAvatarScript) => {
|
|
|
112
168
|
// rendered once, at the largest size it is shown; smaller placements scale it down
|
|
113
169
|
const height = even(Math.max(...plan.placements.map((p) => percent(p.position.scale, canvas.height))));
|
|
114
170
|
const segments = plan.segments.map((segment) => ({ ...segment, audioStamp: fileStamp(segment.audio) }));
|
|
115
|
-
const identity = { version: 4, segments, source: plan.source, sourceStamp:
|
|
171
|
+
const identity = { version: 4, segments, source: plan.source, sourceStamp: await sourceStamp(plan.source), duration: plan.duration, height, padTop: PAD_TOP };
|
|
116
172
|
const hash = createHash("sha256").update(JSON.stringify(identity)).digest("hex").slice(0, 12);
|
|
117
173
|
const dir = MulmoStudioContextMethods.getImageProjectDirPath(context);
|
|
118
174
|
const name = `avatar_${plan.speakerId.replace(/[^\w-]/g, "_")}_${hash}`;
|
|
@@ -1,6 +1,28 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
2
|
+
import { MulmoStudioContext, MulmoBeat, MulmoImagePromptMedia, MulmoCanvasDimension } from "../types/index.js";
|
|
2
3
|
import type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
|
|
3
4
|
export type { ResolvedImageReference } from "../methods/mulmo_image_prompt_media.js";
|
|
5
|
+
type ImageAgentInfo = ReturnType<typeof MulmoPresentationStyleMethods.getImageAgentInfo>;
|
|
6
|
+
type MovieAgentInfo = ReturnType<typeof MulmoPresentationStyleMethods.getMovieAgentInfo>;
|
|
7
|
+
export declare const referenceImageGraphParams: (imageAgentInfo: ImageAgentInfo, canvasSize: MulmoCanvasDimension) => {
|
|
8
|
+
model: string | undefined;
|
|
9
|
+
canvasSize: {
|
|
10
|
+
width: number;
|
|
11
|
+
height: number;
|
|
12
|
+
};
|
|
13
|
+
vertexai_project: string | undefined;
|
|
14
|
+
vertexai_location: string | undefined;
|
|
15
|
+
};
|
|
16
|
+
export declare const referenceMovieGraphParams: (movieAgentInfo: MovieAgentInfo, canvasSize: MulmoCanvasDimension) => {
|
|
17
|
+
model: string | undefined;
|
|
18
|
+
canvasSize: {
|
|
19
|
+
width: number;
|
|
20
|
+
height: number;
|
|
21
|
+
};
|
|
22
|
+
generateAudio: boolean | undefined;
|
|
23
|
+
vertexai_project: string | undefined;
|
|
24
|
+
vertexai_location: string | undefined;
|
|
25
|
+
};
|
|
4
26
|
export declare const generateReferenceImage: (inputs: {
|
|
5
27
|
context: MulmoStudioContext;
|
|
6
28
|
key: string;
|
|
@@ -6,6 +6,20 @@ import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods, MulmoImagePromp
|
|
|
6
6
|
import { ImageMediaType, } from "../types/index.js";
|
|
7
7
|
import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent } from "../agents/index.js";
|
|
8
8
|
import { agentGenerationError, imageReferenceAction, imageFileTarget, movieFileTarget } from "../utils/error_cause.js";
|
|
9
|
+
// Reference media are generated outside the beat graphs, so they must carry the same Vertex AI target the beats do.
|
|
10
|
+
export const referenceImageGraphParams = (imageAgentInfo, canvasSize) => ({
|
|
11
|
+
model: imageAgentInfo.imageParams.model,
|
|
12
|
+
canvasSize,
|
|
13
|
+
vertexai_project: imageAgentInfo.imageParams.vertexai_project,
|
|
14
|
+
vertexai_location: imageAgentInfo.imageParams.vertexai_location,
|
|
15
|
+
});
|
|
16
|
+
export const referenceMovieGraphParams = (movieAgentInfo, canvasSize) => ({
|
|
17
|
+
model: movieAgentInfo.movieParams.model,
|
|
18
|
+
canvasSize,
|
|
19
|
+
generateAudio: movieAgentInfo.movieParams.generateAudio,
|
|
20
|
+
vertexai_project: movieAgentInfo.movieParams.vertexai_project,
|
|
21
|
+
vertexai_location: movieAgentInfo.movieParams.vertexai_location,
|
|
22
|
+
});
|
|
9
23
|
// Resolve the normalized reference list to local file paths (with labels), in reference order.
|
|
10
24
|
// imageKey is the imageRefs key of the image being generated (used for warnings and url
|
|
11
25
|
// download file naming). A named reference that is missing from imageRefs warns and is
|
|
@@ -62,10 +76,7 @@ export const generateReferenceImage = async (inputs) => {
|
|
|
62
76
|
sessionType: "imageReference",
|
|
63
77
|
},
|
|
64
78
|
},
|
|
65
|
-
params:
|
|
66
|
-
model: imageAgentInfo.imageParams.model,
|
|
67
|
-
canvasSize: image.canvasSize ?? context.presentationStyle.canvasSize,
|
|
68
|
-
},
|
|
79
|
+
params: referenceImageGraphParams(imageAgentInfo, image.canvasSize ?? context.presentationStyle.canvasSize),
|
|
69
80
|
},
|
|
70
81
|
},
|
|
71
82
|
};
|
|
@@ -145,11 +156,7 @@ const generateReferenceMovie = async (inputs) => {
|
|
|
145
156
|
sessionType: "imageReference",
|
|
146
157
|
},
|
|
147
158
|
},
|
|
148
|
-
params:
|
|
149
|
-
model: movieAgentInfo.movieParams.model,
|
|
150
|
-
canvasSize: context.presentationStyle.canvasSize,
|
|
151
|
-
generateAudio: movieAgentInfo.movieParams.generateAudio,
|
|
152
|
-
},
|
|
159
|
+
params: referenceMovieGraphParams(movieAgentInfo, context.presentationStyle.canvasSize),
|
|
153
160
|
},
|
|
154
161
|
},
|
|
155
162
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import { GraphAILogger } from "graphai";
|
|
3
|
-
import { provider2ImageAgent, deprecatedGoogleImageModelHints, vertexAIGlobalOnlyImageModels, } from "../types/provider2agent.js";
|
|
3
|
+
import { provider2ImageAgent, deprecatedGoogleImageModelHints, vertexAIGlobalOnlyImageModels, isGeminiImageModel, vertexImageLocation, } from "../types/provider2agent.js";
|
|
4
4
|
import { apiKeyMissingError, agentIncorrectAPIKeyError, agentGenerationError, agentInvalidResponseError, imageAction, imageFileTarget, unsupportedModelTarget, hasCause, getGenAIErrorReason, resultify, } from "../utils/error_cause.js";
|
|
5
5
|
import { getAspectRatio } from "../utils/utils.js";
|
|
6
6
|
import { ASPECT_RATIOS, PRO_ASPECT_RATIOS } from "../types/const.js";
|
|
@@ -85,7 +85,7 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
85
85
|
const apiKey = config?.apiKey;
|
|
86
86
|
const ai = params.vertexai_project
|
|
87
87
|
? (() => {
|
|
88
|
-
const location = params.vertexai_location
|
|
88
|
+
const location = vertexImageLocation(model, params.vertexai_location);
|
|
89
89
|
if (vertexAIGlobalOnlyImageModels.has(model) && location !== "global") {
|
|
90
90
|
GraphAILogger.warn(`imageGenAIAgent: model "${model}" on Vertex AI is only available in location "global", but got "${location}". Set imageParams.vertexai_location to "global".`);
|
|
91
91
|
}
|
|
@@ -99,7 +99,7 @@ export const imageGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
99
99
|
}
|
|
100
100
|
return new GoogleGenAI({ apiKey, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
|
|
101
101
|
})();
|
|
102
|
-
if (model
|
|
102
|
+
if (isGeminiImageModel(model)) {
|
|
103
103
|
const contentParams = (() => {
|
|
104
104
|
const contents = getGeminiContents(prompt, referenceImages);
|
|
105
105
|
return {
|
|
@@ -5,7 +5,7 @@ import { apiKeyMissingError, agentGenerationError, agentInvalidResponseError, im
|
|
|
5
5
|
import { getAspectRatio } from "../utils/utils.js";
|
|
6
6
|
import { ffmpegGetMediaDuration } from "../utils/ffmpeg_utils.js";
|
|
7
7
|
import { ASPECT_RATIOS } from "../types/const.js";
|
|
8
|
-
import { getModelDuration, provider2MovieAgent, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS } from "../types/provider2agent.js";
|
|
8
|
+
import { getModelDuration, provider2MovieAgent, AUDIO_MODE_NEVER, AUDIO_MODE_ALWAYS, defaultMovieModel, veoExtensionCount, VEO_EXTENSION_MODEL, VEO_SEGMENT_SEC, } from "../types/provider2agent.js";
|
|
9
9
|
// Per-request timeout so a stalled GenAI video API call rejects instead of hanging.
|
|
10
10
|
const GENAI_REQUEST_TIMEOUT_MS = 120_000;
|
|
11
11
|
// Wall-clock cap on the long-running video operation poll loop (Veo runs minutes).
|
|
@@ -89,14 +89,12 @@ const createVeo31Payload = (model, prompt, aspectRatio, source) => ({
|
|
|
89
89
|
...source,
|
|
90
90
|
});
|
|
91
91
|
const generateExtendedVideo = async (ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile, isVertexAI) => {
|
|
92
|
-
const
|
|
93
|
-
const maxExtensionDuration = 8;
|
|
94
|
-
const extensionsNeeded = Math.ceil((requestedDuration - initialDuration) / maxExtensionDuration);
|
|
92
|
+
const extensionsNeeded = veoExtensionCount(requestedDuration);
|
|
95
93
|
GraphAILogger.info(`Veo 3.1 video extension: ${extensionsNeeded} extensions needed for ${requestedDuration}s target`);
|
|
96
94
|
const generateIteration = async (iteration, accumulatedDuration, previousVideo) => {
|
|
97
95
|
const isInitial = iteration === 0;
|
|
98
96
|
const remainingDuration = requestedDuration - accumulatedDuration;
|
|
99
|
-
const extensionDuration = isInitial ?
|
|
97
|
+
const extensionDuration = isInitial ? VEO_SEGMENT_SEC : (getModelDuration("google", model, remainingDuration) ?? VEO_SEGMENT_SEC);
|
|
100
98
|
const getSource = () => {
|
|
101
99
|
if (isInitial)
|
|
102
100
|
return imagePath ? { image: loadImageAsBase64(imagePath) } : undefined;
|
|
@@ -172,7 +170,7 @@ const generateStandardVideo = async (ai, model, prompt, aspectRatio, imagePath,
|
|
|
172
170
|
export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
173
171
|
const { prompt, imagePath, lastFrameImagePath, referenceImages, movieFile } = namedInputs;
|
|
174
172
|
const aspectRatio = getAspectRatio(params.canvasSize, ASPECT_RATIOS);
|
|
175
|
-
const model = params.model ??
|
|
173
|
+
const model = params.model ?? defaultMovieModel("google", params);
|
|
176
174
|
const apiKey = config?.apiKey;
|
|
177
175
|
try {
|
|
178
176
|
const requestedDuration = params.duration ?? 8;
|
|
@@ -211,7 +209,7 @@ export const movieGenAIAgent = async ({ namedInputs, params, config, }) => {
|
|
|
211
209
|
return new GoogleGenAI({ apiKey, httpOptions: { timeout: GENAI_REQUEST_TIMEOUT_MS } });
|
|
212
210
|
})();
|
|
213
211
|
// Veo 3.1: Video extension mode for videos longer than 8s
|
|
214
|
-
if (model ===
|
|
212
|
+
if (model === VEO_EXTENSION_MODEL && requestedDuration > VEO_SEGMENT_SEC && params.canvasSize) {
|
|
215
213
|
return generateExtendedVideo(ai, model, prompt, aspectRatio, imagePath, requestedDuration, movieFile, isVertexAI);
|
|
216
214
|
}
|
|
217
215
|
// Standard mode
|
|
@@ -484,7 +484,7 @@ export const scriptTemplates = [
|
|
|
484
484
|
],
|
|
485
485
|
filename: "html",
|
|
486
486
|
htmlImageParams: {
|
|
487
|
-
model: "claude-sonnet-4-
|
|
487
|
+
model: "claude-sonnet-4-6",
|
|
488
488
|
provider: "anthropic",
|
|
489
489
|
},
|
|
490
490
|
lang: "en",
|
|
@@ -987,7 +987,7 @@ export const scriptTemplates = [
|
|
|
987
987
|
],
|
|
988
988
|
filename: "presentation",
|
|
989
989
|
htmlImageParams: {
|
|
990
|
-
model: "claude-sonnet-4-
|
|
990
|
+
model: "claude-sonnet-4-6",
|
|
991
991
|
provider: "anthropic",
|
|
992
992
|
},
|
|
993
993
|
lang: "en",
|
|
@@ -49,7 +49,7 @@ export const templateDataSet = {
|
|
|
49
49
|
"```",
|
|
50
50
|
html: "Another LLM will generate actual slides from the prompt and data for each beat. Adding optional data would help it to generate more compelling slide. Mention the reference in one of beats, if it exists. The valid type of reference is 'article', 'paper', 'image', 'video', 'audio'. Use the JSON below as a template.\n" +
|
|
51
51
|
"```JSON\n" +
|
|
52
|
-
'{"$mulmocast":{"version":"1.1","credit":"closing"},"references":[{"url":"https://www.somegreatwebsite.com/article/123","title":"Title of the article we are referencing","type":"[TYPE OF ARTICLE: article, paper, image, video, audio]"}],"title":"[TITLE: Brief, engaging title for the topic]","htmlImageParams":{"provider":"anthropic","model":"claude-sonnet-4-
|
|
52
|
+
'{"$mulmocast":{"version":"1.1","credit":"closing"},"references":[{"url":"https://www.somegreatwebsite.com/article/123","title":"Title of the article we are referencing","type":"[TYPE OF ARTICLE: article, paper, image, video, audio]"}],"title":"[TITLE: Brief, engaging title for the topic]","htmlImageParams":{"provider":"anthropic","model":"claude-sonnet-4-6"},"lang":"en","beats":[{"text":"[NARRATION: Narration for the beat.]","htmlPrompt":{"prompt":"[PROMPT to create appropriate HTML page for the beat.]"}},{"text":"[NARRATION: Narration for the beat.]","htmlPrompt":{"prompt":"[PROMPT to create appropriate HTML page for the beat with the data.]","data":{"description":"DATA TO BE PRESENTED IN THIS BEAT (in any format)]","net_income":{"Q2 FY2024":320,"Q3 FY2024":333,"Q4 FY2024":350},"unit":"USD (Million)"}}}],"canvasSize":{"width":1536,"height":1024}}\n' +
|
|
53
53
|
"```",
|
|
54
54
|
image_prompt: "Another AI will generate images for each beat based on the image prompt of that beat. Movie prompts must be written in English. Mention the reference in one of beats, if it exists. Use the JSON below as a template.\n" +
|
|
55
55
|
"```JSON\n" +
|
package/lib/index.remotion.d.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
1
|
export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
|
|
2
2
|
export { REMOTION_PACKAGES, REMOTION_RENDER_PACKAGES, REMOTION_SCENE_PACKAGES, remotionInstallCommand } from "./utils/remotion/packages.js";
|
|
3
3
|
export { ensureRemotionPackages, missingInstalledRemotionPackages } from "./utils/remotion/render.js";
|
|
4
|
+
export { REMOTION_REVIEW_FRACTIONS } from "./utils/remotion/frames.js";
|
|
5
|
+
export { renderRemotionFrames } from "./utils/remotion/render_frames.js";
|
|
6
|
+
export type { RemotionFrame, RemotionFramesRequest } from "./utils/remotion/render_frames.js";
|
package/lib/index.remotion.js
CHANGED
|
@@ -2,3 +2,5 @@
|
|
|
2
2
|
export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
|
|
3
3
|
export { REMOTION_PACKAGES, REMOTION_RENDER_PACKAGES, REMOTION_SCENE_PACKAGES, remotionInstallCommand } from "./utils/remotion/packages.js";
|
|
4
4
|
export { ensureRemotionPackages, missingInstalledRemotionPackages } from "./utils/remotion/render.js";
|
|
5
|
+
export { REMOTION_REVIEW_FRACTIONS } from "./utils/remotion/frames.js";
|
|
6
|
+
export { renderRemotionFrames } from "./utils/remotion/render_frames.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
// Entry point for code that only needs the remotion component guide — a tool description built into
|
|
2
|
+
// a browser bundle, for example. `mulmocast/remotion` also carries the renderer, which needs Node.
|
|
3
|
+
export { REMOTION_COMPONENT_GUIDE } from "./utils/remotion/system_prompt.js";
|
|
@@ -8,7 +8,7 @@ import { userAssert } from "../utils/utils.js";
|
|
|
8
8
|
import { slideThemes } from "../data/slideThemes.js";
|
|
9
9
|
import { text2ImageProviderSchema, text2HtmlImageProviderSchema, text2MovieProviderSchema, text2SpeechProviderSchema, mulmoCanvasDimensionSchema, mulmoTransitionSchema, mulmoFillOptionSchema, } from "../types/schema.js";
|
|
10
10
|
import { MulmoBeatMethods } from "./mulmo_beat.js";
|
|
11
|
-
import { provider2ImageAgent, provider2MovieAgent, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
|
|
11
|
+
import { provider2ImageAgent, provider2MovieAgent, defaultMovieModel, provider2LLMAgent, provider2TTSAgent, provider2SoundEffectAgent, provider2LipSyncAgent, defaultProviders, getModelAudio, AUDIO_MODE_ALWAYS, } from "../types/provider2agent.js";
|
|
12
12
|
const defaultTextSlideStyles = [
|
|
13
13
|
'*,*::before,*::after{box-sizing:border-box}body,h1,h2,h3,h4,p,figure,blockquote,dl,dd{margin:0}ul[role="list"],ol[role="list"]{list-style:none}html:focus-within{scroll-behavior:smooth}body{min-height:100vh;text-rendering:optimizeSpeed;line-height:1.5}a:not([class]){text-decoration-skip-ink:auto}img,picture{max-width:100%;display:block}input,button,textarea,select{font:inherit}@media(prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,*::before,*::after{animation-duration:.01ms !important;animation-iteration-count:1 !important;transition-duration:.01ms !important;scroll-behavior:auto !important}}',
|
|
14
14
|
"body { margin: 60px; margin-top: 40px; color:#333; background-color:#fff; font-size: 30px; font-family: Arial, sans-serif; box-sizing: border-box; height: 100vh }",
|
|
@@ -160,7 +160,7 @@ export const MulmoPresentationStyleMethods = {
|
|
|
160
160
|
const imageParams = { ...presentationStyle.imageParams, ...beat?.imageParams };
|
|
161
161
|
const provider = MulmoPresentationStyleMethods.getText2ImageProvider(imageParams?.provider) ?? defaultProviders.text2image;
|
|
162
162
|
const agentInfo = provider2ImageAgent[provider];
|
|
163
|
-
// The default text2image model is gpt-image-
|
|
163
|
+
// The default text2image model is gpt-image-2.5-sunburst from OpenAI.
|
|
164
164
|
const defaultImageParams = {
|
|
165
165
|
provider,
|
|
166
166
|
model: agentInfo.defaultModel,
|
|
@@ -189,7 +189,7 @@ export const MulmoPresentationStyleMethods = {
|
|
|
189
189
|
return true;
|
|
190
190
|
}
|
|
191
191
|
const provider = text2MovieProviderSchema.parse(movieParams?.provider ?? defaultProviders.text2movie);
|
|
192
|
-
const model = movieParams?.model ??
|
|
192
|
+
const model = movieParams?.model ?? defaultMovieModel(provider, movieParams);
|
|
193
193
|
return getModelAudio(provider, model)?.mode === AUDIO_MODE_ALWAYS;
|
|
194
194
|
},
|
|
195
195
|
getSoundEffectAgentInfo(presentationStyle, beat) {
|
|
@@ -43,20 +43,22 @@ export declare const provider2TTSAgent: {
|
|
|
43
43
|
};
|
|
44
44
|
export declare const gptImages: string[];
|
|
45
45
|
export declare const deprecatedOpenAIImageModelHints: {
|
|
46
|
-
readonly "dall-e-2": "Use 'gpt-image-
|
|
47
|
-
readonly "dall-e-3": "Use 'gpt-image-
|
|
46
|
+
readonly "dall-e-2": "Use 'gpt-image-2.5-sunburst' or another supported model.";
|
|
47
|
+
readonly "dall-e-3": "Use 'gpt-image-2.5-sunburst' or another supported model.";
|
|
48
48
|
};
|
|
49
49
|
export type DeprecatedOpenAIImageModel = keyof typeof deprecatedOpenAIImageModelHints;
|
|
50
50
|
export declare const deprecatedGoogleImageModelHints: {
|
|
51
|
-
readonly "imagen-3.0-generate-002": "Use 'gemini-
|
|
52
|
-
readonly "imagen-4.0-generate-001": "Use 'gemini-
|
|
53
|
-
readonly "imagen-4.0-ultra-generate-001": "Use 'gemini-
|
|
54
|
-
readonly "imagen-4.0-fast-generate-001": "Use 'gemini-
|
|
55
|
-
readonly "imagen-4.0-generate-preview-06-06": "Use 'gemini-
|
|
56
|
-
readonly "imagen-4.0-ultra-generate-preview-06-06": "Use 'gemini-
|
|
51
|
+
readonly "imagen-3.0-generate-002": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
|
|
52
|
+
readonly "imagen-4.0-generate-001": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
|
|
53
|
+
readonly "imagen-4.0-ultra-generate-001": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
|
|
54
|
+
readonly "imagen-4.0-fast-generate-001": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
|
|
55
|
+
readonly "imagen-4.0-generate-preview-06-06": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
|
|
56
|
+
readonly "imagen-4.0-ultra-generate-preview-06-06": "Use 'gemini-3.1-flash-lite-image' or 'gemini-3-pro-image' instead.";
|
|
57
57
|
};
|
|
58
58
|
export type DeprecatedGoogleImageModel = keyof typeof deprecatedGoogleImageModelHints;
|
|
59
|
+
export declare const isGeminiImageModel: (model: string) => boolean;
|
|
59
60
|
export declare const vertexAIGlobalOnlyImageModels: ReadonlySet<string>;
|
|
61
|
+
export declare const vertexImageLocation: (model: string, location?: string) => string;
|
|
60
62
|
export declare const provider2ImageAgent: {
|
|
61
63
|
openai: {
|
|
62
64
|
agentName: string;
|
|
@@ -169,34 +171,34 @@ export declare const provider2LipSyncAgent: {
|
|
|
169
171
|
export declare const provider2LLMAgent: {
|
|
170
172
|
readonly openai: {
|
|
171
173
|
readonly agentName: "openAIAgent";
|
|
172
|
-
readonly defaultModel: "gpt-5";
|
|
174
|
+
readonly defaultModel: "gpt-5.6-sol";
|
|
173
175
|
readonly keyName: "OPENAI_API_KEY";
|
|
174
176
|
readonly baseURLKeyName: "OPENAI_BASE_URL";
|
|
175
177
|
readonly apiVersionKeyName: "OPENAI_API_VERSION";
|
|
176
178
|
readonly max_tokens: 8192;
|
|
177
|
-
readonly models: readonly ["gpt-5", "gpt-5-nano", "gpt-5-mini", "gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano", "o3", "o3-mini", "o3-pro", "o1", "o1-pro", "gpt-4o", "gpt-4o-mini"];
|
|
179
|
+
readonly models: readonly ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5", "gpt-5-nano", "gpt-5-mini", "gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano", "o3", "o3-mini", "o3-pro", "o1", "o1-pro", "gpt-4o", "gpt-4o-mini"];
|
|
178
180
|
};
|
|
179
181
|
readonly anthropic: {
|
|
180
182
|
readonly agentName: "anthropicAgent";
|
|
181
|
-
readonly defaultModel: "claude-sonnet-4-
|
|
183
|
+
readonly defaultModel: "claude-sonnet-4-6";
|
|
182
184
|
readonly max_tokens: 8192;
|
|
183
|
-
readonly models: readonly ["claude-
|
|
185
|
+
readonly models: readonly ["claude-sonnet-4-6", "claude-sonnet-4-5-20250929", "claude-haiku-4-5-20251001"];
|
|
184
186
|
readonly keyName: "ANTHROPIC_API_KEY";
|
|
185
187
|
readonly apiKeyNameOverride: "ANTHROPIC_API_TOKEN";
|
|
186
188
|
};
|
|
187
189
|
readonly gemini: {
|
|
188
190
|
readonly agentName: "geminiAgent";
|
|
189
|
-
readonly defaultModel: "gemini-
|
|
191
|
+
readonly defaultModel: "gemini-3.8-flash";
|
|
190
192
|
readonly max_tokens: 8192;
|
|
191
|
-
readonly models: readonly ["gemini-3.1-pro-preview", "gemini-3-flash-preview", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite"];
|
|
193
|
+
readonly models: readonly ["gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3-flash-preview", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite"];
|
|
192
194
|
readonly keyName: "GEMINI_API_KEY";
|
|
193
195
|
};
|
|
194
196
|
readonly groq: {
|
|
195
197
|
readonly agentName: "groqAgent";
|
|
196
|
-
readonly defaultModel: "
|
|
198
|
+
readonly defaultModel: "openai/gpt-oss-20b";
|
|
197
199
|
readonly keyName: "GROQ_API_KEY";
|
|
198
200
|
readonly max_tokens: 4096;
|
|
199
|
-
readonly models: readonly ["
|
|
201
|
+
readonly models: readonly ["openai/gpt-oss-20b", "openai/gpt-oss-120b"];
|
|
200
202
|
};
|
|
201
203
|
readonly mock: {
|
|
202
204
|
readonly agentName: "mediaMockAgent";
|
|
@@ -229,6 +231,13 @@ export declare const ProviderName: {
|
|
|
229
231
|
export declare const isImageProvider: (provider: string | undefined) => provider is keyof typeof provider2ImageAgent;
|
|
230
232
|
export declare const getMaxImageReferenceImages: (provider: string | undefined, model: string) => number | undefined;
|
|
231
233
|
export declare const getModelAudio: (provider: keyof typeof provider2MovieAgent, model: string) => MovieAudioSpec | undefined;
|
|
234
|
+
export declare const defaultMovieModel: (provider: keyof typeof provider2MovieAgent, movieParams?: {
|
|
235
|
+
vertexai_project?: string;
|
|
236
|
+
}) => string;
|
|
237
|
+
export declare const VEO_EXTENSION_MODEL = "veo-3.1-generate-preview";
|
|
238
|
+
export declare const VEO_SEGMENT_SEC = 8;
|
|
239
|
+
export declare const veoExtensionCount: (requestedSec: number) => number;
|
|
240
|
+
export declare const veoExtendedSeconds: (requestedSec: number) => number;
|
|
232
241
|
export declare const getModelDuration: (provider: keyof typeof provider2MovieAgent, model: string, movieDuration?: number) => number | undefined;
|
|
233
242
|
export type ModelPricing = {
|
|
234
243
|
unit: "tokens" | "chars" | "seconds" | "images";
|