mulmocast 2.17.0 → 2.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -38
- package/lib/actions/captions.js +1 -1
- package/lib/actions/image_agents.d.ts +1 -0
- package/lib/actions/image_agents.js +2 -2
- package/lib/actions/images.d.ts +2 -0
- package/lib/actions/images.js +1 -0
- package/lib/actions/pdf.js +23 -2
- package/lib/agents/movie_genai_agent.d.ts +24 -0
- package/lib/agents/movie_genai_agent.js +56 -43
- package/lib/cli/commands/bundle/builder.d.ts +2 -0
- package/lib/cli/commands/html/builder.d.ts +2 -0
- package/lib/cli/commands/markdown/builder.d.ts +2 -0
- package/lib/cli/commands/viewer/builder.d.ts +2 -0
- package/lib/cli/common.d.ts +2 -0
- package/lib/cli/common.js +5 -0
- package/lib/cli/helpers.js +3 -1
- package/lib/methods/mulmo_studio_context.d.ts +4 -0
- package/lib/methods/mulmo_studio_context.js +7 -0
- package/lib/types/provider2agent.d.ts +1 -4
- package/lib/types/provider2agent.js +27 -24
- package/lib/types/type.d.ts +2 -0
- package/lib/utils/estimate_usage.js +6 -5
- package/lib/utils/gemini_omni_video.d.ts +49 -0
- package/lib/utils/gemini_omni_video.js +36 -0
- package/lib/utils/html_render.d.ts +6 -5
- package/lib/utils/html_render.js +37 -30
- package/lib/utils/image_plugins/chart.js +2 -1
- package/lib/utils/image_plugins/html_tailwind.js +5 -4
- package/lib/utils/image_plugins/markdown.js +2 -1
- package/lib/utils/image_plugins/mermaid.js +2 -1
- package/lib/utils/image_plugins/slide.js +2 -1
- package/lib/utils/image_plugins/text_slide.js +2 -1
- package/lib/utils/image_plugins/vision.js +7 -0
- package/lib/utils/render_network_guard.d.ts +2 -0
- package/lib/utils/render_network_guard.js +36 -0
- package/lib/utils/render_network_policy.d.ts +15 -0
- package/lib/utils/render_network_policy.js +85 -0
- package/package.json +1 -1
- package/scripts/test/test_genai.json +10 -9
- package/scripts/test/test_genai_movie.json +1 -1
- package/scripts/test/test_image_references.json +3 -2
- package/scripts/test/test_movie_references.json +2 -1
- package/scripts/test/test_veo31_lite.json +3 -3
package/lib/utils/html_render.js
CHANGED
|
@@ -4,6 +4,8 @@ import nodePath from "node:path";
|
|
|
4
4
|
import crypto from "node:crypto";
|
|
5
5
|
import { marked } from "marked";
|
|
6
6
|
import puppeteer from "puppeteer";
|
|
7
|
+
import { guardRenderPage } from "./render_network_guard.js";
|
|
8
|
+
import { strictNetworkLaunchArgs, withRenderContentSecurityPolicy } from "./render_network_policy.js";
|
|
7
9
|
import { GraphAILogger } from "graphai";
|
|
8
10
|
const isCI = process.env.CI === "true";
|
|
9
11
|
const VIDEO_LOAD_TIMEOUT_MS = 15000;
|
|
@@ -83,15 +85,16 @@ const resolveWaitUntil = (html) => {
|
|
|
83
85
|
* and navigate via page.goto (setContent uses about:blank origin
|
|
84
86
|
* which blocks file:// loading).
|
|
85
87
|
*/
|
|
86
|
-
const
|
|
88
|
+
const renderTempPagePath = () => nodePath.join(os.tmpdir(), `mulmocast_render_${crypto.randomUUID()}.html`);
|
|
89
|
+
const loadHtmlIntoPage = async (page, html, timeout_ms, forcedFilePath) => {
|
|
87
90
|
const waitUntil = resolveWaitUntil(html);
|
|
88
91
|
const hasFileUrls = /file:\/\//.test(html);
|
|
89
92
|
const hasExternalScripts = /script src=["']https?:\/\//.test(html);
|
|
90
93
|
// puppeteer's setContent does not accept "networkidle0" (no navigation, so network-idle states aren't meaningful in types).
|
|
91
94
|
// Route any HTML that needs network-idle semantics through the page.goto path to preserve the original wait behavior.
|
|
92
95
|
const needsNetworkIdle = waitUntil === "networkidle0";
|
|
93
|
-
if (hasFileUrls || hasExternalScripts || needsNetworkIdle) {
|
|
94
|
-
const tmpFile =
|
|
96
|
+
if (forcedFilePath || hasFileUrls || hasExternalScripts || needsNetworkIdle) {
|
|
97
|
+
const tmpFile = forcedFilePath ?? renderTempPagePath();
|
|
95
98
|
fs.writeFileSync(tmpFile, html);
|
|
96
99
|
try {
|
|
97
100
|
await page.goto(`file://${tmpFile}`, { waitUntil, timeout: timeout_ms });
|
|
@@ -109,14 +112,28 @@ const loadHtmlIntoPage = async (page, html, timeout_ms) => {
|
|
|
109
112
|
await page.setContent(html, { waitUntil, timeout: timeout_ms });
|
|
110
113
|
}
|
|
111
114
|
};
|
|
112
|
-
|
|
115
|
+
const RENDER_LOAD_TIMEOUT_MS = 30000;
|
|
116
|
+
const renderLaunchArgs = ({ strictNetwork = false }) => [
|
|
117
|
+
...(isCI ? ["--no-sandbox"] : []),
|
|
118
|
+
"--allow-file-access-from-files",
|
|
119
|
+
...(strictNetwork ? strictNetworkLaunchArgs() : []),
|
|
120
|
+
];
|
|
121
|
+
// Strict mode always loads through a file: the guard's WebRTC removal never reaches a setContent document.
|
|
122
|
+
const loadGuardedPage = async (page, html, allowedFileRoots) => {
|
|
123
|
+
const pagePath = renderTempPagePath();
|
|
124
|
+
await guardRenderPage(page, [pagePath, ...allowedFileRoots]);
|
|
125
|
+
await loadHtmlIntoPage(page, withRenderContentSecurityPolicy(html), RENDER_LOAD_TIMEOUT_MS, pagePath);
|
|
126
|
+
};
|
|
127
|
+
const openRenderPage = async (browser, html, { strictNetwork = false, allowedFileRoots = [] }) => {
|
|
128
|
+
const page = await browser.newPage();
|
|
129
|
+
await (strictNetwork ? loadGuardedPage(page, html, allowedFileRoots) : loadHtmlIntoPage(page, html, RENDER_LOAD_TIMEOUT_MS));
|
|
130
|
+
return page;
|
|
131
|
+
};
|
|
132
|
+
export const renderHTMLToImage = async (html, outputPath, width, height, isMermaid = false, omitBackground = false, network = {}) => {
|
|
113
133
|
// Use Puppeteer to render HTML to an image
|
|
114
|
-
const browser = await puppeteer.launch({
|
|
115
|
-
args: isCI ? ["--no-sandbox", "--allow-file-access-from-files"] : ["--allow-file-access-from-files"],
|
|
116
|
-
});
|
|
134
|
+
const browser = await puppeteer.launch({ args: renderLaunchArgs(network) });
|
|
117
135
|
try {
|
|
118
|
-
const page = await browser
|
|
119
|
-
await loadHtmlIntoPage(page, html, 30000);
|
|
136
|
+
const page = await openRenderPage(browser, html, network);
|
|
120
137
|
// Adjust page settings if needed (like width, height, etc.)
|
|
121
138
|
await page.setViewport({ width, height });
|
|
122
139
|
// height:100% ensures background fills viewport; only reset html, let body styles come from custom CSS
|
|
@@ -158,14 +175,10 @@ export const renderHTMLToImage = async (html, outputPath, width, height, isMerma
|
|
|
158
175
|
*
|
|
159
176
|
* The user-defined render() function may be sync or async.
|
|
160
177
|
*/
|
|
161
|
-
export const renderHTMLToFrames = async (html, outputDir, width, height, totalFrames, fps) => {
|
|
162
|
-
const browser = await puppeteer.launch({
|
|
163
|
-
args: isCI ? ["--no-sandbox", "--allow-file-access-from-files"] : ["--allow-file-access-from-files"],
|
|
164
|
-
});
|
|
178
|
+
export const renderHTMLToFrames = async (html, outputDir, width, height, totalFrames, fps, network = {}) => {
|
|
179
|
+
const browser = await puppeteer.launch({ args: renderLaunchArgs(network) });
|
|
165
180
|
try {
|
|
166
|
-
const page = await browser
|
|
167
|
-
// Wait for Tailwind CSS CDN to load
|
|
168
|
-
await loadHtmlIntoPage(page, html, 30000);
|
|
181
|
+
const page = await openRenderPage(browser, html, network);
|
|
169
182
|
await page.setViewport({ width, height });
|
|
170
183
|
await page.addStyleTag({ content: "html{height:100%;margin:0;padding:0;overflow:hidden}" });
|
|
171
184
|
// Scale content to fit viewport (same logic as renderHTMLToImage)
|
|
@@ -199,14 +212,11 @@ export const renderHTMLToFrames = async (html, outputDir, width, height, totalFr
|
|
|
199
212
|
* The animation plays in real-time via requestAnimationFrame, and
|
|
200
213
|
* page.screencast() captures frames directly to an mp4 file.
|
|
201
214
|
*/
|
|
202
|
-
export const renderHTMLToVideo = async (html, videoPath, width, height, totalFrames, fps) => {
|
|
215
|
+
export const renderHTMLToVideo = async (html, videoPath, width, height, totalFrames, fps, network = {}) => {
|
|
203
216
|
const duration_ms = (totalFrames / fps) * 1000;
|
|
204
|
-
const browser = await puppeteer.launch({
|
|
205
|
-
args: isCI ? ["--no-sandbox", "--allow-file-access-from-files"] : ["--allow-file-access-from-files"],
|
|
206
|
-
});
|
|
217
|
+
const browser = await puppeteer.launch({ args: renderLaunchArgs(network) });
|
|
207
218
|
try {
|
|
208
|
-
const page = await browser
|
|
209
|
-
await loadHtmlIntoPage(page, html, 30000);
|
|
219
|
+
const page = await openRenderPage(browser, html, network);
|
|
210
220
|
await page.setViewport({ width, height });
|
|
211
221
|
await page.addStyleTag({ content: "html{height:100%;margin:0;padding:0;overflow:hidden}" });
|
|
212
222
|
await scaleContentToFit(page, width, height);
|
|
@@ -242,13 +252,10 @@ export const renderHTMLToVideo = async (html, videoPath, width, height, totalFra
|
|
|
242
252
|
* Loads the animated HTML, calls window.renderFinal() to set all animations
|
|
243
253
|
* to their end state, then takes a screenshot. Used for PDF/thumbnail generation.
|
|
244
254
|
*/
|
|
245
|
-
export const renderHTMLToFinalFrame = async (html, outputPath, width, height) => {
|
|
246
|
-
const browser = await puppeteer.launch({
|
|
247
|
-
args: isCI ? ["--no-sandbox", "--allow-file-access-from-files"] : ["--allow-file-access-from-files"],
|
|
248
|
-
});
|
|
255
|
+
export const renderHTMLToFinalFrame = async (html, outputPath, width, height, network = {}) => {
|
|
256
|
+
const browser = await puppeteer.launch({ args: renderLaunchArgs(network) });
|
|
249
257
|
try {
|
|
250
|
-
const page = await browser
|
|
251
|
-
await loadHtmlIntoPage(page, html, 30000);
|
|
258
|
+
const page = await openRenderPage(browser, html, network);
|
|
252
259
|
await page.setViewport({ width, height });
|
|
253
260
|
await page.addStyleTag({ content: "html{height:100%;margin:0;padding:0;overflow:hidden}" });
|
|
254
261
|
await scaleContentToFit(page, width, height);
|
|
@@ -266,11 +273,11 @@ export const renderHTMLToFinalFrame = async (html, outputPath, width, height) =>
|
|
|
266
273
|
await browser.close();
|
|
267
274
|
}
|
|
268
275
|
};
|
|
269
|
-
export const renderMarkdownToImage = async (markdown, style, outputPath, width, height) => {
|
|
276
|
+
export const renderMarkdownToImage = async (markdown, style, outputPath, width, height, network = {}) => {
|
|
270
277
|
const header = `<head><style>${style}</style></head>`;
|
|
271
278
|
const body = await marked(markdown);
|
|
272
279
|
const html = `<html>${header}<body>${body}</body></html>`;
|
|
273
|
-
await renderHTMLToImage(html, outputPath, width, height);
|
|
280
|
+
await renderHTMLToImage(html, outputPath, width, height, false, false, network);
|
|
274
281
|
};
|
|
275
282
|
export const interpolate = (template, data) => {
|
|
276
283
|
return template.replace(/\$\{(.*?)\}/g, (_, key) => data[key.trim()] ?? "");
|
|
@@ -3,6 +3,7 @@ import { renderHTMLToImage, interpolate } from "../html_render.js";
|
|
|
3
3
|
import { parrotingImagePath, generateUniqueId } from "./utils.js";
|
|
4
4
|
import { resolveCombinedStyle } from "./bg_image_util.js";
|
|
5
5
|
import { chartHtml, escapedChartTemplateValues, resolveChartPlugins, stringifyChartData } from "./chart_html.js";
|
|
6
|
+
import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
|
|
6
7
|
export const imageType = "chart";
|
|
7
8
|
const processChart = async (params) => {
|
|
8
9
|
const { beat, imagePath, canvasSize } = params;
|
|
@@ -19,7 +20,7 @@ const processChart = async (params) => {
|
|
|
19
20
|
chart_width: chart_width.toString(),
|
|
20
21
|
chart_plugins: resolveChartPlugins(chartType),
|
|
21
22
|
});
|
|
22
|
-
await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
|
|
23
|
+
await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, false, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
|
|
23
24
|
return imagePath;
|
|
24
25
|
};
|
|
25
26
|
const dumpHtml = async (params) => {
|
|
@@ -7,6 +7,7 @@ import { framesToVideo } from "../ffmpeg_utils.js";
|
|
|
7
7
|
import { parrotingImagePath } from "./utils.js";
|
|
8
8
|
import { swipeElementsToHtml, swipeElementsToScript } from "../swipe_to_html.js";
|
|
9
9
|
import { htmlTailwindMarkup } from "./html_tailwind_markup.js";
|
|
10
|
+
import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
|
|
10
11
|
export const imageType = "html_tailwind";
|
|
11
12
|
/**
|
|
12
13
|
* Resolve image:name references to file:// absolute paths using imageRefs.
|
|
@@ -178,13 +179,13 @@ const processHtmlTailwindAnimated = async (params) => {
|
|
|
178
179
|
// imagePath is set to the .mp4 path by imagePluginAgent for animated beats
|
|
179
180
|
const videoPath = imagePath;
|
|
180
181
|
if (animConfig.movie) {
|
|
181
|
-
await renderHTMLToVideo(htmlData, videoPath, canvasSize.width, canvasSize.height, totalFrames, fps);
|
|
182
|
+
await renderHTMLToVideo(htmlData, videoPath, canvasSize.width, canvasSize.height, totalFrames, fps, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
|
|
182
183
|
}
|
|
183
184
|
else {
|
|
184
185
|
const framesDir = videoPath.replace(/\.[^/.]+$/, "_frames");
|
|
185
186
|
fs.mkdirSync(framesDir, { recursive: true });
|
|
186
187
|
try {
|
|
187
|
-
await renderHTMLToFrames(htmlData, framesDir, canvasSize.width, canvasSize.height, totalFrames, fps);
|
|
188
|
+
await renderHTMLToFrames(htmlData, framesDir, canvasSize.width, canvasSize.height, totalFrames, fps, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
|
|
188
189
|
await framesToVideo(framesDir, videoPath, fps, canvasSize.width, canvasSize.height);
|
|
189
190
|
}
|
|
190
191
|
finally {
|
|
@@ -197,7 +198,7 @@ const processHtmlTailwindAnimated = async (params) => {
|
|
|
197
198
|
// even when exact duration is unknown (e.g., PDF generation without audio).
|
|
198
199
|
const finalFramePath = imagePath.replace(/_animated\.mp4$/, ".png");
|
|
199
200
|
const finalHtml = buildAnimatedHtml(params, FINAL_FRAME_TOTAL, fps);
|
|
200
|
-
await renderHTMLToFinalFrame(finalHtml, finalFramePath, canvasSize.width, canvasSize.height);
|
|
201
|
+
await renderHTMLToFinalFrame(finalHtml, finalFramePath, canvasSize.width, canvasSize.height, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
|
|
201
202
|
// Return video path when video was generated, otherwise return the static PNG path
|
|
202
203
|
return duration !== undefined ? imagePath : finalFramePath;
|
|
203
204
|
};
|
|
@@ -216,7 +217,7 @@ const processHtmlTailwindStatic = async (params) => {
|
|
|
216
217
|
const resolvedAllRefs = resolveMovieRefs(resolvedImageRefs, params.movieRefs ?? {});
|
|
217
218
|
const resolvedImages = resolveRelativeImagePaths(resolvedAllRefs, context.fileDirs.mulmoFileDirPath);
|
|
218
219
|
const htmlData = resolveRelativeModelPathsInScript(resolvedImages, context.fileDirs.mulmoFileDirPath);
|
|
219
|
-
await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height);
|
|
220
|
+
await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, false, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
|
|
220
221
|
return imagePath;
|
|
221
222
|
};
|
|
222
223
|
const processHtmlTailwind = async (params) => {
|
|
@@ -5,6 +5,7 @@ import { resolveCombinedStyle } from "./bg_image_util.js";
|
|
|
5
5
|
import { renderMarkdownContent, renderMarkdownLayout, layoutToMarkdown, toMarkdownString } from "./markdown_layout.js";
|
|
6
6
|
import { resolveImageRefs, resolveMovieRefs } from "./html_tailwind.js";
|
|
7
7
|
import { isObject } from "graphai";
|
|
8
|
+
import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
|
|
8
9
|
/**
|
|
9
10
|
* This path renders once to a PNG, so a random element id is fine. The browser path needs
|
|
10
11
|
* a stable one and supplies its own — that generator is the only thing the two callers of
|
|
@@ -77,7 +78,7 @@ const processMarkdown = async (params) => {
|
|
|
77
78
|
const { html: rawHtml, hasMermaid } = await generateHtml(params);
|
|
78
79
|
const resolvedImages = resolveImageRefs(rawHtml, params.imageRefs ?? {});
|
|
79
80
|
const html = resolveMovieRefs(resolvedImages, params.movieRefs ?? {});
|
|
80
|
-
await renderHTMLToImage(html, imagePath, canvasSize.width, canvasSize.height, hasMermaid);
|
|
81
|
+
await renderHTMLToImage(html, imagePath, canvasSize.width, canvasSize.height, hasMermaid, false, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
|
|
81
82
|
return imagePath;
|
|
82
83
|
};
|
|
83
84
|
const dumpHtml = async (params) => {
|
|
@@ -5,6 +5,7 @@ import { parrotingImagePath, generateUniqueId } from "./utils.js";
|
|
|
5
5
|
import { escapedMermaidTemplateValues, mermaidHtml } from "./mermaid_html.js";
|
|
6
6
|
import { resolveCombinedStyle } from "./bg_image_util.js";
|
|
7
7
|
import { resolveImageRefs, resolveMovieRefs, resolveRelativeImagePaths } from "./html_tailwind.js";
|
|
8
|
+
import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
|
|
8
9
|
export const imageType = "mermaid";
|
|
9
10
|
/** Mermaid markup for this path. The markup itself is shared with the browser path. */
|
|
10
11
|
export const generateMermaidHtml = (code, title) => mermaidHtml(code, generateUniqueId("mermaid"), title);
|
|
@@ -23,7 +24,7 @@ const processMermaid = async (params) => {
|
|
|
23
24
|
const resolvedImageRefs = resolveImageRefs(rawHtml, params.imageRefs ?? {});
|
|
24
25
|
const resolvedAllRefs = resolveMovieRefs(resolvedImageRefs, params.movieRefs ?? {});
|
|
25
26
|
const htmlData = resolveRelativeImagePaths(resolvedAllRefs, context.fileDirs.mulmoFileDirPath);
|
|
26
|
-
await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, true);
|
|
27
|
+
await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, true, false, MulmoStudioContextMethods.getRenderNetworkOptions(context));
|
|
27
28
|
}
|
|
28
29
|
return imagePath;
|
|
29
30
|
};
|
|
@@ -7,6 +7,7 @@ import { pathToDataUrl } from "../../methods/mulmo_media_source.js";
|
|
|
7
7
|
import { MulmoMediaSourceMethods } from "../../methods/mulmo_media_source.js";
|
|
8
8
|
import { MulmoPresentationStyleMethods } from "../../methods/mulmo_presentation_style.js";
|
|
9
9
|
import { imageAction, imageFileTarget, unknownMediaType } from "../error_cause.js";
|
|
10
|
+
import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
|
|
10
11
|
export const imageType = "slide";
|
|
11
12
|
const slideImageRefError = (refKey) => ({
|
|
12
13
|
type: unknownMediaType,
|
|
@@ -167,7 +168,7 @@ const processSlide = async (params) => {
|
|
|
167
168
|
const reference = beat.image.reference;
|
|
168
169
|
const resolvedBranding = await resolveAndConvertBranding(params);
|
|
169
170
|
const html = generateSlideHTML(theme, slide, reference, resolvedBranding);
|
|
170
|
-
await renderHTMLToImage(html, imagePath, canvasSize.width, canvasSize.height);
|
|
171
|
+
await renderHTMLToImage(html, imagePath, canvasSize.width, canvasSize.height, false, false, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
|
|
171
172
|
return imagePath;
|
|
172
173
|
};
|
|
173
174
|
/**
|
|
@@ -2,6 +2,7 @@ import { renderMarkdownToImage } from "../html_render.js";
|
|
|
2
2
|
import { parrotingImagePath } from "./utils.js";
|
|
3
3
|
import { resolveCombinedStyle } from "./bg_image_util.js";
|
|
4
4
|
import { marked } from "marked";
|
|
5
|
+
import { MulmoStudioContextMethods } from "../../methods/mulmo_studio_context.js";
|
|
5
6
|
export const imageType = "textSlide";
|
|
6
7
|
const processTextSlide = async (params) => {
|
|
7
8
|
const { beat, imagePath, canvasSize } = params;
|
|
@@ -17,7 +18,7 @@ const processTextSlide = async (params) => {
|
|
|
17
18
|
const marginTop = slide.subtitle ? canvasSize.height * 0.4 : canvasSize.height * 0.45;
|
|
18
19
|
return `body {margin-top: ${marginTop}px;}`;
|
|
19
20
|
})();
|
|
20
|
-
await renderMarkdownToImage(markdown, combinedStyle + topMargin, imagePath, canvasSize.width, canvasSize.height);
|
|
21
|
+
await renderMarkdownToImage(markdown, combinedStyle + topMargin, imagePath, canvasSize.width, canvasSize.height, MulmoStudioContextMethods.getRenderNetworkOptions(params.context));
|
|
21
22
|
return imagePath;
|
|
22
23
|
};
|
|
23
24
|
const dumpMarkdown = (params) => {
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
import { parrotingImagePath } from "./utils.js";
|
|
2
2
|
import { htmlPlugin, templateNameTofunctionName } from "mulmocast-vision";
|
|
3
3
|
import { resolve as resolvePath } from "path";
|
|
4
|
+
import { agentGenerationError, imageAction, unsupportedImageTypeTarget } from "../error_cause.js";
|
|
4
5
|
export const imageType = "vision";
|
|
5
6
|
const processVision = async (params) => {
|
|
6
7
|
const { beat, imagePath, context } = params;
|
|
7
8
|
const rootDir = context.fileDirs.nodeModuleRootPath ? resolvePath(context.fileDirs.nodeModuleRootPath, "mulmocast-vision") : undefined;
|
|
8
9
|
if (!beat?.image || beat.image.type !== imageType)
|
|
9
10
|
return;
|
|
11
|
+
// mulmocast-vision launches its own browser, which the strict network guard cannot reach.
|
|
12
|
+
if (context.strictNetwork) {
|
|
13
|
+
throw new Error("vision beats cannot be rendered with --strict-network: mulmocast-vision renders in its own browser, outside the network guard", {
|
|
14
|
+
cause: agentGenerationError("visionPlugin", imageAction, unsupportedImageTypeTarget),
|
|
15
|
+
});
|
|
16
|
+
}
|
|
10
17
|
const handler = new htmlPlugin({ rootDir });
|
|
11
18
|
await handler.callNamedFunction(templateNameTofunctionName(beat.image.style), beat.image.data, {
|
|
12
19
|
functionName: beat.image.style,
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import { GraphAILogger } from "graphai";
|
|
3
|
+
import { WEBRTC_REMOVAL_SCRIPT } from "./remotion/network_policy.js";
|
|
4
|
+
import { POPUP_BLOCK_SCRIPT, isAllowedRenderRequest, realRoot } from "./render_network_policy.js";
|
|
5
|
+
const fileRealPath = (filePath) => {
|
|
6
|
+
try {
|
|
7
|
+
return fs.realpathSync.native(filePath);
|
|
8
|
+
}
|
|
9
|
+
catch {
|
|
10
|
+
return undefined;
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
// Must run before the page loads anything. The WebRTC removal only reaches documents created after this call,
|
|
14
|
+
// which is why strict mode always navigates to a file instead of using setContent.
|
|
15
|
+
export const guardRenderPage = async (page, allowedFileRoots) => {
|
|
16
|
+
const realRoots = allowedFileRoots.map((root) => realRoot(root, fileRealPath));
|
|
17
|
+
const reported = new Set();
|
|
18
|
+
await page.setRequestInterception(true);
|
|
19
|
+
page.on("request", (request) => {
|
|
20
|
+
if (request.isInterceptResolutionHandled())
|
|
21
|
+
return;
|
|
22
|
+
const url = request.url();
|
|
23
|
+
// The page can close while a request is still pending; that rejection is not a failure of the render.
|
|
24
|
+
if (isAllowedRenderRequest(url, realRoots, fileRealPath)) {
|
|
25
|
+
request.continue().catch(() => undefined);
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
if (!reported.has(url)) {
|
|
29
|
+
reported.add(url);
|
|
30
|
+
GraphAILogger.info(`strict network: blocked ${url}`);
|
|
31
|
+
}
|
|
32
|
+
request.abort("blockedbyclient").catch(() => undefined);
|
|
33
|
+
});
|
|
34
|
+
await page.evaluateOnNewDocument(WEBRTC_REMOVAL_SCRIPT);
|
|
35
|
+
await page.evaluateOnNewDocument(POPUP_BLOCK_SCRIPT);
|
|
36
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export declare const RENDER_CDN_HOSTS: string[];
|
|
2
|
+
export declare const RENDER_JSDELIVR_PATHS: string[];
|
|
3
|
+
export type RenderNetworkOptions = {
|
|
4
|
+
strictNetwork?: boolean;
|
|
5
|
+
allowedFileRoots?: readonly string[];
|
|
6
|
+
};
|
|
7
|
+
export declare const STRICT_NETWORK_ENV = "MULMO_STRICT_NETWORK";
|
|
8
|
+
export declare const strictNetworkFromEnv: (env: Record<string, string | undefined>) => boolean;
|
|
9
|
+
export type RealPath = (filePath: string) => string | undefined;
|
|
10
|
+
export declare const isAllowedRenderRequest: (url: string, realRoots?: readonly string[], realPath?: RealPath) => boolean;
|
|
11
|
+
export declare const realRoot: (root: string, realPath: RealPath) => string;
|
|
12
|
+
export declare const strictNetworkLaunchArgs: () => string[];
|
|
13
|
+
export declare const POPUP_BLOCK_SCRIPT = "Object.defineProperty(globalThis, \"open\", { value: () => null, writable: false, configurable: false });";
|
|
14
|
+
export declare const RENDER_CONTENT_SECURITY_POLICY = "connect-src 'self' data: blob:";
|
|
15
|
+
export declare const withRenderContentSecurityPolicy: (html: string) => string;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import nodePath from "node:path";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
// What an HTML renderer's page may reach in strict network mode: files under the allowed roots, in-memory
|
|
4
|
+
// URLs, and the CDNs our own templates load from. Kept free of I/O so the rules can be tested without a browser.
|
|
5
|
+
export const RENDER_CDN_HOSTS = ["cdn.tailwindcss.com", "cdn.jsdelivr.net", "fonts.googleapis.com", "fonts.gstatic.com"];
|
|
6
|
+
// jsDelivr serves any file of any package and publishes per-version and per-file hit counts, so a page free to
|
|
7
|
+
// choose the path could signal through them; only the exact files our templates load are allowed.
|
|
8
|
+
export const RENDER_JSDELIVR_PATHS = [
|
|
9
|
+
"/npm/mermaid/dist/mermaid.min.js",
|
|
10
|
+
"/npm/mermaid@10/dist/mermaid.min.js",
|
|
11
|
+
"/npm/chart.js",
|
|
12
|
+
"/npm/chart.js@4",
|
|
13
|
+
"/npm/chartjs-chart-sankey",
|
|
14
|
+
"/npm/chartjs-chart-treemap@3",
|
|
15
|
+
];
|
|
16
|
+
const JSDELIVR_HOST = "cdn.jsdelivr.net";
|
|
17
|
+
const IN_MEMORY_SCHEMES = ["data:", "blob:", "about:"];
|
|
18
|
+
export const STRICT_NETWORK_ENV = "MULMO_STRICT_NETWORK";
|
|
19
|
+
export const strictNetworkFromEnv = (env) => ["1", "true"].includes(env[STRICT_NETWORK_ENV] ?? "");
|
|
20
|
+
const parseUrl = (url) => {
|
|
21
|
+
try {
|
|
22
|
+
return new URL(url);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
const filePathOf = (parsed) => {
|
|
29
|
+
try {
|
|
30
|
+
return fileURLToPath(parsed);
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return undefined;
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
const isWithin = (root, filePath) => {
|
|
37
|
+
const relative = nodePath.relative(root, filePath);
|
|
38
|
+
return relative === "" || (!relative.startsWith("..") && !nodePath.isAbsolute(relative));
|
|
39
|
+
};
|
|
40
|
+
// The URL parser has already collapsed dot segments (encoded or not); the roots must already be real paths.
|
|
41
|
+
const isInsideFileRoots = (parsed, realRoots, realPath) => {
|
|
42
|
+
const filePath = filePathOf(parsed);
|
|
43
|
+
const realFilePath = filePath === undefined ? undefined : realPath(filePath);
|
|
44
|
+
return realFilePath !== undefined && realRoots.some((root) => isWithin(root, realFilePath));
|
|
45
|
+
};
|
|
46
|
+
const identityRealPath = (filePath) => nodePath.resolve(filePath);
|
|
47
|
+
const isAllowedCdnUrl = (parsed) => {
|
|
48
|
+
if (parsed.protocol !== "https:" || parsed.port !== "" || !RENDER_CDN_HOSTS.includes(parsed.hostname))
|
|
49
|
+
return false;
|
|
50
|
+
return parsed.hostname !== JSDELIVR_HOST || (parsed.search === "" && RENDER_JSDELIVR_PATHS.includes(parsed.pathname));
|
|
51
|
+
};
|
|
52
|
+
export const isAllowedRenderRequest = (url, realRoots = [], realPath = identityRealPath) => {
|
|
53
|
+
const parsed = parseUrl(url);
|
|
54
|
+
if (!parsed)
|
|
55
|
+
return false;
|
|
56
|
+
if (IN_MEMORY_SCHEMES.includes(parsed.protocol))
|
|
57
|
+
return true;
|
|
58
|
+
if (parsed.protocol === "file:")
|
|
59
|
+
return isInsideFileRoots(parsed, realRoots, realPath);
|
|
60
|
+
return isAllowedCdnUrl(parsed);
|
|
61
|
+
};
|
|
62
|
+
// A root that does not exist yet (the temp page before it is written) is its real parent plus its name.
|
|
63
|
+
export const realRoot = (root, realPath) => {
|
|
64
|
+
const resolved = nodePath.resolve(root);
|
|
65
|
+
const parent = nodePath.dirname(resolved);
|
|
66
|
+
return realPath(resolved) ?? nodePath.join(realPath(parent) ?? parent, nodePath.basename(resolved));
|
|
67
|
+
};
|
|
68
|
+
// Popups and preconnect hints open connections outside the page's request interception, so the whole browser
|
|
69
|
+
// talks through a proxy that does not exist, except for the CDNs (loopback included, which Chromium bypasses by default).
|
|
70
|
+
const UNREACHABLE_PROXY = "http://127.0.0.1:1";
|
|
71
|
+
export const strictNetworkLaunchArgs = () => [`--proxy-server=${UNREACHABLE_PROXY}`, `--proxy-bypass-list=${RENDER_CDN_HOSTS.join(";")};<-loopback>`];
|
|
72
|
+
// A popup the page writes into gets no evaluateOnNewDocument script, so it would still have WebRTC.
|
|
73
|
+
export const POPUP_BLOCK_SCRIPT = 'Object.defineProperty(globalThis, "open", { value: () => null, writable: false, configurable: false });';
|
|
74
|
+
// Request interception does not see WebSocket or EventSource connections; this does.
|
|
75
|
+
export const RENDER_CONTENT_SECURITY_POLICY = "connect-src 'self' data: blob:";
|
|
76
|
+
const CSP_META = `<meta http-equiv="Content-Security-Policy" content="${RENDER_CONTENT_SECURITY_POLICY}">`;
|
|
77
|
+
// The meta must come before anything a script could hide in, so it is never placed by searching for <head>
|
|
78
|
+
// (a comment or script text can contain one): right after a leading doctype, else first. Before a doctype it
|
|
79
|
+
// would switch the page to quirks mode; first otherwise, the parser still puts it in the head.
|
|
80
|
+
const LEADING_DOCTYPE = /^(\uFEFF?)(\s*<!doctype[^>]*>)?/i;
|
|
81
|
+
export const withRenderContentSecurityPolicy = (html) => {
|
|
82
|
+
const [prefix = "", bom = "", doctype] = LEADING_DOCTYPE.exec(html) ?? [];
|
|
83
|
+
const insertAt = doctype ? prefix.length : bom.length;
|
|
84
|
+
return [html.slice(0, insertAt), CSP_META, html.slice(insertAt)].join("");
|
|
85
|
+
};
|
package/package.json
CHANGED
|
@@ -55,24 +55,25 @@
|
|
|
55
55
|
"moviePrompt": "a woman takes a selfie with her phone"
|
|
56
56
|
},
|
|
57
57
|
{
|
|
58
|
-
"id": "
|
|
59
|
-
"text": "movie generated by
|
|
58
|
+
"id": "genai_omni",
|
|
59
|
+
"text": "movie generated by gemini omni",
|
|
60
60
|
"moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
|
|
61
61
|
"movieParams": {
|
|
62
|
-
"model": "
|
|
62
|
+
"model": "gemini-omni-1.1-flash"
|
|
63
63
|
}
|
|
64
64
|
},
|
|
65
65
|
{
|
|
66
|
-
"id": "
|
|
67
|
-
"text": "movie generated by
|
|
66
|
+
"id": "genai_omni_long",
|
|
67
|
+
"text": "a twelve-second movie generated by gemini omni, extended once",
|
|
68
|
+
"duration": 12,
|
|
68
69
|
"moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
|
|
69
70
|
"movieParams": {
|
|
70
|
-
"model": "
|
|
71
|
+
"model": "gemini-omni-1.1-flash"
|
|
71
72
|
}
|
|
72
73
|
},
|
|
73
74
|
{
|
|
74
|
-
"id": "
|
|
75
|
-
"text": "movie generated by
|
|
75
|
+
"id": "genai_omni_image",
|
|
76
|
+
"text": "movie generated by gemini omni from a first frame",
|
|
76
77
|
"duration": 5,
|
|
77
78
|
"imagePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
|
|
78
79
|
"imageParams": {
|
|
@@ -80,7 +81,7 @@
|
|
|
80
81
|
},
|
|
81
82
|
"moviePrompt": "a woman is walking through a busy Tokyo street at night, she is wearing dark sunglasses",
|
|
82
83
|
"movieParams": {
|
|
83
|
-
"model": "
|
|
84
|
+
"model": "gemini-omni-1.1-flash"
|
|
84
85
|
}
|
|
85
86
|
}
|
|
86
87
|
]
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
},
|
|
16
16
|
"movieParams": {
|
|
17
17
|
"provider": "google",
|
|
18
|
-
"model": "veo-3.1-generate-
|
|
18
|
+
"model": "veo-3.1-generate-001",
|
|
19
|
+
"vertexai_project": "YOUR_PROJECT_ID"
|
|
19
20
|
},
|
|
20
21
|
"lang": "en",
|
|
21
22
|
"beats": [
|
|
@@ -24,7 +25,7 @@
|
|
|
24
25
|
"text": "veo-3.1 with both ASSET and STYLE reference images",
|
|
25
26
|
"moviePrompt": "the woman runs through narrow alleyways, neon signs blurring in the rain",
|
|
26
27
|
"movieParams": {
|
|
27
|
-
"model": "veo-3.1-generate-
|
|
28
|
+
"model": "veo-3.1-generate-001",
|
|
28
29
|
"referenceImages": [
|
|
29
30
|
{ "imageName": "character", "referenceType": "ASSET" },
|
|
30
31
|
{ "imageName": "black_cat", "referenceType": "ASSET" }
|
|
@@ -5,13 +5,13 @@
|
|
|
5
5
|
"canvasSize": { "width": 1280, "height": 720 },
|
|
6
6
|
"beats": [
|
|
7
7
|
{
|
|
8
|
-
"id": "
|
|
9
|
-
"text": "Google GenAI
|
|
8
|
+
"id": "google_genai_omni",
|
|
9
|
+
"text": "Google GenAI gemini-omni-1.1-flash text-to-video",
|
|
10
10
|
"duration": 4,
|
|
11
11
|
"moviePrompt": "A calm ocean wave rolling onto a sandy beach at golden hour",
|
|
12
12
|
"movieParams": {
|
|
13
13
|
"provider": "google",
|
|
14
|
-
"model": "
|
|
14
|
+
"model": "gemini-omni-1.1-flash"
|
|
15
15
|
}
|
|
16
16
|
},
|
|
17
17
|
{
|