mulmocast 2.16.0 → 2.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +110 -38
- package/lib/actions/captions.js +1 -1
- package/lib/actions/image_agents.d.ts +1 -0
- package/lib/actions/image_agents.js +2 -2
- package/lib/actions/image_references.d.ts +2 -0
- package/lib/actions/image_references.js +3 -1
- package/lib/actions/images.d.ts +2 -0
- package/lib/actions/images.js +1 -0
- package/lib/actions/pdf.js +23 -2
- package/lib/agents/image_openai_agent.js +4 -3
- package/lib/agents/movie_genai_agent.d.ts +24 -0
- package/lib/agents/movie_genai_agent.js +56 -43
- package/lib/agents/tts_gemini_agent.d.ts +56 -0
- package/lib/agents/tts_gemini_agent.js +43 -38
- package/lib/cli/commands/bundle/builder.d.ts +2 -0
- package/lib/cli/commands/html/builder.d.ts +2 -0
- package/lib/cli/commands/markdown/builder.d.ts +2 -0
- package/lib/cli/commands/viewer/builder.d.ts +2 -0
- package/lib/cli/common.d.ts +2 -0
- package/lib/cli/common.js +5 -0
- package/lib/cli/helpers.js +3 -1
- package/lib/methods/mulmo_studio_context.d.ts +4 -0
- package/lib/methods/mulmo_studio_context.js +7 -0
- package/lib/types/provider2agent.d.ts +3 -4
- package/lib/types/provider2agent.js +38 -26
- package/lib/types/schema.d.ts +1 -1
- package/lib/types/type.d.ts +2 -0
- package/lib/utils/estimate_usage.js +12 -11
- package/lib/utils/gemini_omni_video.d.ts +49 -0
- package/lib/utils/gemini_omni_video.js +36 -0
- package/lib/utils/gemini_tts.d.ts +45 -0
- package/lib/utils/gemini_tts.js +35 -0
- package/lib/utils/html_render.d.ts +6 -5
- package/lib/utils/html_render.js +37 -30
- package/lib/utils/image_plugins/chart.js +2 -1
- package/lib/utils/image_plugins/html_tailwind.js +5 -4
- package/lib/utils/image_plugins/markdown.js +2 -1
- package/lib/utils/image_plugins/mermaid.js +2 -1
- package/lib/utils/image_plugins/slide.js +2 -1
- package/lib/utils/image_plugins/text_slide.js +2 -1
- package/lib/utils/image_plugins/vision.js +7 -0
- package/lib/utils/render_network_guard.d.ts +2 -0
- package/lib/utils/render_network_guard.js +36 -0
- package/lib/utils/render_network_policy.d.ts +15 -0
- package/lib/utils/render_network_policy.js +85 -0
- package/package.json +2 -2
- package/scripts/test/test_genai.json +10 -9
- package/scripts/test/test_genai_movie.json +1 -1
- package/scripts/test/test_image_references.json +3 -2
- package/scripts/test/test_movie_references.json +2 -1
- package/scripts/test/test_veo31_lite.json +3 -3
package/README.md
CHANGED
|
@@ -76,6 +76,7 @@ npm install -g mulmocast
|
|
|
76
76
|
```
|
|
77
77
|
|
|
78
78
|
You'll also need to install ffmpeg:
|
|
79
|
+
|
|
79
80
|
```bash
|
|
80
81
|
# For macOS with Homebrew
|
|
81
82
|
brew install ffmpeg
|
|
@@ -85,11 +86,13 @@ brew install ffmpeg
|
|
|
85
86
|
```
|
|
86
87
|
|
|
87
88
|
You can also use [`Dockerfile`](./Dockerfile) which helps you install the pre-requisits.
|
|
89
|
+
|
|
88
90
|
```
|
|
89
91
|
docker build -t mulmo-cli .
|
|
90
92
|
```
|
|
91
93
|
|
|
92
94
|
You can use the Docker image like this:
|
|
95
|
+
|
|
93
96
|
```
|
|
94
97
|
docker run -e OPENAI_API_KEY=<your_openai_api_key> -it mulmo-cli mulmo tool scripting -i -t children_book -o ./ -s story
|
|
95
98
|
```
|
|
@@ -99,16 +102,19 @@ docker run -e OPENAI_API_KEY=<your_openai_api_key> -it mulmo-cli mulmo tool scri
|
|
|
99
102
|
Create a `.env` file in your project directory with the following API keys:
|
|
100
103
|
|
|
101
104
|
### Required
|
|
105
|
+
|
|
102
106
|
```bash
|
|
103
107
|
OPENAI_API_KEY=your_openai_api_key
|
|
104
108
|
```
|
|
105
109
|
|
|
106
110
|
#### (Optional) For Google's image and TTS.
|
|
111
|
+
|
|
107
112
|
```bash
|
|
108
113
|
GEMINI_API_KEY=your_google_gemini_api_key
|
|
109
114
|
```
|
|
110
115
|
|
|
111
116
|
#### (Optional) For AI providers
|
|
117
|
+
|
|
112
118
|
```bash
|
|
113
119
|
# For Anthropic Claude (htmlPrompt feature)
|
|
114
120
|
ANTHROPIC_API_TOKEN=your_anthropic_api_token
|
|
@@ -117,22 +123,26 @@ ANTHROPIC_API_TOKEN=your_anthropic_api_token
|
|
|
117
123
|
For htmlPrompt configuration, see [docs/image.md](./docs/image.md#2-htmlprompt).
|
|
118
124
|
|
|
119
125
|
#### (Optional) For Movie models
|
|
126
|
+
|
|
120
127
|
```bash
|
|
121
128
|
REPLICATE_API_TOKEN=your_replicate_api_key
|
|
122
129
|
```
|
|
123
130
|
|
|
124
131
|
#### (Optional) For TTS models
|
|
132
|
+
|
|
125
133
|
```bash
|
|
126
134
|
# For ElevenLabs TTS
|
|
127
135
|
ELEVENLABS_API_KEY=your_elevenlabs_api_key
|
|
128
136
|
```
|
|
129
137
|
|
|
130
138
|
#### (Optional) to access web in mulmo tool
|
|
139
|
+
|
|
131
140
|
```bash
|
|
132
141
|
BROWSERLESS_API_TOKEN=your_browserless_api_token # to access web in mulmo tool
|
|
133
142
|
```
|
|
134
143
|
|
|
135
144
|
#### (Optional) Local usage tracking
|
|
145
|
+
|
|
136
146
|
```bash
|
|
137
147
|
MULMOCAST_DUMP_USAGE=1 # print JSON usage dump to stdout after each CLI action
|
|
138
148
|
MULMOCAST_DUMP_USAGE=/tmp/usage.json # ...or write it to a file
|
|
@@ -150,6 +160,7 @@ gcloud auth application-default login
|
|
|
150
160
|
```
|
|
151
161
|
|
|
152
162
|
Configure in MulmoScript:
|
|
163
|
+
|
|
153
164
|
```json
|
|
154
165
|
{
|
|
155
166
|
"imageParams": {
|
|
@@ -161,10 +172,10 @@ Configure in MulmoScript:
|
|
|
161
172
|
}
|
|
162
173
|
```
|
|
163
174
|
|
|
164
|
-
| Parameter
|
|
165
|
-
|
|
166
|
-
| `vertexai_project`
|
|
167
|
-
| `vertexai_location` | Region
|
|
175
|
+
| Parameter | Description | Default |
|
|
176
|
+
| ------------------- | ----------------------- | -------------------------------------- |
|
|
177
|
+
| `vertexai_project` | Google Cloud Project ID | None (enables Vertex AI mode when set) |
|
|
178
|
+
| `vertexai_location` | Region | `us-central1` |
|
|
168
179
|
|
|
169
180
|
For detailed setup instructions, see [Vertex AI Setup Guide](./docs/vertexai_en.md).
|
|
170
181
|
|
|
@@ -188,6 +199,7 @@ LLM_OPENAI_API_VERSION=2025-04-01-preview # optional
|
|
|
188
199
|
```
|
|
189
200
|
|
|
190
201
|
MulmoScript configuration (same as OpenAI):
|
|
202
|
+
|
|
191
203
|
```json
|
|
192
204
|
{
|
|
193
205
|
"imageParams": {
|
|
@@ -253,6 +265,7 @@ Priority chain: `config (defaults)` < `template/style` < `script` < `config.over
|
|
|
253
265
|
> **Note**: `kind: "path"` entries in `mulmo.config.json` are resolved relative to the **script file directory**, not the config file location. This is consistent with all other path resolution in MulmoScript.
|
|
254
266
|
|
|
255
267
|
Verify the merged result with:
|
|
268
|
+
|
|
256
269
|
```bash
|
|
257
270
|
mulmo tool info merged --script <script.json>
|
|
258
271
|
```
|
|
@@ -261,36 +274,44 @@ mulmo tool info merged --script <script.json>
|
|
|
261
274
|
|
|
262
275
|
1. Create a MulmoScript JSON file with `mulmo tool scripting`
|
|
263
276
|
2. Generate audio with `mulmo audio`
|
|
264
|
-
3. Generate images with `mulmo images`
|
|
277
|
+
3. Generate images with `mulmo images`
|
|
265
278
|
4. Create final video with `mulmo movie`
|
|
266
279
|
|
|
267
280
|
## Quick Start
|
|
281
|
+
|
|
268
282
|
### Step 1: Generate a MulmoScript with interactive mode
|
|
283
|
+
|
|
269
284
|
- Step 1-1: Run the script generation command
|
|
285
|
+
|
|
270
286
|
```bash
|
|
271
287
|
mulmo tool scripting -i -t children_book -o ./ -s story
|
|
272
288
|
```
|
|
289
|
+
|
|
273
290
|
This will initiate the script creation process.
|
|
274
|
-
|
|
291
|
+
|
|
275
292
|
- Step 1-2: Interactive conversation with AI
|
|
276
293
|
|
|
277
294
|
After running the command, you'll engage in an interactive conversation with the AI to create your story script. The AI will guide you through the process and help shape your content based on your inputs. Once completed, a JSON file (like `story-1746600802426.json`) will be generated.
|
|
278
295
|
|
|
279
296
|
### Step 2: Generate both audio and images, then combine into video
|
|
297
|
+
|
|
280
298
|
```bash
|
|
281
299
|
mulmo movie {generated_script_file}
|
|
282
300
|
```
|
|
301
|
+
|
|
283
302
|
Replace `{generated_script_file}` with the output file from STEP 1, such as `story-1746600802426.json`.
|
|
284
303
|
|
|
285
|
-
Optionally, you can specify
|
|
304
|
+
Optionally, you can specify _\__clipboard_ as the script file name to paste the script from the clipboard, which is generated by other interactive environment such as ChatGPT or Claude.
|
|
286
305
|
|
|
287
306
|
## Quick Start for Ghibli-Style Animation
|
|
288
307
|
|
|
289
308
|
[](https://www.youtube.com/watch?v=gz_Oe5p2n3s)
|
|
290
|
-
|
|
309
|
+
_Click the image above to watch an example of what you can create_
|
|
291
310
|
|
|
292
311
|
### Step 1: Check/Update Environment Setup
|
|
312
|
+
|
|
293
313
|
Verify your `.env` file contains:
|
|
314
|
+
|
|
294
315
|
```bash
|
|
295
316
|
OPENAI_API_KEY=your_openai_api_key
|
|
296
317
|
```
|
|
@@ -298,23 +319,29 @@ OPENAI_API_KEY=your_openai_api_key
|
|
|
298
319
|
> **Note:** Ensure your OpenAI organization is verified to access the GPT Image models. Visit https://platform.openai.com/settings/organization/general and complete the "Verifications" section.
|
|
299
320
|
|
|
300
321
|
### Step 2: Generate a Ghibli-style MulmoScript
|
|
322
|
+
|
|
301
323
|
```bash
|
|
302
324
|
mulmo tool scripting -i -t ghibli_comic -o ./ -s story
|
|
303
325
|
```
|
|
326
|
+
|
|
304
327
|
This will initiate an interactive conversation with the AI to create your Ghibli-inspired story. Once completed, a JSON file (e.g., `story-1747834931950.json`) will be generated.
|
|
305
328
|
|
|
306
329
|
### Step 3: Translate the MulmoScript
|
|
330
|
+
|
|
307
331
|
```bash
|
|
308
332
|
mulmo translate {generated_script_file_from_step_2}
|
|
309
333
|
```
|
|
310
334
|
|
|
311
335
|
### Step 4: Generate video with Japanese subtitles
|
|
336
|
+
|
|
312
337
|
```bash
|
|
313
338
|
mulmo movie {generated_script_file_from_step_2} -c ja
|
|
314
339
|
```
|
|
340
|
+
|
|
315
341
|
The `-c ja` flag adds Japanese subtitles to your video.
|
|
316
342
|
|
|
317
343
|
When the process completes, the CLI will display the path to your generated video file:
|
|
344
|
+
|
|
318
345
|
```bash
|
|
319
346
|
Video created successfully! 14.939 sec
|
|
320
347
|
writing: /Users/username/path/to/output/story-1747834931950__ja.mp4
|
|
@@ -344,10 +371,7 @@ You can create a minimal JSON with just beats and complete it with `mulmo tool c
|
|
|
344
371
|
|
|
345
372
|
```json
|
|
346
373
|
{
|
|
347
|
-
"beats": [
|
|
348
|
-
{ "text": "Hello, welcome to MulmoCast!" },
|
|
349
|
-
{ "text": "This is a simple example." }
|
|
350
|
-
]
|
|
374
|
+
"beats": [{ "text": "Hello, welcome to MulmoCast!" }, { "text": "This is a simple example." }]
|
|
351
375
|
}
|
|
352
376
|
```
|
|
353
377
|
|
|
@@ -366,7 +390,6 @@ mulmo tool complete my_beats.json -s ani -o my_script.json
|
|
|
366
390
|
|
|
367
391
|
This generates a complete MulmoScript with all required fields (canvasSize, speechParams, imageParams, etc.) from the style.
|
|
368
392
|
|
|
369
|
-
|
|
370
393
|
## Generate content from MulmoScript
|
|
371
394
|
|
|
372
395
|
Mulmo provides several commands to handle different aspects of content creation:
|
|
@@ -403,11 +426,11 @@ Payload shape:
|
|
|
403
426
|
{
|
|
404
427
|
"records": 3,
|
|
405
428
|
"byModel": [
|
|
406
|
-
{ "provider": "openai",
|
|
407
|
-
{ "provider": "gemini",
|
|
429
|
+
{ "provider": "openai", "model": "gpt-4o-mini-tts", "records": 1, "inputChars": 20 },
|
|
430
|
+
{ "provider": "gemini", "model": "gemini-3.8-flash-tts", "records": 1, "inputTokens": 9, "outputTokens": 59, "totalTokens": 68 },
|
|
408
431
|
{ "provider": "elevenlabs", "model": "eleven_multilingual_v2", "records": 1, "inputChars": 18 }
|
|
409
432
|
],
|
|
410
|
-
"snapshot": [
|
|
433
|
+
"snapshot": [/* per-call UsageRecord[] */]
|
|
411
434
|
}
|
|
412
435
|
```
|
|
413
436
|
|
|
@@ -437,7 +460,18 @@ const estimates = estimateUsage(script, { targetLangs: ["ja"] });
|
|
|
437
460
|
|
|
438
461
|
Each metric carries a `precision` flag: `"exact"` for values that are deterministic from the script (TTS character counts, tokenized OpenAI prompts, fixed image-output token tables) and `"estimated"` for heuristics (LLM output length, speech duration derived from text). `costUSD` appears when pricing data exists in `modelPricing` (`src/types/provider2agent.ts`), where every price records the date it was last verified. See [docs/api.md § Pre-run estimation](./docs/api.md#pre-run-estimation-estimateusage).
|
|
439
462
|
|
|
463
|
+
### Strict network mode (scripts you did not write)
|
|
464
|
+
|
|
465
|
+
HTML-based beats (`html_tailwind`, `markdown`, `textSlide`, `chart`, `mermaid`, `slide`, captions, HTML prompts) and PDF output are rendered in a headless browser that normally has full network access. For a script from an untrusted source, add `--strict-network` (or set `MULMO_STRICT_NETWORK=1`; library callers set `context.strictNetwork`):
|
|
466
|
+
|
|
467
|
+
```bash
|
|
468
|
+
mulmo images untrusted.json --strict-network
|
|
469
|
+
```
|
|
470
|
+
|
|
471
|
+
The renderer may then load only files in the script's folder and mulmocast's output folders, `data:` / `blob:` URLs, and the CDNs mulmocast's own templates use (`cdn.tailwindcss.com`, `fonts.googleapis.com`, `fonts.gstatic.com`, and on `cdn.jsdelivr.net` only the mermaid and Chart.js files the templates load). Anything else — other local files, remote images in the script, `fetch`, WebSockets, WebRTC, popups, preconnect hints — is blocked, and each blocked request is logged (`strict network: blocked …`). Files next to the script in the same folder remain readable, so keep an untrusted script in a folder of its own (a script loaded by URL gets the output folders only). Requests to the allowed CDNs still reach those providers, so treat what a page sends there as visible to them. Strict mode guards what the browser loads; files the script names with `kind: "path"` (images, movies, logos, mermaid or remotion code) are read by mulmocast itself and are not restricted yet ([#1622](https://github.com/receptron/mulmocast-cli/issues/1622)). Remotion beats are always guarded this way. The `vision` beat renders in its own browser inside `mulmocast-vision`, which the guard cannot reach, so in strict mode a `vision` beat stops with an error instead of rendering unguarded.
|
|
472
|
+
|
|
440
473
|
## Cache and Re-run
|
|
474
|
+
|
|
441
475
|
When running the same `mulmo` command multiple times, previously generated files are treated as cache. For example, audio or image files will not be regenerated if they already exist.
|
|
442
476
|
|
|
443
477
|
To force regeneration, delete the old files — including temporary files — under the output directory before re-running the command or use -f (force) option.
|
|
@@ -464,7 +498,24 @@ Set a theme once with `slideParams.theme`, then use `"type": "slide"` in each be
|
|
|
464
498
|
{
|
|
465
499
|
"$mulmocast": { "version": "1.1" },
|
|
466
500
|
"slideParams": {
|
|
467
|
-
"theme": {
|
|
501
|
+
"theme": {
|
|
502
|
+
"colors": {
|
|
503
|
+
"bg": "0F172A",
|
|
504
|
+
"bgCard": "1E293B",
|
|
505
|
+
"bgCardAlt": "334155",
|
|
506
|
+
"text": "F8FAFC",
|
|
507
|
+
"textMuted": "CBD5E1",
|
|
508
|
+
"textDim": "64748B",
|
|
509
|
+
"primary": "3B82F6",
|
|
510
|
+
"accent": "8B5CF6",
|
|
511
|
+
"success": "22C55E",
|
|
512
|
+
"warning": "F59E0B",
|
|
513
|
+
"danger": "EF4444",
|
|
514
|
+
"info": "14B8A6",
|
|
515
|
+
"highlight": "EC4899"
|
|
516
|
+
},
|
|
517
|
+
"fonts": { "title": "Georgia", "body": "Calibri", "mono": "Consolas" }
|
|
518
|
+
}
|
|
468
519
|
},
|
|
469
520
|
"beats": [
|
|
470
521
|
{
|
|
@@ -492,18 +543,18 @@ For detailed layout specifications and content block reference, see the [Slide D
|
|
|
492
543
|
|
|
493
544
|
MulmoCast includes 100 pre-designed CSS styles for markdown slides, organized in 10 categories:
|
|
494
545
|
|
|
495
|
-
| Category
|
|
496
|
-
|
|
497
|
-
| business
|
|
498
|
-
| tech
|
|
499
|
-
| creative
|
|
500
|
-
| minimalist | Clean and simple designs
|
|
501
|
-
| nature
|
|
502
|
-
| dark
|
|
503
|
-
| colorful
|
|
504
|
-
| vintage
|
|
505
|
-
| japanese
|
|
506
|
-
| geometric
|
|
546
|
+
| Category | Description | Example Styles |
|
|
547
|
+
| ---------- | ------------------------------- | --------------------------------------------- |
|
|
548
|
+
| business | Professional corporate designs | corporate-blue, executive-gray, finance-green |
|
|
549
|
+
| tech | Technology and developer themes | cyber-neon, matrix-green, terminal-dark |
|
|
550
|
+
| creative | Artistic and expressive styles | artistic-splash, watercolor-soft, graffiti |
|
|
551
|
+
| minimalist | Clean and simple designs | clean-white, zen-beige, nordic-light |
|
|
552
|
+
| nature | Nature-inspired themes | forest-green, ocean-blue, sunset-orange |
|
|
553
|
+
| dark | Dark mode and elegant themes | charcoal-elegant, midnight-blue, obsidian |
|
|
554
|
+
| colorful | Vibrant and bold colors | vibrant-pink, candy-colors, aurora |
|
|
555
|
+
| vintage | Retro and classic styles | retro-70s, art-deco, typewriter |
|
|
556
|
+
| japanese | Japanese aesthetic designs | washi-paper, sakura-pink, zen-garden |
|
|
557
|
+
| geometric | Pattern-based designs | hexagon-pattern, isometric, bauhaus |
|
|
507
558
|
|
|
508
559
|
### Usage
|
|
509
560
|
|
|
@@ -543,12 +594,12 @@ https://github.com/receptron/mulmocast-cli/tree/main/scripts
|
|
|
543
594
|
|
|
544
595
|
## Directory Structure
|
|
545
596
|
|
|
546
|
-
| Directory
|
|
547
|
-
|
|
548
|
-
| `output/`
|
|
549
|
-
| `output/audio/`
|
|
550
|
-
| `output/images/`
|
|
551
|
-
| `output/cache/`
|
|
597
|
+
| Directory | Description |
|
|
598
|
+
| ---------------- | ----------------------------------------------------------------------- |
|
|
599
|
+
| `output/` | Artifacts generated by commands — e.g., `.json`, audio, and video files |
|
|
600
|
+
| `output/audio/` | Temporary audio fragments used in the final audio assembly |
|
|
601
|
+
| `output/images/` | Image files generated by the `images` command |
|
|
602
|
+
| `output/cache/` | Internal cache files for various processing steps |
|
|
552
603
|
|
|
553
604
|
> These directories are automatically created as needed by the `mulmo` commands.
|
|
554
605
|
|
|
@@ -597,6 +648,9 @@ Options:
|
|
|
597
648
|
-g, --grouped Output all files under output/<basename>/ directory
|
|
598
649
|
[boolean] [default: false]
|
|
599
650
|
--backup create backup media file [boolean] [default: false]
|
|
651
|
+
--strict-network Let HTML renderers reach only local files and the
|
|
652
|
+
CDNs mulmocast's templates use (also
|
|
653
|
+
MULMO_STRICT_NETWORK=1) [boolean] [default: false]
|
|
600
654
|
-p, --presentationStyle Presentation Style [string]
|
|
601
655
|
```
|
|
602
656
|
|
|
@@ -621,6 +675,9 @@ Options:
|
|
|
621
675
|
-g, --grouped Output all files under output/<basename>/ directory
|
|
622
676
|
[boolean] [default: false]
|
|
623
677
|
--backup create backup media file [boolean] [default: false]
|
|
678
|
+
--strict-network Let HTML renderers reach only local files and the
|
|
679
|
+
CDNs mulmocast's templates use (also
|
|
680
|
+
MULMO_STRICT_NETWORK=1) [boolean] [default: false]
|
|
624
681
|
-p, --presentationStyle Presentation Style [string]
|
|
625
682
|
-a, --audiodir Audio output directory [string]
|
|
626
683
|
```
|
|
@@ -646,6 +703,9 @@ Options:
|
|
|
646
703
|
-g, --grouped Output all files under output/<basename>/ directory
|
|
647
704
|
[boolean] [default: false]
|
|
648
705
|
--backup create backup media file [boolean] [default: false]
|
|
706
|
+
--strict-network Let HTML renderers reach only local files and the
|
|
707
|
+
CDNs mulmocast's templates use (also
|
|
708
|
+
MULMO_STRICT_NETWORK=1) [boolean] [default: false]
|
|
649
709
|
-p, --presentationStyle Presentation Style [string]
|
|
650
710
|
-i, --imagedir Image output directory [string]
|
|
651
711
|
```
|
|
@@ -671,6 +731,9 @@ Options:
|
|
|
671
731
|
-g, --grouped Output all files under output/<basename>/ directory
|
|
672
732
|
[boolean] [default: false]
|
|
673
733
|
--backup create backup media file [boolean] [default: false]
|
|
734
|
+
--strict-network Let HTML renderers reach only local files and the
|
|
735
|
+
CDNs mulmocast's templates use (also
|
|
736
|
+
MULMO_STRICT_NETWORK=1) [boolean] [default: false]
|
|
674
737
|
-p, --presentationStyle Presentation Style [string]
|
|
675
738
|
-a, --audiodir Audio output directory [string]
|
|
676
739
|
-i, --imagedir Image output directory [string]
|
|
@@ -700,6 +763,9 @@ Options:
|
|
|
700
763
|
-g, --grouped Output all files under output/<basename>/ directory
|
|
701
764
|
[boolean] [default: false]
|
|
702
765
|
--backup create backup media file [boolean] [default: false]
|
|
766
|
+
--strict-network Let HTML renderers reach only local files and the
|
|
767
|
+
CDNs mulmocast's templates use (also
|
|
768
|
+
MULMO_STRICT_NETWORK=1) [boolean] [default: false]
|
|
703
769
|
-p, --presentationStyle Presentation Style [string]
|
|
704
770
|
-i, --imagedir Image output directory [string]
|
|
705
771
|
--pdf_mode PDF mode
|
|
@@ -729,6 +795,9 @@ Options:
|
|
|
729
795
|
-g, --grouped Output all files under output/<basename>/ directory
|
|
730
796
|
[boolean] [default: false]
|
|
731
797
|
--backup create backup media file [boolean] [default: false]
|
|
798
|
+
--strict-network Let HTML renderers reach only local files and the
|
|
799
|
+
CDNs mulmocast's templates use (also
|
|
800
|
+
MULMO_STRICT_NETWORK=1) [boolean] [default: false]
|
|
732
801
|
-p, --presentationStyle Presentation Style [string]
|
|
733
802
|
--image_width Image width (e.g., 400px, 50%, auto) [string]
|
|
734
803
|
```
|
|
@@ -754,6 +823,9 @@ Options:
|
|
|
754
823
|
-g, --grouped Output all files under output/<basename>/ directory
|
|
755
824
|
[boolean] [default: false]
|
|
756
825
|
--backup create backup media file [boolean] [default: false]
|
|
826
|
+
--strict-network Let HTML renderers reach only local files and the
|
|
827
|
+
CDNs mulmocast's templates use (also
|
|
828
|
+
MULMO_STRICT_NETWORK=1) [boolean] [default: false]
|
|
757
829
|
-p, --presentationStyle Presentation Style [string]
|
|
758
830
|
--image_width Image width (e.g., 400px, 50%, auto) [string]
|
|
759
831
|
```
|
|
@@ -779,6 +851,9 @@ Options:
|
|
|
779
851
|
-g, --grouped Output all files under output/<basename>/ directory
|
|
780
852
|
[boolean] [default: false]
|
|
781
853
|
--backup create backup media file [boolean] [default: false]
|
|
854
|
+
--strict-network Let HTML renderers reach only local files and the
|
|
855
|
+
CDNs mulmocast's templates use (also
|
|
856
|
+
MULMO_STRICT_NETWORK=1) [boolean] [default: false]
|
|
782
857
|
-p, --presentationStyle Presentation Style [string]
|
|
783
858
|
```
|
|
784
859
|
|
|
@@ -962,9 +1037,6 @@ Examples:
|
|
|
962
1037
|
mulmo tool info styles --format json
|
|
963
1038
|
```
|
|
964
1039
|
|
|
965
|
-
|
|
966
|
-
|
|
967
1040
|
## Contributing
|
|
968
1041
|
|
|
969
1042
|
For developers interested in contributing to this project, please see [CONTRIBUTING.md](./CONTRIBUTING.md).
|
|
970
|
-
|
package/lib/actions/captions.js
CHANGED
|
@@ -99,7 +99,7 @@ const generateBeatCaptions = async (beat, context, index) => {
|
|
|
99
99
|
styles: (mergedCaptionParams.styles ?? []).join(";\n"),
|
|
100
100
|
bottomOffset: `${mergedCaptionParams.bottomOffset ?? 0}`,
|
|
101
101
|
});
|
|
102
|
-
await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, true);
|
|
102
|
+
await renderHTMLToImage(htmlData, imagePath, canvasSize.width, canvasSize.height, false, true, MulmoStudioContextMethods.getRenderNetworkOptions(context));
|
|
103
103
|
return {
|
|
104
104
|
file: imagePath,
|
|
105
105
|
startAt: beatStartAt + introPadding + beatDuration * cumulativeRatios[subIndex],
|
|
@@ -292,6 +292,6 @@ export const imagePluginAgent = async (namedInputs) => {
|
|
|
292
292
|
}
|
|
293
293
|
};
|
|
294
294
|
export const htmlImageGeneratorAgent = async (namedInputs) => {
|
|
295
|
-
const { file, canvasSize, htmlText } = namedInputs;
|
|
296
|
-
await renderHTMLToImage(htmlText, file, canvasSize.width, canvasSize.height);
|
|
295
|
+
const { file, canvasSize, htmlText, context } = namedInputs;
|
|
296
|
+
await renderHTMLToImage(htmlText, file, canvasSize.width, canvasSize.height, false, false, MulmoStudioContextMethods.getRenderNetworkOptions(context));
|
|
297
297
|
};
|
|
@@ -6,10 +6,12 @@ type ImageAgentInfo = ReturnType<typeof MulmoPresentationStyleMethods.getImageAg
|
|
|
6
6
|
type MovieAgentInfo = ReturnType<typeof MulmoPresentationStyleMethods.getMovieAgentInfo>;
|
|
7
7
|
export declare const referenceImageGraphParams: (imageAgentInfo: ImageAgentInfo, canvasSize: MulmoCanvasDimension) => {
|
|
8
8
|
model: string | undefined;
|
|
9
|
+
moderation: string | undefined;
|
|
9
10
|
canvasSize: {
|
|
10
11
|
width: number;
|
|
11
12
|
height: number;
|
|
12
13
|
};
|
|
14
|
+
quality: string | undefined;
|
|
13
15
|
vertexai_project: string | undefined;
|
|
14
16
|
vertexai_location: string | undefined;
|
|
15
17
|
};
|
|
@@ -6,10 +6,12 @@ import { MulmoPresentationStyleMethods, MulmoMediaSourceMethods, MulmoImagePromp
|
|
|
6
6
|
import { ImageMediaType, } from "../types/index.js";
|
|
7
7
|
import { imageOpenaiAgent, mediaMockAgent, imageGenAIAgent, imageReplicateAgent, movieGenAIAgent, movieReplicateAgent } from "../agents/index.js";
|
|
8
8
|
import { agentGenerationError, imageReferenceAction, imageFileTarget, movieFileTarget } from "../utils/error_cause.js";
|
|
9
|
-
// Reference media are generated outside the beat graphs, so they must carry the same
|
|
9
|
+
// Reference media are generated outside the beat graphs, so they must carry the same params the beats do.
|
|
10
10
|
export const referenceImageGraphParams = (imageAgentInfo, canvasSize) => ({
|
|
11
11
|
model: imageAgentInfo.imageParams.model,
|
|
12
|
+
moderation: imageAgentInfo.imageParams.moderation,
|
|
12
13
|
canvasSize,
|
|
14
|
+
quality: imageAgentInfo.imageParams.quality,
|
|
13
15
|
vertexai_project: imageAgentInfo.imageParams.vertexai_project,
|
|
14
16
|
vertexai_location: imageAgentInfo.imageParams.vertexai_location,
|
|
15
17
|
});
|
package/lib/actions/images.d.ts
CHANGED
|
@@ -275,11 +275,13 @@ export declare const beat_graph_data: {
|
|
|
275
275
|
file: string;
|
|
276
276
|
canvasSize: import("../types/type.js").MulmoCanvasDimension;
|
|
277
277
|
htmlText: string;
|
|
278
|
+
context: MulmoStudioContext;
|
|
278
279
|
}) => Promise<void>;
|
|
279
280
|
inputs: {
|
|
280
281
|
htmlText: string;
|
|
281
282
|
canvasSize: string;
|
|
282
283
|
file: string;
|
|
284
|
+
context: string;
|
|
283
285
|
};
|
|
284
286
|
};
|
|
285
287
|
imageGenerator: {
|
package/lib/actions/images.js
CHANGED
package/lib/actions/pdf.js
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
|
+
import os from "os";
|
|
4
|
+
import crypto from "crypto";
|
|
3
5
|
import puppeteer from "puppeteer";
|
|
4
6
|
import { GraphAILogger, sleep } from "graphai";
|
|
5
7
|
import { MulmoPresentationStyleMethods } from "../methods/index.js";
|
|
6
8
|
import { localizedText, isHttp } from "../utils/utils.js";
|
|
7
9
|
import { getOutputPdfFilePath, writingMessage, getHTMLFile, mulmoCreditPath } from "../utils/file.js";
|
|
8
10
|
import { interpolate } from "../utils/html_render.js";
|
|
11
|
+
import { guardRenderPage } from "../utils/render_network_guard.js";
|
|
12
|
+
import { strictNetworkLaunchArgs, withRenderContentSecurityPolicy } from "../utils/render_network_policy.js";
|
|
9
13
|
import { safeFetch, FETCH_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
10
14
|
import { MulmoStudioContextMethods } from "../methods/mulmo_studio_context.js";
|
|
11
15
|
const isCI = process.env.CI === "true";
|
|
@@ -138,17 +142,34 @@ export const pdfFilePath = (context, pdfMode) => {
|
|
|
138
142
|
return getOutputPdfFilePath(fileDirs.outDirPath, studio.filename, pdfMode, lang);
|
|
139
143
|
};
|
|
140
144
|
const PDF_CONTENT_TIMEOUT_MS = 60000;
|
|
145
|
+
// Strict mode navigates to a file because the guard's WebRTC removal never reaches a setContent document.
|
|
146
|
+
const loadPdfPage = async (page, html, { strictNetwork, allowedFileRoots }) => {
|
|
147
|
+
if (!strictNetwork) {
|
|
148
|
+
await page.setContent(html, { waitUntil: "domcontentloaded", timeout: PDF_CONTENT_TIMEOUT_MS });
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
const tmpFile = path.join(os.tmpdir(), `mulmocast_pdf_${crypto.randomUUID()}.html`);
|
|
152
|
+
await guardRenderPage(page, [tmpFile, ...(allowedFileRoots ?? [])]);
|
|
153
|
+
fs.writeFileSync(tmpFile, withRenderContentSecurityPolicy(html));
|
|
154
|
+
try {
|
|
155
|
+
await page.goto(`file://${tmpFile}`, { waitUntil: "domcontentloaded", timeout: PDF_CONTENT_TIMEOUT_MS });
|
|
156
|
+
}
|
|
157
|
+
finally {
|
|
158
|
+
fs.rmSync(tmpFile, { force: true });
|
|
159
|
+
}
|
|
160
|
+
};
|
|
141
161
|
const generatePDF = async (context, pdfMode, pdfSize) => {
|
|
142
162
|
const outputPdfPath = pdfFilePath(context, pdfMode);
|
|
143
163
|
const html = await generatePDFHTML(context, pdfMode, pdfSize);
|
|
144
164
|
const canvasSize = MulmoPresentationStyleMethods.getCanvasSize(context.presentationStyle);
|
|
145
165
|
const pdfOptions = createPDFOptions(pdfSize, pdfMode, canvasSize);
|
|
166
|
+
const network = MulmoStudioContextMethods.getRenderNetworkOptions(context);
|
|
146
167
|
const browser = await puppeteer.launch({
|
|
147
|
-
args: isCI ? ["--no-sandbox"] : [],
|
|
168
|
+
args: [...(isCI ? ["--no-sandbox"] : []), ...(network.strictNetwork ? strictNetworkLaunchArgs() : [])],
|
|
148
169
|
});
|
|
149
170
|
try {
|
|
150
171
|
const page = await browser.newPage();
|
|
151
|
-
await page
|
|
172
|
+
await loadPdfPage(page, html, network);
|
|
152
173
|
await sleep(1000);
|
|
153
174
|
await page.pdf({
|
|
154
175
|
path: outputPdfPath,
|
|
@@ -4,7 +4,7 @@ import { GraphAILogger } from "graphai";
|
|
|
4
4
|
import { toFile, AuthenticationError, RateLimitError, APIError } from "openai";
|
|
5
5
|
import { createOpenAIClient } from "../utils/openai_client.js";
|
|
6
6
|
import { safeFetch, FETCH_DOWNLOAD_TIMEOUT_MS } from "../utils/fetch.js";
|
|
7
|
-
import { provider2ImageAgent, gptImages, deprecatedOpenAIImageModelHints } from "../types/provider2agent.js";
|
|
7
|
+
import { provider2ImageAgent, gptImages, gptImageQuality, deprecatedOpenAIImageModelHints } from "../types/provider2agent.js";
|
|
8
8
|
import { apiKeyMissingError, agentGenerationError, openAIAgentGenerationError, agentIncorrectAPIKeyError, agentAPIRateLimitError, agentInvalidResponseError, imageAction, imageFileTarget, unsupportedModelTarget, } from "../utils/error_cause.js";
|
|
9
9
|
const isDeprecatedOpenAIImageModel = (model) => model in deprecatedOpenAIImageModelHints;
|
|
10
10
|
export const buildDeprecatedModelMessage = (model) => {
|
|
@@ -63,8 +63,9 @@ export const imageOpenaiAgent = async ({ namedInputs, params, config, }) => {
|
|
|
63
63
|
if (gptImages.includes(model)) {
|
|
64
64
|
imageOptions.moderation = moderation || "auto";
|
|
65
65
|
imageOptions.background = "opaque";
|
|
66
|
-
|
|
67
|
-
|
|
66
|
+
const resolvedQuality = gptImageQuality(model, quality);
|
|
67
|
+
if (resolvedQuality) {
|
|
68
|
+
imageOptions.quality = resolvedQuality;
|
|
68
69
|
}
|
|
69
70
|
}
|
|
70
71
|
const response = await (async () => {
|
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
import type { AgentFunction, AgentFunctionInfo } from "graphai";
|
|
2
2
|
import type { AgentBufferResult, GenAIImageAgentConfig, GoogleMovieAgentParams, MovieAgentInputs } from "../types/agent.js";
|
|
3
|
+
import { type GeminiOmniImage, type GeminiOmniVideoRequest } from "../utils/gemini_omni_video.js";
|
|
4
|
+
type GeminiOmniInteraction = {
|
|
5
|
+
id?: string;
|
|
6
|
+
output_video?: {
|
|
7
|
+
data?: string;
|
|
8
|
+
mime_type?: string;
|
|
9
|
+
};
|
|
10
|
+
};
|
|
11
|
+
export type GeminiOmniVideoClient = {
|
|
12
|
+
interactions: {
|
|
13
|
+
create: (params: GeminiOmniVideoRequest, options?: {
|
|
14
|
+
timeout?: number;
|
|
15
|
+
}) => Promise<GeminiOmniInteraction>;
|
|
16
|
+
};
|
|
17
|
+
};
|
|
18
|
+
export type GeminiOmniVideoInput = {
|
|
19
|
+
model: string;
|
|
20
|
+
prompt: string;
|
|
21
|
+
aspectRatio: string;
|
|
22
|
+
requestedSec: number;
|
|
23
|
+
firstFrame?: GeminiOmniImage;
|
|
24
|
+
lastFrame?: GeminiOmniImage;
|
|
25
|
+
};
|
|
26
|
+
export declare const generateGeminiOmniVideo: (ai: GeminiOmniVideoClient, input: GeminiOmniVideoInput) => Promise<Buffer>;
|
|
3
27
|
export declare const movieGenAIAgent: AgentFunction<GoogleMovieAgentParams, AgentBufferResult, MovieAgentInputs, GenAIImageAgentConfig>;
|
|
4
28
|
declare const movieGenAIAgentInfo: AgentFunctionInfo;
|
|
5
29
|
export default movieGenAIAgentInfo;
|