@amg555/frameforge-mcp 2.3.0 → 2.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +38 -10
- package/CHANGELOG.md +35 -0
- package/CLAUDE.md +112 -12
- package/README.md +20 -2
- package/START_HERE.md +89 -33
- package/install.sh +1 -1
- package/package.json +5 -5
- package/src/cli/doctor.js +31 -1
- package/src/cli/setup-keys.js +27 -6
- package/src/mcp-server/server.js +117 -6
- package/src/pipeline/external-adapter.js +55 -15
- package/src/pipeline/providers.js +46 -0
|
@@ -1,17 +1,45 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "frameforge",
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
},
|
|
3
|
+
"displayName": "FrameForge — AI Video Production",
|
|
4
|
+
"description": "Full AI video production engine. Produce short-form reels (1080×1920), YouTube videos (1920×1080), or code-driven animated explainers — directly from your IDE. Playwright + FFmpeg renders every frame. 17 MCP tools.",
|
|
5
|
+
"version": "2.3.1",
|
|
6
|
+
"homepage": "https://github.com/AMG555/FrameForge",
|
|
8
7
|
"plugins": [
|
|
9
8
|
{
|
|
10
9
|
"name": "video-pipeline",
|
|
11
|
-
"
|
|
12
|
-
"
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
"path": "plugins/video-pipeline/.claude-plugin/plugin.json",
|
|
11
|
+
"skills": [
|
|
12
|
+
{
|
|
13
|
+
"name": "video-pipeline",
|
|
14
|
+
"displayName": "FrameForge Video Pipeline",
|
|
15
|
+
"description": "End-to-end video production: research → script → TTS → images → render → publish. Three paths: (1) IDE writes scenes (most common, no API key), (2) code-driven Playwright animation, (3) auto-research pipeline. 17 MCP tools.",
|
|
16
|
+
"triggerPhrases": [
|
|
17
|
+
"make a video about",
|
|
18
|
+
"produce a reel on",
|
|
19
|
+
"create a short",
|
|
20
|
+
"edit this into a reel",
|
|
21
|
+
"make an explainer",
|
|
22
|
+
"animated video",
|
|
23
|
+
"code-driven video",
|
|
24
|
+
"generate video",
|
|
25
|
+
"run the pipeline",
|
|
26
|
+
"publish video",
|
|
27
|
+
"overnight batch",
|
|
28
|
+
"queue videos"
|
|
29
|
+
]
|
|
30
|
+
}
|
|
31
|
+
]
|
|
15
32
|
}
|
|
16
|
-
]
|
|
33
|
+
],
|
|
34
|
+
"mcpServer": {
|
|
35
|
+
"command": "npx",
|
|
36
|
+
"args": ["--yes", "--package=@amg555/frameforge-mcp", "frameforge-mcp"],
|
|
37
|
+
"description": "Runs the FrameForge MCP server via npx (no install needed). Requires ffmpeg and edge-tts on PATH."
|
|
38
|
+
},
|
|
39
|
+
"requirements": {
|
|
40
|
+
"node": ">=18.0.0",
|
|
41
|
+
"binaries": ["ffmpeg", "ffprobe"],
|
|
42
|
+
"optional": ["edge-tts", "python3"],
|
|
43
|
+
"notes": "edge-tts: pip install edge-tts. Playwright (for code-driven animation): npm install playwright && npx playwright install chromium (run once in project dir)."
|
|
44
|
+
}
|
|
17
45
|
}
|
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,41 @@ All notable changes to FrameForge are documented here.
|
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
+
## [2.3.2] — 2026-11-06
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- **Tool 18: `export_to_editor`** — send a finished animation MP4 + EDL to any local editing app via an HTTP bridge: Premiere Pro (`:7777`), DaVinci Resolve (`:8080`), CapCut (`:9090`), After Effects (`:7878`), or Remotion (`:3001`). Pass `editorUrl` to override any port. Bridge setup docs: `docs/editor-bridges/`.
|
|
12
|
+
- **VoxForge TTS provider** (`PROVIDER_TTS=voxforge`) — named first-class provider in `TTSProvider`. Tries `/v1/audio/speech` (OpenAI-compatible) then falls back to `/v1/tts`. Configured via `VOXFORGE_URL`, `VOXFORGE_KEY`, `VOXFORGE_VOICE`, `VOXFORGE_MODEL`. Also surfaced in `multi_voice_tts` platform enum and `setup-keys.js` wizard.
|
|
13
|
+
- **Editing app presets in `ExternalAdapter`** — `premiere`, `resolve`, `capcut`, `aftereffects` (local bridge), `kling` (cloud), plus updated `voxforge` preset pointing to `/v1/audio/speech`.
|
|
14
|
+
- **`render_animation` mode `"kit"`** — now exposed in the MCP tool enum. Generates an EDL file in `out/` for import into Premiere / Resolve.
|
|
15
|
+
|
|
16
|
+
### Changed
|
|
17
|
+
|
|
18
|
+
- **`CLAUDE.md`**: code-driven animation promoted to **primary path**; FFmpeg + AI images demoted to "quick draft"; new sections for VoxForge voice and editor bridge workflow; tool table reorganised into Primary / Secondary / Auto-research groups (18 tools).
|
|
19
|
+
- **`START_HERE.md`**: opening question set updated to surface animated explainer as a first-class format choice.
|
|
20
|
+
- **`setup-keys.js`**: VoxForge added to TTS provider menu (second option, right after edge-tts). Four new config keys persisted: `VOXFORGE_URL`, `VOXFORGE_KEY`, `VOXFORGE_VOICE`, `VOXFORGE_MODEL`.
|
|
21
|
+
- **`scripts/deep-validate.js`**: +3 checks (`export_to_editor`, `voxforge`, tool count label updated to 18).
|
|
22
|
+
- Version bump: `2.3.1` → `2.3.2` across all version strings.
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## [2.3.1] — 2026-11-04
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
|
|
30
|
+
- **`.claude-plugin/marketplace.json`** — Claude Code plugin marketplace registration. FrameForge is now discoverable via `claude plugin marketplace add AMG555/FrameForge`.
|
|
31
|
+
- **`plugins/video-pipeline/.claude-plugin/plugin.json`** — per-plugin manifest for the video-pipeline skill.
|
|
32
|
+
|
|
33
|
+
### Changed
|
|
34
|
+
|
|
35
|
+
- **`src/cli/setup-keys.js`** — `skip` is now the **default** option in the LLM provider menu. Users who write their own scenes (the most common path) are no longer forced through an API key section.
|
|
36
|
+
- **`CLAUDE.md`** — Path 2 expanded: transcript/`cuePhrase()` word-sync workflow, `video_job.py` spend-capped batch, golden stills regression test, hook variants (A/B testing opening shots), mobile crop check, still-review budget.
|
|
37
|
+
- **`START_HERE.md`** — restructured into three named paths (A: IDE-scenes, B: code-driven animation, C: auto-research). Reel vs. YouTube format choice surfaced in opening questions. Path B includes transcription, hook variants, and storyboard gate.
|
|
38
|
+
- Version bump: `2.3.0` → `2.3.1` across all version strings.
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
7
42
|
## [2.3.0] — 2026-11-02
|
|
8
43
|
|
|
9
44
|
### Added
|
package/CLAUDE.md
CHANGED
|
@@ -58,7 +58,9 @@ If you're not sure what a tool does — call it with the obvious parameters. The
|
|
|
58
58
|
|
|
59
59
|
## THREE PATHS — PICK ONE
|
|
60
60
|
|
|
61
|
-
|
|
61
|
+
> **The primary path is Path 2 (code-driven animation).** Every frame is a pure function of time `t`, rendered by Playwright + FFmpeg. This is what produces the high-quality, viral-style motion graphics clips. Path 1 (FFmpeg + AI images) is the fast fallback when Playwright is unavailable or the user wants a quick draft. Path 3 (auto-research) is for unattended batch.
|
|
62
|
+
|
|
63
|
+
### Path 1 — You write the scenes + AI images (quick draft)
|
|
62
64
|
|
|
63
65
|
Your IDE model writes the scenes. FrameForge generates TTS + AI images + renders with FFmpeg.
|
|
64
66
|
|
|
@@ -94,10 +96,14 @@ Output files appear in `.frameforge-files/`:
|
|
|
94
96
|
|
|
95
97
|
### Path 2 — Code-driven animated video (Playwright + FFmpeg, no AI images)
|
|
96
98
|
|
|
97
|
-
Use this when: the user wants motion graphics, Pollinations is unavailable, or they ask for "animated"
|
|
99
|
+
Use this when: the user wants motion graphics, Pollinations is unavailable, or they ask for "animated", "code-rendered", "explainer", "reel", or "YouTube-style" video. Every frame is a pure function of time `t`.
|
|
98
100
|
|
|
99
101
|
> **Iron rule:** Never use `Math.random()`, `Date.now()`, `setTimeout`, or module-level mutable state in scene code. Use `hash(i)` from `reel/engine.js` for deterministic randomness.
|
|
100
102
|
|
|
103
|
+
**Pick a format first:**
|
|
104
|
+
- **Reel / Short / TikTok** → `width: 1080, height: 1920` (portrait 9:16)
|
|
105
|
+
- **YouTube / Explainer** → `width: 1920, height: 1080` (landscape 16:9)
|
|
106
|
+
|
|
101
107
|
**Step 0 — Beat sheet first. Always. No exceptions.**
|
|
102
108
|
|
|
103
109
|
Before writing any scene code, reply to the user with:
|
|
@@ -106,17 +112,22 @@ Before writing any scene code, reply to the user with:
|
|
|
106
112
|
|
|
107
113
|
**Stop and wait for approval before Step 1.** Fixing a beat sheet costs one message. Rewriting a finished scene costs everything.
|
|
108
114
|
|
|
109
|
-
**Step 1 — Generate TTS
|
|
115
|
+
**Step 1 — Generate TTS and transcription (exact timing is ground truth):**
|
|
110
116
|
```
|
|
111
117
|
multi_voice_tts({ text: "Full narration text.", outputPath: ".frameforge-files/voice_0.m4a" })
|
|
112
118
|
```
|
|
119
|
+
Then transcribe for word-level timestamps (required for `cuePhrase()` sync):
|
|
120
|
+
```
|
|
121
|
+
// run in projectDir after scaffold: python scripts/transcribe.py .frameforge-files/voice_0.m4a
|
|
122
|
+
// → produces voice_0.captions.json with per-word start/end times
|
|
123
|
+
```
|
|
113
124
|
|
|
114
125
|
**Step 2 — Scaffold the project:**
|
|
115
126
|
```
|
|
116
127
|
scaffold_animation_project({
|
|
117
128
|
outDir: ".frameforge-files/my-animation/",
|
|
118
129
|
title: "My Video",
|
|
119
|
-
width: 1920, height: 1080,
|
|
130
|
+
width: 1920, height: 1080, // or 1080×1920 for portrait reel
|
|
120
131
|
scenes: [{ narration: "...", caption: "...", visual_description: "..." }],
|
|
121
132
|
audioPath: ".frameforge-files/voice_0.m4a"
|
|
122
133
|
})
|
|
@@ -138,16 +149,49 @@ validate_storyboard({ storyboardPath: ".frameforge-files/my-animation/STORYBOARD
|
|
|
138
149
|
- Read `ANIMATION_GUIDE.md` in the project folder first
|
|
139
150
|
- Every style must be a **pure function of `t`**
|
|
140
151
|
- Use `layer('name').on(t => ({ ...styles }))` for each scene
|
|
152
|
+
- Sync animation to narration using `cuePhrase('the exact words spoken')` — this reads `voice_0.captions.json` and returns the frame when those words start. **The build fails if the phrase isn't found**, so re-recording VO can never silently desync the animation.
|
|
141
153
|
|
|
142
154
|
**Step 5 — Pre-flight, then stills, then full render:**
|
|
143
155
|
```
|
|
144
156
|
render_animation({ projectDir: ".frameforge-files/my-animation/", mode: "look" }) // fonts, assets, NaN checks
|
|
145
157
|
render_animation({ projectDir: ".frameforge-files/my-animation/", mode: "sheet" }) // contact sheet — inspect this
|
|
146
158
|
render_animation({ projectDir: ".frameforge-files/my-animation/", mode: "strip", timestamps: [2.1, 2.6] }) // motion check
|
|
159
|
+
render_animation({ projectDir: ".frameforge-files/my-animation/", mode: "crop", timestamps: ["9:16", 3.0] }) // mobile check
|
|
147
160
|
render_animation({ projectDir: ".frameforge-files/my-animation/", mode: "full",
|
|
148
161
|
audioPath: ".frameforge-files/voice_0.m4a" }) // only after stills pass
|
|
149
162
|
```
|
|
150
163
|
|
|
164
|
+
**Still review budget (minimum, before full render):**
|
|
165
|
+
- At least one `sheet` per shot
|
|
166
|
+
- One `strip` per key motion or transition
|
|
167
|
+
- One `crop` per story-carrying face or detail
|
|
168
|
+
|
|
169
|
+
**Step 6 — Optional: hook variants**
|
|
170
|
+
|
|
171
|
+
For reels and Shorts, make 2–3 versions with different opening hooks:
|
|
172
|
+
- Change only the first shot (0s–3s) in `edit.js`
|
|
173
|
+
- `render_animation({ mode: "stills", timestamps: [0.0, 1.5, 3.0] })` to compare openings
|
|
174
|
+
- Full render each variant, deliver as `hook_a.mp4`, `hook_b.mp4`, etc.
|
|
175
|
+
|
|
176
|
+
**Step 7 — Optional: unattended / spend-capped batch**
|
|
177
|
+
```
|
|
178
|
+
// run from projectDir
|
|
179
|
+
python scripts/video_job.py submit --spec spec.json --max-usd 2.00
|
|
180
|
+
// → { jobId } — detached process, survives your session closing
|
|
181
|
+
python scripts/video_job.py status --id <jobId>
|
|
182
|
+
python scripts/video_job.py result --id <jobId>
|
|
183
|
+
```
|
|
184
|
+
Each step only starts if the running cost won't exceed `--max-usd`. Receipts go to `jobs/receipts.jsonl`.
|
|
185
|
+
|
|
186
|
+
**Step 8 — Golden stills regression test (before any refactor)**
|
|
187
|
+
|
|
188
|
+
After the video passes QC, save the test stills:
|
|
189
|
+
```
|
|
190
|
+
render_animation({ projectDir, mode: "storyboard" }) // 4 keyframes, file to out/storyboard.jpg
|
|
191
|
+
// → copy out/storyboard.jpg to test/golden/storyboard.jpg
|
|
192
|
+
```
|
|
193
|
+
Before future refactors, re-run and compare. If pixels shift, the refactor broke determinism.
|
|
194
|
+
|
|
151
195
|
> Requires one-time setup in the project directory: `npm install playwright && npx playwright install chromium`
|
|
152
196
|
|
|
153
197
|
---
|
|
@@ -165,9 +209,20 @@ get_job_status({ jobId }) // poll every 30s until status = "SUCCEEDED"
|
|
|
165
209
|
|
|
166
210
|
---
|
|
167
211
|
|
|
168
|
-
## ALL
|
|
212
|
+
## ALL 18 MCP TOOLS
|
|
169
213
|
|
|
170
|
-
**
|
|
214
|
+
**Primary — code-driven animation (no LLM key, no image gen API):**
|
|
215
|
+
|
|
216
|
+
| Tool | What it does |
|
|
217
|
+
|------|-------------|
|
|
218
|
+
| `scaffold_animation_project` | Scaffold a Playwright + FFmpeg animation project (engine, render scripts, STORYBOARD.md) |
|
|
219
|
+
| `validate_storyboard` | Enforce the 4 hard gates before any scene code |
|
|
220
|
+
| `render_animation` | Playwright renderer: `sheet` / `strip` / `storyboard` / `crop` / `full` / `mux` / `kit` / `look` |
|
|
221
|
+
| `multi_voice_tts` | TTS per-call: `edge-tts` · **`voxforge`** · `elevenlabs` · `azure` · `openai` · `webhook` |
|
|
222
|
+
| `export_to_editor` | Send finished animation to Premiere / Resolve / CapCut / After Effects / Remotion |
|
|
223
|
+
| `external_platform_call` | HTTP bridge to VoxForge, Runway, Kling, Pika, or any platform |
|
|
224
|
+
|
|
225
|
+
**Secondary — FFmpeg + AI images (fast draft, no Playwright needed):**
|
|
171
226
|
|
|
172
227
|
| Tool | What it does |
|
|
173
228
|
|------|-------------|
|
|
@@ -177,15 +232,10 @@ get_job_status({ jobId }) // poll every 30s until status = "SUCCEEDED"
|
|
|
177
232
|
| `edit_scenes` | Patch captions, swap media files, append scenes |
|
|
178
233
|
| `assemble_video` | FFmpeg render + quality gate → master MP4 |
|
|
179
234
|
| `render_tts` | Generate voice audio for one scene |
|
|
180
|
-
| `multi_voice_tts` | Override TTS provider per call without touching `.env` |
|
|
181
235
|
| `generate_scene_image` | Generate one scene image |
|
|
182
236
|
| `publish_video` | Deliver via file copy or HTTP webhook |
|
|
183
|
-
| `external_platform_call` | HTTP bridge to ElevenLabs, Runway, Replicate, etc. |
|
|
184
|
-
| `scaffold_animation_project` | Create a code-driven animation project |
|
|
185
|
-
| `validate_storyboard` | Check STORYBOARD.md against the 4-gate contract |
|
|
186
|
-
| `render_animation` | Playwright renderer: sheet / strip / storyboard / crop / full / look |
|
|
187
237
|
|
|
188
|
-
**
|
|
238
|
+
**Auto-research (needs a free LLM key):**
|
|
189
239
|
|
|
190
240
|
| Tool | What it does |
|
|
191
241
|
|------|-------------|
|
|
@@ -196,6 +246,56 @@ get_job_status({ jobId }) // poll every 30s until status = "SUCCEEDED"
|
|
|
196
246
|
|
|
197
247
|
---
|
|
198
248
|
|
|
249
|
+
## VOICE — VoxForge custom audio
|
|
250
|
+
|
|
251
|
+
`multi_voice_tts` with `platform: "voxforge"` routes to your VoxForge instance:
|
|
252
|
+
|
|
253
|
+
```
|
|
254
|
+
multi_voice_tts({
|
|
255
|
+
text: "Your narration.",
|
|
256
|
+
outputPath: ".frameforge-files/voice_0.m4a",
|
|
257
|
+
platform: "voxforge",
|
|
258
|
+
voice: "my-cloned-voice-id", // speaker ID from VoxForge
|
|
259
|
+
apiKey: "vf-..." // or set VOXFORGE_KEY in .env
|
|
260
|
+
})
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Set in `.env` for all calls:
|
|
264
|
+
```
|
|
265
|
+
PROVIDER_TTS=voxforge
|
|
266
|
+
VOXFORGE_URL=https://api.voxforge.io # or your self-hosted URL
|
|
267
|
+
VOXFORGE_KEY=vf-...
|
|
268
|
+
VOXFORGE_VOICE=my-cloned-voice-id
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
---
|
|
272
|
+
|
|
273
|
+
## EDITOR BRIDGES — send animation to editing apps
|
|
274
|
+
|
|
275
|
+
After `render_animation full`, send the finished MP4 into any editing app:
|
|
276
|
+
|
|
277
|
+
```
|
|
278
|
+
// Export EDL timeline first (for apps that support it)
|
|
279
|
+
render_animation({ projectDir: "...", mode: "kit" })
|
|
280
|
+
|
|
281
|
+
// Send to DaVinci Resolve (default bridge port :8080)
|
|
282
|
+
export_to_editor({ projectDir: "...", editor: "resolve" })
|
|
283
|
+
|
|
284
|
+
// Send to Premiere Pro (bridge port :7777)
|
|
285
|
+
export_to_editor({ projectDir: "...", editor: "premiere" })
|
|
286
|
+
|
|
287
|
+
// Send to CapCut (bridge port :9090)
|
|
288
|
+
export_to_editor({ projectDir: "...", editor: "capcut" })
|
|
289
|
+
|
|
290
|
+
// Custom endpoint
|
|
291
|
+
export_to_editor({ projectDir: "...", editor: "custom", editorUrl: "http://localhost:4000" })
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Bridge servers run locally alongside the app. Setup docs: `docs/editor-bridges/`.
|
|
295
|
+
Override ports: `RESOLVE_BRIDGE_URL`, `PREMIERE_BRIDGE_URL`, `CAPCUT_BRIDGE_URL`, `AE_BRIDGE_URL`.
|
|
296
|
+
|
|
297
|
+
---
|
|
298
|
+
|
|
199
299
|
## SCENE FIELDS
|
|
200
300
|
|
|
201
301
|
| Field | Required | Notes |
|
package/README.md
CHANGED
|
@@ -6,9 +6,10 @@
|
|
|
6
6
|
|
|
7
7
|
**Any IDE. Any topic. Any style. 1080p videos from a single prompt.**
|
|
8
8
|
|
|
9
|
+
[](https://www.npmjs.com/package/@amg555/frameforge-mcp)
|
|
9
10
|
[](LICENSE)
|
|
10
11
|
[](https://nodejs.org/)
|
|
11
|
-
[](CHANGELOG.md)
|
|
12
13
|
[](https://modelcontextprotocol.io/)
|
|
13
14
|
[](#testing)
|
|
14
15
|
|
|
@@ -272,6 +273,16 @@ Every video passes 7 checks before delivery. Any failure blocks the output.
|
|
|
272
273
|
|
|
273
274
|
## CLI commands
|
|
274
275
|
|
|
276
|
+
**Via npx (no clone needed):**
|
|
277
|
+
|
|
278
|
+
| Command | What it does |
|
|
279
|
+
|---------|-------------|
|
|
280
|
+
| `npx --package=@amg555/frameforge-mcp frameforge-setup` | Interactive key wizard — saves to `~/.frameforge/config.json` |
|
|
281
|
+
| `npx --package=@amg555/frameforge-mcp frameforge-doctor` | Check Node, FFmpeg, edge-tts, and env vars |
|
|
282
|
+
| `npx --package=@amg555/frameforge-mcp frameforge --topic "..."` | Full pipeline: research → render → deliver |
|
|
283
|
+
|
|
284
|
+
**Via cloned repo:**
|
|
285
|
+
|
|
275
286
|
| Command | What it does |
|
|
276
287
|
|---------|-------------|
|
|
277
288
|
| `npm run doctor` | Check Node, FFmpeg, edge-tts, and env vars |
|
|
@@ -323,7 +334,14 @@ Report vulnerabilities: [SECURITY.md](SECURITY.md)
|
|
|
323
334
|
|
|
324
335
|
## Deployment
|
|
325
336
|
|
|
326
|
-
**
|
|
337
|
+
**npx (no install — recommended for users):**
|
|
338
|
+
|
|
339
|
+
```bash
|
|
340
|
+
npx --package=@amg555/frameforge-mcp frameforge-setup
|
|
341
|
+
# then add the IDE config from the quickstart above
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
**VPS / Pi / self-hosted (cloned repo):**
|
|
327
345
|
|
|
328
346
|
```bash
|
|
329
347
|
git clone https://github.com/AMG555/FrameForge.git && cd FrameForge
|
package/START_HERE.md
CHANGED
|
@@ -17,18 +17,20 @@ produce it — from research through render to final MP4.
|
|
|
17
17
|
|
|
18
18
|
## Opening Message (say this verbatim to the user)
|
|
19
19
|
|
|
20
|
-
> 👋 I'm your FrameForge video studio. I can make
|
|
21
|
-
> documentary clips, tutorials, or any video you describe — using AI research, voiceover,
|
|
22
|
-
> animated scenes, and a full render pipeline.
|
|
20
|
+
> 👋 I'm your FrameForge video studio. I can make:
|
|
23
21
|
>
|
|
24
|
-
>
|
|
22
|
+
> - **Short-form reels** (Instagram, TikTok, YouTube Shorts — 15–60s, 9:16 portrait)
|
|
23
|
+
> - **YouTube videos** (explainers, tutorials, long-form — 3–15 min, 16:9 landscape)
|
|
24
|
+
> - **Code-driven animated explainers** (motion graphics, no stock images needed — any length)
|
|
25
|
+
>
|
|
26
|
+
> Let's start with a few quick questions:
|
|
25
27
|
>
|
|
26
28
|
> 1. **What's the topic or idea?** (e.g. "How black holes form", "5 tips for sourdough bread", "A travel reel about Tokyo")
|
|
27
|
-
> 2. **
|
|
28
|
-
> 3. **
|
|
29
|
-
> 4. **
|
|
29
|
+
> 2. **Format?** `Reel / Short / TikTok (9:16)` · `YouTube video (16:9)` · `Animated explainer (any)`
|
|
30
|
+
> 3. **Style?** `educational` · `comedic` · `documentary` · `cinematic` · `tutorial` · or describe your own
|
|
31
|
+
> 4. **How long?** `Short (15–60s)` · `Medium (1–3 min)` · `Long (5–10 min)`
|
|
30
32
|
> 5. **Voice?** `Auto-select` · or name a style: `professional narrator`, `friendly teacher`, `energetic host`
|
|
31
|
-
> 6. **Any images or
|
|
33
|
+
> 6. **Any footage, images, or audio to include?** (paste file paths, or say "generate everything")
|
|
32
34
|
>
|
|
33
35
|
> Answer as many or as few as you like. I'll fill in the rest.
|
|
34
36
|
|
|
@@ -36,16 +38,24 @@ produce it — from research through render to final MP4.
|
|
|
36
38
|
|
|
37
39
|
## After the User Answers — Agent Instructions
|
|
38
40
|
|
|
39
|
-
Once you have the user's answers,
|
|
41
|
+
Once you have the user's answers, choose the right render path and follow it precisely.
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Render Paths
|
|
40
46
|
|
|
41
|
-
###
|
|
47
|
+
### PATH A — IDE-written scenes (fastest, most common)
|
|
42
48
|
|
|
43
|
-
|
|
44
|
-
|
|
49
|
+
Use for: any video where you (the IDE agent) write the narration and visual descriptions.
|
|
50
|
+
No LLM key needed — FrameForge generates TTS + AI images and renders with FFmpeg.
|
|
51
|
+
|
|
52
|
+
**Step A1 — Beat Sheet Gate (MANDATORY)**
|
|
53
|
+
|
|
54
|
+
Before calling any render tool, produce a beat sheet and **stop** — wait for approval:
|
|
45
55
|
|
|
46
56
|
```
|
|
47
57
|
BEAT SHEET — "<title>"
|
|
48
|
-
Duration: Xs | Style: <style> | Format: <
|
|
58
|
+
Duration: Xs | Style: <style> | Format: <landscape/portrait>
|
|
49
59
|
|
|
50
60
|
Scene 1 [0:00–0:08] <what happens + what the viewer learns> → TRANSITION
|
|
51
61
|
Scene 2 [0:08–0:18] <what happens + what the viewer learns> → TRANSITION
|
|
@@ -56,43 +66,89 @@ Voice: <voice description>
|
|
|
56
66
|
Visual style: <1-sentence description>
|
|
57
67
|
```
|
|
58
68
|
|
|
59
|
-
Gate rules —
|
|
69
|
+
Gate rules — revise if ANY are true:
|
|
60
70
|
- A scene has no event (action that changes something)
|
|
61
71
|
- Two scenes share the same timestamp range
|
|
62
72
|
- Any seam is missing a transition label
|
|
63
|
-
-
|
|
73
|
+
- Total duration doesn't match requested length ±10%
|
|
64
74
|
|
|
65
|
-
|
|
75
|
+
**Step A2 — Render**
|
|
66
76
|
|
|
67
|
-
|
|
77
|
+
```
|
|
78
|
+
create_video_from_script({
|
|
79
|
+
scenes: [ { narration: "...", caption: "...", visual_description: "...", camera_hint: "zoom_in" } ],
|
|
80
|
+
title: "My Video",
|
|
81
|
+
orientation: "landscape", // "portrait" for Reels/Shorts/TikTok
|
|
82
|
+
voice: "en-US-BrianNeural", // or use language: "en"
|
|
83
|
+
withMusic: true
|
|
84
|
+
})
|
|
85
|
+
```
|
|
68
86
|
|
|
69
|
-
|
|
87
|
+
**Step A3 — Publish**
|
|
70
88
|
|
|
71
|
-
|
|
89
|
+
Ask: "Save locally, or deliver somewhere?" Then call `publish_video`.
|
|
72
90
|
|
|
73
|
-
|
|
74
|
-
`voice` from the user's answers. The script must match the beat sheet scene-for-scene.
|
|
91
|
+
---
|
|
75
92
|
|
|
76
|
-
###
|
|
93
|
+
### PATH B — Code-driven animated video (motion graphics, explainers, reels)
|
|
77
94
|
|
|
78
|
-
|
|
79
|
-
|
|
95
|
+
Use for: motion graphics, animated explainers, reels that must look high-quality without stock images.
|
|
96
|
+
Requires one-time: `npm install playwright && npx playwright install chromium` (run in the project dir).
|
|
80
97
|
|
|
81
|
-
|
|
98
|
+
**Step B1 — Beat sheet first (same gate as Path A — MANDATORY)**
|
|
82
99
|
|
|
83
|
-
|
|
100
|
+
**Step B2 — TTS and transcription**
|
|
84
101
|
|
|
85
|
-
|
|
86
|
-
|
|
102
|
+
```
|
|
103
|
+
multi_voice_tts({ text: "Full narration.", outputPath: ".frameforge-files/voice_0.m4a" })
|
|
104
|
+
// After scaffolding: python scripts/transcribe.py .frameforge-files/voice_0.m4a
|
|
105
|
+
// → voice_0.captions.json — word timestamps for cuePhrase() sync
|
|
106
|
+
```
|
|
87
107
|
|
|
88
|
-
|
|
89
|
-
preview. Show the user the sheet path and ask if they want any changes.
|
|
108
|
+
**Step B3 — Scaffold**
|
|
90
109
|
|
|
91
|
-
|
|
110
|
+
```
|
|
111
|
+
scaffold_animation_project({
|
|
112
|
+
outDir: ".frameforge-files/my-animation/",
|
|
113
|
+
title: "My Video",
|
|
114
|
+
width: 1920, height: 1080, // 1080×1920 for portrait reels
|
|
115
|
+
scenes: [...],
|
|
116
|
+
audioPath: ".frameforge-files/voice_0.m4a"
|
|
117
|
+
})
|
|
118
|
+
```
|
|
92
119
|
|
|
93
|
-
|
|
120
|
+
**Step B4 — Validate STORYBOARD.md**
|
|
94
121
|
|
|
95
|
-
|
|
122
|
+
```
|
|
123
|
+
validate_storyboard({ storyboardPath: ".frameforge-files/my-animation/STORYBOARD.md" })
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
**Step B5 — Write `edit.js`, then stills, then full render**
|
|
127
|
+
|
|
128
|
+
```
|
|
129
|
+
render_animation({ projectDir: "...", mode: "look" }) // pre-flight
|
|
130
|
+
render_animation({ projectDir: "...", mode: "sheet" }) // one sheet per shot
|
|
131
|
+
render_animation({ projectDir: "...", mode: "strip", timestamps: [2.1, 2.6] }) // motion check
|
|
132
|
+
render_animation({ projectDir: "...", mode: "full", audioPath: ".frameforge-files/voice_0.m4a" })
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
**For reels: make 2–3 hook variants** (change only the first 3s of `edit.js`, full-render each):
|
|
136
|
+
```
|
|
137
|
+
render_animation({ projectDir: "...", mode: "stills", timestamps: [0.0, 1.5, 3.0] })
|
|
138
|
+
// deliver hook_a.mp4, hook_b.mp4 so the user can A/B test
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
---
|
|
142
|
+
|
|
143
|
+
### PATH C — Auto-research (needs `PROVIDER_LLM` + key in `.env`)
|
|
144
|
+
|
|
145
|
+
Use for: overnight batch, topics where you want FrameForge to research and script everything.
|
|
146
|
+
|
|
147
|
+
```
|
|
148
|
+
queue_full_pipeline({ topic: "How GPS works", style: "educational" })
|
|
149
|
+
→ { jobId }
|
|
150
|
+
get_job_status({ jobId }) // poll every 30s
|
|
151
|
+
```
|
|
96
152
|
|
|
97
153
|
---
|
|
98
154
|
|
package/install.sh
CHANGED
package/package.json
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@amg555/frameforge-mcp",
|
|
3
|
-
"version": "2.3.
|
|
3
|
+
"version": "2.3.2",
|
|
4
4
|
"description": "FrameForge — AI video production engine for agentic IDEs. npx @amg555/frameforge-mcp. FFmpeg + TTS run on your machine. 17 MCP tools.",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
8
|
"main": "src/mcp-server/server.js",
|
|
9
9
|
"bin": {
|
|
10
|
-
"frameforge":
|
|
11
|
-
"frameforge-mcp":
|
|
12
|
-
"frameforge-setup":
|
|
10
|
+
"frameforge": "src/cli/make.js",
|
|
11
|
+
"frameforge-mcp": "src/mcp-server/stdio.js",
|
|
12
|
+
"frameforge-setup": "src/cli/setup-keys.js",
|
|
13
13
|
"frameforge-doctor": "src/cli/doctor.js"
|
|
14
14
|
},
|
|
15
15
|
"scripts": {
|
|
@@ -55,6 +55,6 @@
|
|
|
55
55
|
"license": "MIT",
|
|
56
56
|
"repository": {
|
|
57
57
|
"type": "git",
|
|
58
|
-
"url": "https://github.com/AMG555/FrameForge"
|
|
58
|
+
"url": "git+https://github.com/AMG555/FrameForge.git"
|
|
59
59
|
}
|
|
60
60
|
}
|
package/src/cli/doctor.js
CHANGED
|
@@ -89,7 +89,37 @@ if (edgeTts.status === 0) {
|
|
|
89
89
|
);
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
-
// 5.
|
|
92
|
+
// 5. Playwright / Chromium (required for code-driven animation — Path 2)
|
|
93
|
+
const playwright = spawnSync('node', ['-e', "require('playwright')"], {
|
|
94
|
+
encoding: 'utf8', timeout: 5000,
|
|
95
|
+
// Look in the project's own node_modules first (how it ships in the npm package)
|
|
96
|
+
cwd: path.resolve(__dirname, '../..'),
|
|
97
|
+
});
|
|
98
|
+
const playwrightGlobal = spawnSync('node', ['-e', "require('playwright')"], {
|
|
99
|
+
encoding: 'utf8', timeout: 5000,
|
|
100
|
+
});
|
|
101
|
+
const hasPlaywright = playwright.status === 0 || playwrightGlobal.status === 0;
|
|
102
|
+
|
|
103
|
+
// Check for a locally installed Chromium browser
|
|
104
|
+
const chromiumPaths = [
|
|
105
|
+
path.join(require('os').homedir(), '.cache', 'ms-playwright'), // Linux / macOS default
|
|
106
|
+
path.join(process.env.LOCALAPPDATA || '', 'ms-playwright'), // Windows
|
|
107
|
+
path.join(require('os').homedir(), 'Library', 'Caches', 'ms-playwright'), // macOS alt
|
|
108
|
+
];
|
|
109
|
+
const hasChromium = chromiumPaths.some(p => { try { return require('fs').readdirSync(p).length > 0; } catch { return false; } });
|
|
110
|
+
|
|
111
|
+
if (hasPlaywright && hasChromium) {
|
|
112
|
+
pass('Playwright', 'playwright package + Chromium browser installed (code-driven animation ready)');
|
|
113
|
+
} else if (hasPlaywright && !hasChromium) {
|
|
114
|
+
warn('Playwright', 'playwright package found but Chromium browser not installed',
|
|
115
|
+
'Run in your animation project dir: npx playwright install chromium');
|
|
116
|
+
} else {
|
|
117
|
+
warn('Playwright', 'Not installed — required for code-driven animation (Path 2)',
|
|
118
|
+
'Run in your animation project dir: npm install playwright && npx playwright install chromium\n' +
|
|
119
|
+
' (Path 1 via create_video_from_script does not need Playwright)');
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// 6. System Font (for video subtitles/captions)
|
|
93
123
|
const FONT_CANDIDATES = [
|
|
94
124
|
'C:\\Windows\\Fonts\\arialbd.ttf',
|
|
95
125
|
'C:\\Windows\\Fonts\\calibrib.ttf',
|
package/src/cli/setup-keys.js
CHANGED
|
@@ -171,6 +171,15 @@ async function main() {
|
|
|
171
171
|
This wizard writes your provider API keys to: ${path.resolve(outFile)}
|
|
172
172
|
Keys are never sent anywhere — they stay on your machine.
|
|
173
173
|
Press Enter to accept a default. Ctrl+C to cancel.
|
|
174
|
+
|
|
175
|
+
✅ No keys are required to make videos.
|
|
176
|
+
Your IDE writes the scenes — FrameForge uses free providers by default:
|
|
177
|
+
• edge-tts — free voice (no account)
|
|
178
|
+
• pollinations — free images (no account)
|
|
179
|
+
• FFmpeg — free local render (already on your machine)
|
|
180
|
+
|
|
181
|
+
You only need keys for the auto-research pipeline (FrameForge writes the
|
|
182
|
+
script itself). Skip those sections if you plan to write scenes yourself.
|
|
174
183
|
`);
|
|
175
184
|
|
|
176
185
|
const cfg = {};
|
|
@@ -178,22 +187,27 @@ Press Enter to accept a default. Ctrl+C to cancel.
|
|
|
178
187
|
// ── LLM ──────────────────────────────────────────────────────────────────
|
|
179
188
|
section('1 / 5 — LLM Provider (research + script generation)');
|
|
180
189
|
console.log(' Used for: researching topics, writing scripts, fact-checking.');
|
|
190
|
+
console.log(' ⚡ SKIP THIS if your IDE agent writes the scenes (most common path).');
|
|
181
191
|
console.log(' Free option: Groq (no credit card needed for free tier).\n');
|
|
182
192
|
|
|
183
193
|
cfg.PROVIDER_LLM = await menu('Choose LLM provider', [
|
|
194
|
+
['skip', '⏭ Skip — I write my own scenes (most common, no key needed)'],
|
|
184
195
|
['groq', 'Groq — free, fast (llama-3.3-70b). Get key: console.groq.com'],
|
|
185
196
|
['cerebras', 'Cerebras — fastest inference (2000+ tok/s). Get key: cloud.cerebras.ai'],
|
|
197
|
+
['gemini', 'Google Gemini 2.0 Flash — free tier. Get key: aistudio.google.com'],
|
|
186
198
|
['openrouter', 'OpenRouter — all models with one key. Get key: openrouter.ai'],
|
|
187
199
|
['anthropic', 'Anthropic Claude Sonnet 4.5. Get key: console.anthropic.com'],
|
|
188
|
-
['gemini', 'Google Gemini 2.0 Flash. Get key: aistudio.google.com'],
|
|
189
200
|
['mistral', 'Mistral Large. Get key: console.mistral.ai'],
|
|
190
201
|
['together', 'Together AI Llama-3.3-70B-Turbo. Get key: api.together.ai'],
|
|
191
202
|
['openai', 'OpenAI GPT-4o. Get key: platform.openai.com'],
|
|
192
203
|
['ollama', 'Ollama — fully local, no key needed. Install: ollama.com'],
|
|
193
204
|
['webhook', 'Custom/VoxForge/any OpenAI-compatible endpoint'],
|
|
194
|
-
], '
|
|
205
|
+
], 'skip');
|
|
195
206
|
|
|
196
|
-
if (cfg.PROVIDER_LLM === '
|
|
207
|
+
if (cfg.PROVIDER_LLM === 'skip') {
|
|
208
|
+
console.log(' ✅ LLM skipped — your IDE will write scenes via create_video_from_script.\n');
|
|
209
|
+
cfg.PROVIDER_LLM = undefined; // don't write to config
|
|
210
|
+
} else if (cfg.PROVIDER_LLM === 'groq') {
|
|
197
211
|
cfg.GROQ_API_KEY = await askSecret(' Groq API key (starts with gsk_)');
|
|
198
212
|
} else if (cfg.PROVIDER_LLM === 'cerebras') {
|
|
199
213
|
cfg.CEREBRAS_API_KEY = await askSecret(' Cerebras API key');
|
|
@@ -226,13 +240,20 @@ Press Enter to accept a default. Ctrl+C to cancel.
|
|
|
226
240
|
|
|
227
241
|
cfg.PROVIDER_TTS = await menu('Choose TTS provider', [
|
|
228
242
|
['edge-tts', 'edge-tts — free, offline, 400+ voices. No key needed.'],
|
|
243
|
+
['voxforge', 'VoxForge — custom voice cloning (github.com/amg555/voxforge). Get key: your VoxForge instance.'],
|
|
229
244
|
['elevenlabs', 'ElevenLabs — best quality + voice cloning. elevenlabs.io'],
|
|
230
245
|
['azure', 'Azure Cognitive Speech. portal.azure.com'],
|
|
231
246
|
['openai', 'OpenAI TTS (uses OPENAI_API_KEY above)'],
|
|
232
|
-
['webhook', 'Custom/
|
|
247
|
+
['webhook', 'Custom/any OpenAI-compatible TTS endpoint'],
|
|
233
248
|
], 'edge-tts');
|
|
234
249
|
|
|
235
|
-
if (cfg.PROVIDER_TTS === '
|
|
250
|
+
if (cfg.PROVIDER_TTS === 'voxforge') {
|
|
251
|
+
cfg.VOXFORGE_URL = await ask(' VoxForge base URL', 'https://api.voxforge.io');
|
|
252
|
+
cfg.VOXFORGE_KEY = await askSecret(' VoxForge API key (leave blank if self-hosted without auth)');
|
|
253
|
+
cfg.VOXFORGE_VOICE = await ask(' Voice ID or cloned speaker ID', 'narrator');
|
|
254
|
+
cfg.VOXFORGE_MODEL = await ask(' Model ID (leave blank for default)', 'voxforge-v1');
|
|
255
|
+
console.log(' ℹ️ Self-hosted? Set VOXFORGE_URL to your instance URL (e.g. http://localhost:8080).');
|
|
256
|
+
} else if (cfg.PROVIDER_TTS === 'elevenlabs') {
|
|
236
257
|
cfg.ELEVENLABS_API_KEY = await askSecret(' ElevenLabs API key');
|
|
237
258
|
cfg.ELEVENLABS_VOICE_ID = await ask(' Voice ID (leave blank for default Adam)', 'pNInz6obpgDQGcFmaJgB');
|
|
238
259
|
console.log(' ℹ️ For voice cloning: set ELEVENLABS_VOICE_ID to your cloned voice ID.');
|
|
@@ -312,7 +333,7 @@ Press Enter to accept a default. Ctrl+C to cancel.
|
|
|
312
333
|
|
|
313
334
|
const groups = {
|
|
314
335
|
'# ── LLM ─────────────────────────────────────────────────────────': ['PROVIDER_LLM','GROQ_API_KEY','CEREBRAS_API_KEY','OPENROUTER_API_KEY','ANTHROPIC_API_KEY','GEMINI_API_KEY','MISTRAL_API_KEY','TOGETHER_API_KEY','OPENAI_API_KEY','OLLAMA_BASE_URL','OLLAMA_MODEL','WEBHOOK_LLM_URL','WEBHOOK_LLM_HEADERS','WEBHOOK_LLM_RESPONSE_PATH','LLM_MODEL'],
|
|
315
|
-
'# ── TTS ─────────────────────────────────────────────────────────': ['PROVIDER_TTS','ELEVENLABS_API_KEY','ELEVENLABS_VOICE_ID','AZURE_SPEECH_KEY','AZURE_SPEECH_REGION','WEBHOOK_TTS_URL','WEBHOOK_TTS_RESPONSE','WEBHOOK_TTS_RESPONSE_PATH','TTS_VOICE','TTS_RATE'],
|
|
336
|
+
'# ── TTS ─────────────────────────────────────────────────────────': ['PROVIDER_TTS','VOXFORGE_URL','VOXFORGE_KEY','VOXFORGE_VOICE','VOXFORGE_MODEL','ELEVENLABS_API_KEY','ELEVENLABS_VOICE_ID','AZURE_SPEECH_KEY','AZURE_SPEECH_REGION','WEBHOOK_TTS_URL','WEBHOOK_TTS_RESPONSE','WEBHOOK_TTS_RESPONSE_PATH','TTS_VOICE','TTS_RATE'],
|
|
316
337
|
'# ── Image generation ─────────────────────────────────────────────': ['PROVIDER_IMAGE','FALAI_API_KEY','REPLICATE_API_KEY','WEBHOOK_IMAGE_URL','WEBHOOK_IMAGE_RESPONSE','WEBHOOK_IMAGE_RESPONSE_PATH','POLLINATIONS_MODEL'],
|
|
317
338
|
'# ── Delivery webhooks ────────────────────────────────────────────': ['PUBLISH_WEBHOOK_URL','REVIEW_NOTIFY_URL'],
|
|
318
339
|
'# ── External platform adapter ────────────────────────────────────': ['EXTERNAL_PLATFORM_NAME','EXTERNAL_PLATFORM_URL','EXTERNAL_PLATFORM_KEY','EXTERNAL_PLATFORM_INGEST_PATH','EXTERNAL_PLATFORM_RENDER_PATH','EXTERNAL_PLATFORM_STATUS_PATH'],
|
package/src/mcp-server/server.js
CHANGED
|
@@ -592,14 +592,14 @@ Example scene:
|
|
|
592
592
|
// ── Tool 12: Multi-platform TTS ──────────────────────────────────────────
|
|
593
593
|
// Call any voice platform per-request without changing global .env
|
|
594
594
|
multi_voice_tts: {
|
|
595
|
-
description: 'Convert text to speech using any supported voice platform per-call. Overrides the global PROVIDER_TTS for this one request. Supports: edge-tts, elevenlabs, azure, openai,
|
|
595
|
+
description: 'Convert text to speech using any supported voice platform per-call. Overrides the global PROVIDER_TTS for this one request. Supports: edge-tts, voxforge, elevenlabs, azure, openai, webhook.',
|
|
596
596
|
inputSchema: {
|
|
597
597
|
type: 'object',
|
|
598
598
|
required: ['text', 'outputPath'],
|
|
599
599
|
properties: {
|
|
600
600
|
text: { type: 'string', description: 'Text to synthesise' },
|
|
601
601
|
outputPath: { type: 'string', description: 'Where to save the audio file (.m4a or .mp3)' },
|
|
602
|
-
platform: { type: 'string', enum: ['edge-tts','elevenlabs','azure','openai','webhook'], description: 'TTS platform. Overrides PROVIDER_TTS for this call.' },
|
|
602
|
+
platform: { type: 'string', enum: ['edge-tts','voxforge','elevenlabs','azure','openai','webhook'], description: 'TTS platform. Overrides PROVIDER_TTS for this call. "voxforge" = github.com/amg555/voxforge; uses VOXFORGE_URL + VOXFORGE_KEY.' },
|
|
603
603
|
voice: { type: 'string', description: 'Voice ID. edge-tts: e.g. en-US-GuyNeural. ElevenLabs: voice_id. Azure: short name. OpenAI: alloy/echo/fable/nova/onyx/shimmer.' },
|
|
604
604
|
rate: { type: 'string', description: 'Speaking rate, e.g. "-10%" for slower, "+20%" for faster (edge-tts)' },
|
|
605
605
|
apiKey: { type: 'string', description: 'API key for this call (optional — falls back to env var)' },
|
|
@@ -608,9 +608,11 @@ Example scene:
|
|
|
608
608
|
sceneIndex: { type: 'number', description: 'Optional scene index for naming the output file' },
|
|
609
609
|
},
|
|
610
610
|
},
|
|
611
|
-
handler: async ({ text, outputPath, platform, voice, rate, apiKey, webhookUrl, productionId = 'adhoc', sceneIndex }) => {
|
|
611
|
+
handler: async ({ text, outputPath, platform: _platform, voice, rate, apiKey, webhookUrl, productionId = 'adhoc', sceneIndex }) => {
|
|
612
612
|
const syncFs = require('fs');
|
|
613
613
|
const pathMod = require('path');
|
|
614
|
+
// Normalise platform name so 'voxforge' routes correctly
|
|
615
|
+
const platform = _platform ? _platform.toLowerCase() : undefined;
|
|
614
616
|
|
|
615
617
|
// Guard: outputPath must not escape FILES_DIR — prevent path traversal
|
|
616
618
|
if (outputPath) {
|
|
@@ -1003,17 +1005,18 @@ Modes:
|
|
|
1003
1005
|
"crop" — focal-point 9:16 mobile crop preview; pass timestamps as [aspect_ratio_string, t]
|
|
1004
1006
|
"full" — full render → out/<title>.mp4
|
|
1005
1007
|
"mux" — re-mux audio only (no re-render, use after changing audio)
|
|
1008
|
+
"kit" — export EDL/XML for Premiere Pro / DaVinci Resolve; run after "full"
|
|
1006
1009
|
"look" — pre-render linter (fonts, clipped text, NaN transforms, missing assets)
|
|
1007
1010
|
|
|
1008
1011
|
Review budget: at least one "sheet" per shot, one "strip" per key motion/transition, one "crop" per story-carrying face.
|
|
1009
|
-
Returns { outputPath, passed } for "full", or { outputPath } for sheet/strip/storyboard.
|
|
1012
|
+
Returns { outputPath, passed } for "full", or { outputPath } for sheet/strip/storyboard/kit.
|
|
1010
1013
|
Requires: npm install playwright && npx playwright install chromium (one-time setup in projectDir).`,
|
|
1011
1014
|
inputSchema: {
|
|
1012
1015
|
type: 'object',
|
|
1013
1016
|
required: ['projectDir', 'mode'],
|
|
1014
1017
|
properties: {
|
|
1015
1018
|
projectDir: { type: 'string', description: 'The directory created by scaffold_animation_project' },
|
|
1016
|
-
mode: { type: 'string', enum: ['sheet', 'stills', 'strip', 'storyboard', 'crop', 'full', 'mux', 'look'], description: 'Render mode' },
|
|
1019
|
+
mode: { type: 'string', enum: ['sheet', 'stills', 'strip', 'storyboard', 'crop', 'full', 'mux', 'kit', 'look'], description: 'Render mode' },
|
|
1017
1020
|
timestamps: { type: 'array', items: {}, description: 'For mode="stills": [t1, t2, ...]. For mode="strip": [startSeconds, endSeconds]. For mode="crop": ["9:16", t]. For mode="sheet": omit or pass shot times.' },
|
|
1018
1021
|
audioPath: { type: 'string', description: 'Audio file to mux in (for mode="full" or "mux")' },
|
|
1019
1022
|
workers: { type: 'number', default: 4, description: 'Parallel Playwright pages for mode="full"' },
|
|
@@ -1091,6 +1094,9 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1091
1094
|
} else if (mode === 'full' || mode === 'mux') {
|
|
1092
1095
|
const mp4 = files.find(f => f.endsWith('.mp4') && !f.includes('_preview'));
|
|
1093
1096
|
if (mp4) outputPath = pathMod.join(outDir, mp4);
|
|
1097
|
+
} else if (mode === 'kit') {
|
|
1098
|
+
const edl = files.find(f => f.endsWith('.edl') || f.endsWith('.xml'));
|
|
1099
|
+
if (edl) outputPath = pathMod.join(outDir, edl);
|
|
1094
1100
|
} else if (mode === 'stills' || mode === 'strip' || mode === 'crop') {
|
|
1095
1101
|
const stills = files.filter(f => f.endsWith('.png') || f.endsWith('.jpg'));
|
|
1096
1102
|
if (stills.length > 0) outputPath = stills.map(f => pathMod.join(outDir, f));
|
|
@@ -1114,13 +1120,118 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1114
1120
|
},
|
|
1115
1121
|
},
|
|
1116
1122
|
|
|
1123
|
+
// ── Tool 18: Export to editing app ────────────────────────────────────────
|
|
1124
|
+
// Send a finished animation project to any local editing app via MCP bridge.
|
|
1125
|
+
// Works with Premiere Pro, DaVinci Resolve, CapCut, After Effects, or Remotion.
|
|
1126
|
+
export_to_editor: {
|
|
1127
|
+
description: `Send a finished FrameForge animation to a local editing app for post-production.
|
|
1128
|
+
|
|
1129
|
+
Supported editors (configure via env or pass editorUrl directly):
|
|
1130
|
+
"premiere" — Adobe Premiere Pro via local REST bridge (PREMIERE_BRIDGE_URL, default :7777)
|
|
1131
|
+
"resolve" — DaVinci Resolve Remote API (RESOLVE_BRIDGE_URL, default :8080)
|
|
1132
|
+
"capcut" — CapCut Desktop via MCP bridge (CAPCUT_BRIDGE_URL, default :9090)
|
|
1133
|
+
"aftereffects" — Adobe After Effects via AEIO bridge (AE_BRIDGE_URL, default :7878)
|
|
1134
|
+
"remotion" — Remotion local dev server (REMOTION_BASE_URL, default :3001)
|
|
1135
|
+
"custom" — any HTTP endpoint (pass editorUrl)
|
|
1136
|
+
|
|
1137
|
+
What this does:
|
|
1138
|
+
1. Imports the rendered MP4 (and optionally the audio stem) into the editor
|
|
1139
|
+
2. Exports an EDL/XML timeline from the animation project (render_animation kit mode)
|
|
1140
|
+
3. Returns the editor's import confirmation or render job ID
|
|
1141
|
+
|
|
1142
|
+
The bridge servers are lightweight companion apps you run locally alongside the editor.
|
|
1143
|
+
They are not installed by FrameForge — see docs/editor-bridges/ for setup instructions.`,
|
|
1144
|
+
inputSchema: {
|
|
1145
|
+
type: 'object',
|
|
1146
|
+
required: ['projectDir'],
|
|
1147
|
+
properties: {
|
|
1148
|
+
projectDir: { type: 'string', description: 'The animation project directory (from scaffold_animation_project)' },
|
|
1149
|
+
editor: { type: 'string', enum: ['premiere', 'resolve', 'capcut', 'aftereffects', 'remotion', 'custom'], default: 'resolve', description: 'Target editing app' },
|
|
1150
|
+
editorUrl: { type: 'string', description: 'Override the editor bridge URL (e.g. http://localhost:8080). Useful for custom or remote bridges.' },
|
|
1151
|
+
audioPath: { type: 'string', description: 'Audio stem to import alongside the video (optional)' },
|
|
1152
|
+
importOnly: { type: 'boolean', default: false, description: 'If true, only import the clip — do not trigger an export/render in the editor.' },
|
|
1153
|
+
},
|
|
1154
|
+
},
|
|
1155
|
+
handler: async ({ projectDir, editor = 'resolve', editorUrl, audioPath, importOnly = false }) => {
|
|
1156
|
+
const syncFs = require('fs');
|
|
1157
|
+
const pathMod = require('path');
|
|
1158
|
+
const { ExternalAdapter } = require('../pipeline/external-adapter');
|
|
1159
|
+
|
|
1160
|
+
if (!projectDir) throw new Error('projectDir is required');
|
|
1161
|
+
if (!syncFs.existsSync(projectDir)) throw new Error(`Project directory not found: ${projectDir}`);
|
|
1162
|
+
|
|
1163
|
+
// Find the most recently rendered MP4 in out/
|
|
1164
|
+
const outDir = pathMod.join(projectDir, 'out');
|
|
1165
|
+
if (!syncFs.existsSync(outDir)) {
|
|
1166
|
+
throw new Error(`No out/ directory in ${projectDir}. Run render_animation with mode="full" first.`);
|
|
1167
|
+
}
|
|
1168
|
+
const mp4Files = syncFs.readdirSync(outDir)
|
|
1169
|
+
.filter(f => f.endsWith('.mp4') && !f.includes('_preview'))
|
|
1170
|
+
.map(f => pathMod.join(outDir, f))
|
|
1171
|
+
.sort((a, b) => syncFs.statSync(b).mtimeMs - syncFs.statSync(a).mtimeMs);
|
|
1172
|
+
|
|
1173
|
+
if (mp4Files.length === 0) {
|
|
1174
|
+
throw new Error(`No rendered MP4 found in ${outDir}. Run render_animation with mode="full" first.`);
|
|
1175
|
+
}
|
|
1176
|
+
const masterPath = mp4Files[0];
|
|
1177
|
+
|
|
1178
|
+
// Also find the EDL from kit mode (generate it now if missing)
|
|
1179
|
+
const edlFiles = syncFs.readdirSync(outDir).filter(f => f.endsWith('.edl') || f.endsWith('.xml'));
|
|
1180
|
+
let edlPath = edlFiles.length > 0 ? pathMod.join(outDir, edlFiles[0]) : null;
|
|
1181
|
+
|
|
1182
|
+
// Configure the adapter for the target editor
|
|
1183
|
+
// editorUrl overrides the preset base URL
|
|
1184
|
+
const adapterConfig = editorUrl
|
|
1185
|
+
? { name: editor, baseUrl: editorUrl }
|
|
1186
|
+
: { name: editor };
|
|
1187
|
+
|
|
1188
|
+
const adapter = new ExternalAdapter(adapterConfig);
|
|
1189
|
+
|
|
1190
|
+
// Build the file list to import
|
|
1191
|
+
const files = [masterPath];
|
|
1192
|
+
if (audioPath && syncFs.existsSync(audioPath)) files.push(audioPath);
|
|
1193
|
+
if (edlPath) files.push(edlPath);
|
|
1194
|
+
|
|
1195
|
+
let importResult;
|
|
1196
|
+
try {
|
|
1197
|
+
importResult = await adapter.ingestMedia({
|
|
1198
|
+
files,
|
|
1199
|
+
metadata: {
|
|
1200
|
+
source: 'frameforge',
|
|
1201
|
+
projectDir,
|
|
1202
|
+
editor,
|
|
1203
|
+
importOnly: String(importOnly),
|
|
1204
|
+
},
|
|
1205
|
+
});
|
|
1206
|
+
} catch (err) {
|
|
1207
|
+
// If the bridge is not running, give a clear message instead of a raw HTTP error
|
|
1208
|
+
throw new Error(
|
|
1209
|
+
`Could not reach ${editor} bridge at ${adapter.baseUrl}.\n` +
|
|
1210
|
+
`Make sure the bridge is running.\n` +
|
|
1211
|
+
`Error: ${err.message}\n\n` +
|
|
1212
|
+
`See docs/editor-bridges/${editor}.md for setup instructions.`
|
|
1213
|
+
);
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
return {
|
|
1217
|
+
editor,
|
|
1218
|
+
bridgeUrl: adapter.baseUrl,
|
|
1219
|
+
importedFiles: files.map(f => pathMod.basename(f)),
|
|
1220
|
+
masterPath,
|
|
1221
|
+
edlPath,
|
|
1222
|
+
importResult,
|
|
1223
|
+
message: `Imported into ${editor} at ${adapter.baseUrl}. ${edlPath ? 'EDL/XML timeline included.' : 'Tip: run render_animation mode="kit" to also export an EDL timeline.'}`,
|
|
1224
|
+
};
|
|
1225
|
+
},
|
|
1226
|
+
},
|
|
1227
|
+
|
|
1117
1228
|
};
|
|
1118
1229
|
|
|
1119
1230
|
// ── MCP protocol handler (MCP 2024-11-05 spec) ───────────────────────────
|
|
1120
1231
|
|
|
1121
1232
|
const SERVER_INFO = {
|
|
1122
1233
|
name: 'video-pipeline',
|
|
1123
|
-
version: '2.3.
|
|
1234
|
+
version: '2.3.2',
|
|
1124
1235
|
};
|
|
1125
1236
|
|
|
1126
1237
|
const CAPABILITIES = {
|
|
@@ -150,24 +150,20 @@ class ExternalAdapter {
|
|
|
150
150
|
_applyPreset() {
|
|
151
151
|
const name = this.name.toLowerCase();
|
|
152
152
|
const presets = {
|
|
153
|
+
// ── Voice / Audio platforms ─────────────────────────────────────────────
|
|
154
|
+
voxforge: {
|
|
155
|
+
baseUrl: this.baseUrl || process.env.VOXFORGE_URL || 'https://api.voxforge.io',
|
|
156
|
+
ingestPath: '/v1/voices/add', // upload a speaker sample for cloning
|
|
157
|
+
renderPath: '/v1/audio/speech', // OpenAI-compatible TTS synthesis
|
|
158
|
+
statusPath: '/v1/voices', // list voices / health check
|
|
159
|
+
},
|
|
153
160
|
elevenlabs: {
|
|
154
161
|
baseUrl: 'https://api.elevenlabs.io',
|
|
155
|
-
ingestPath: '/v1/voices/add',
|
|
156
|
-
renderPath: '/v1/text-to-speech',
|
|
157
|
-
statusPath: '/v1/voices',
|
|
158
|
-
},
|
|
159
|
-
remotion: {
|
|
160
|
-
baseUrl: process.env.REMOTION_BASE_URL || 'http://localhost:3001',
|
|
161
|
-
ingestPath: '/compositions',
|
|
162
|
-
renderPath: '/renders',
|
|
163
|
-
statusPath: '/renders',
|
|
164
|
-
},
|
|
165
|
-
voxforge: {
|
|
166
|
-
baseUrl: this.baseUrl || 'https://api.voxforge.io',
|
|
167
|
-
ingestPath: '/v1/ingest',
|
|
168
|
-
renderPath: '/v1/render',
|
|
169
|
-
statusPath: '/v1/status',
|
|
162
|
+
ingestPath: '/v1/voices/add',
|
|
163
|
+
renderPath: '/v1/text-to-speech',
|
|
164
|
+
statusPath: '/v1/voices',
|
|
170
165
|
},
|
|
166
|
+
// ── Video generation platforms ──────────────────────────────────────────
|
|
171
167
|
runway: {
|
|
172
168
|
baseUrl: 'https://api.runwayml.com',
|
|
173
169
|
ingestPath: '/v1/assets',
|
|
@@ -180,6 +176,50 @@ class ExternalAdapter {
|
|
|
180
176
|
renderPath: '/generate',
|
|
181
177
|
statusPath: '/generations',
|
|
182
178
|
},
|
|
179
|
+
kling: {
|
|
180
|
+
baseUrl: 'https://api.klingai.com',
|
|
181
|
+
ingestPath: '/v1/videos/image2video',
|
|
182
|
+
renderPath: '/v1/videos/image2video',
|
|
183
|
+
statusPath: '/v1/videos/image2video',
|
|
184
|
+
},
|
|
185
|
+
// ── Local editing app bridges ───────────────────────────────────────────
|
|
186
|
+
// These talk to local companion servers (CEP panel, local REST shim, etc.)
|
|
187
|
+
remotion: {
|
|
188
|
+
baseUrl: process.env.REMOTION_BASE_URL || 'http://localhost:3001',
|
|
189
|
+
ingestPath: '/compositions',
|
|
190
|
+
renderPath: '/renders',
|
|
191
|
+
statusPath: '/renders',
|
|
192
|
+
},
|
|
193
|
+
premiere: {
|
|
194
|
+
// Adobe Premiere Pro — requires the FrameForge CEP panel (see docs/premiere-panel/)
|
|
195
|
+
// OR the unofficial premiere-rest-bridge: github.com/premiere-rest-bridge
|
|
196
|
+
baseUrl: process.env.PREMIERE_BRIDGE_URL || 'http://localhost:7777',
|
|
197
|
+
ingestPath: '/import', // import a clip into the current project
|
|
198
|
+
renderPath: '/export', // export timeline to file
|
|
199
|
+
statusPath: '/status',
|
|
200
|
+
},
|
|
201
|
+
resolve: {
|
|
202
|
+
// DaVinci Resolve — requires local Resolve REST API bridge (port 8080 by default)
|
|
203
|
+
// Enable via: Preferences → System → Remote Control (DaVinci Resolve 18.6+)
|
|
204
|
+
baseUrl: process.env.RESOLVE_BRIDGE_URL || 'http://localhost:8080',
|
|
205
|
+
ingestPath: '/media/import',
|
|
206
|
+
renderPath: '/render/start',
|
|
207
|
+
statusPath: '/render/status',
|
|
208
|
+
},
|
|
209
|
+
capcut: {
|
|
210
|
+
// CapCut Desktop — requires CapCut MCP bridge (https://github.com/capcut-mcp)
|
|
211
|
+
baseUrl: process.env.CAPCUT_BRIDGE_URL || 'http://localhost:9090',
|
|
212
|
+
ingestPath: '/import',
|
|
213
|
+
renderPath: '/export',
|
|
214
|
+
statusPath: '/status',
|
|
215
|
+
},
|
|
216
|
+
aftereffects: {
|
|
217
|
+
// Adobe After Effects — requires AEIO REST bridge or ExtendScript server
|
|
218
|
+
baseUrl: process.env.AE_BRIDGE_URL || 'http://localhost:7878',
|
|
219
|
+
ingestPath: '/import',
|
|
220
|
+
renderPath: '/render',
|
|
221
|
+
statusPath: '/render/status',
|
|
222
|
+
},
|
|
183
223
|
};
|
|
184
224
|
|
|
185
225
|
const preset = presets[name];
|
|
@@ -390,12 +390,58 @@ class TTSProvider {
|
|
|
390
390
|
case 'elevenlabs': return this._elevenLabs({ text, voice: resolvedVoice, outputPath });
|
|
391
391
|
case 'azure': return this._azureTTS({ text, voice: resolvedVoice, outputPath });
|
|
392
392
|
case 'openai': return this._openaiTTS({ text, voice: resolvedVoice, outputPath });
|
|
393
|
+
case 'voxforge': return this._voxforgeTTS({ text, voice: resolvedVoice, outputPath });
|
|
393
394
|
case 'custom':
|
|
394
395
|
case 'webhook': return this._customTTS({ text, voice: resolvedVoice, outputPath });
|
|
395
396
|
default: return this._edgeTTS({ text, voice: resolvedVoice, outputPath });
|
|
396
397
|
}
|
|
397
398
|
}
|
|
398
399
|
|
|
400
|
+
/**
|
|
401
|
+
* VoxForge TTS — github.com/amg555/voxforge
|
|
402
|
+
* OpenAI-compatible /v1/audio/speech endpoint with optional voice cloning.
|
|
403
|
+
* Env vars:
|
|
404
|
+
* VOXFORGE_URL — base URL, e.g. https://api.voxforge.io (or self-hosted)
|
|
405
|
+
* VOXFORGE_KEY — API key (Bearer token)
|
|
406
|
+
* VOXFORGE_VOICE — voice ID or cloned speaker ID (default: 'narrator')
|
|
407
|
+
* VOXFORGE_MODEL — model ID (default: 'voxforge-v1')
|
|
408
|
+
*/
|
|
409
|
+
async _voxforgeTTS({ text, voice, outputPath }) {
|
|
410
|
+
const baseUrl = process.env.VOXFORGE_URL || 'https://api.voxforge.io';
|
|
411
|
+
const apiKey = this._overrides.apiKey || process.env.VOXFORGE_KEY || '';
|
|
412
|
+
const model = process.env.VOXFORGE_MODEL || 'voxforge-v1';
|
|
413
|
+
const resolvedVoice = voice || process.env.VOXFORGE_VOICE || 'narrator';
|
|
414
|
+
const rate = this._overrides.rate || process.env.TTS_RATE || '+0%';
|
|
415
|
+
|
|
416
|
+
const headers = { 'Content-Type': 'application/json' };
|
|
417
|
+
if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`;
|
|
418
|
+
|
|
419
|
+
// Try OpenAI-compatible /v1/audio/speech first; fall back to /v1/tts
|
|
420
|
+
const endpoints = [
|
|
421
|
+
`${baseUrl}/v1/audio/speech`,
|
|
422
|
+
`${baseUrl}/v1/tts`,
|
|
423
|
+
];
|
|
424
|
+
|
|
425
|
+
return withRetry(async () => {
|
|
426
|
+
for (const url of endpoints) {
|
|
427
|
+
try {
|
|
428
|
+
const isOpenAIPath = url.endsWith('/speech');
|
|
429
|
+
const body = isOpenAIPath
|
|
430
|
+
? { model, input: text, voice: resolvedVoice }
|
|
431
|
+
: { text, voice: resolvedVoice, rate, model };
|
|
432
|
+
const audio = await httpPost(url, body, headers);
|
|
433
|
+
await fs.writeFile(outputPath, Buffer.isBuffer(audio) ? audio : Buffer.from(audio, 'binary'));
|
|
434
|
+
const stat = await fs.stat(outputPath);
|
|
435
|
+
if (stat.size > 500) return outputPath;
|
|
436
|
+
} catch (err) {
|
|
437
|
+
if (endpoints.indexOf(url) === endpoints.length - 1) throw err;
|
|
438
|
+
// try next endpoint
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
throw new Error('VoxForge: all endpoints failed');
|
|
442
|
+
}, { maxAttempts: 3, baseDelayMs: 3000, label: 'TTS(voxforge)' });
|
|
443
|
+
}
|
|
444
|
+
|
|
399
445
|
async _customTTS({ text, voice, outputPath }) {
|
|
400
446
|
const url = this._overrides.webhookUrl || process.env.CUSTOM_TTS_URL || process.env.TTS_WEBHOOK_URL;
|
|
401
447
|
if (!url) throw new Error('CUSTOM_TTS_URL or TTS_WEBHOOK_URL not set for custom/webhook TTS provider');
|