@amg555/frameforge-mcp 2.3.8 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/package.json +5 -4
- package/plugins/video-pipeline/skills/video-pipeline/SKILL.md +9 -6
- package/scripts/deep-validate.js +6 -2
- package/src/harness/lib/browser.mjs +18 -1
- package/src/mcp-server/server.js +110 -1
- package/src/schema/scene.schema.json +3 -0
- package/src/video-renderer/scene-animator.js +117 -52
- package/src/worker/server-render-free.js +8 -0
- package/src/worker/server.js +10 -0
- package/src/workflows/assets.js +17 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,40 @@ All notable changes to FrameForge are documented here.
|
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
+
## [2.4.0] — 2026-11-09
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- **MCP Tool `generate_scene_code` (`src/mcp-server/server.js`, `cloudflare-mcp/src/index.js`)**:
|
|
11
|
+
- Synthesizes pure-function-of-time canvas drawing code for `window.seek(t)` matching the Claude Opus 5.5 / `ClaudeAnimationBase` contract.
|
|
12
|
+
- LLM-assisted scene code synthesis with automatic procedural fallback (supporting celestial orbits, candlestick graphs, neural nets, 3D wireframe polyhedra).
|
|
13
|
+
- Automatically wraps generated code with `wrapCodeInHarnessHtml` for headless Chromium execution.
|
|
14
|
+
- **MCP Tool `preview_scene` (`src/mcp-server/server.js`, `cloudflare-mcp/src/index.js`)**:
|
|
15
|
+
- Renders still frame previews (PNGs) of any scene at specified timestamps (defaults to `[0.1, duration/2, duration - 0.1]`).
|
|
16
|
+
- Implements the Layer 3 inspection / feedback loop, letting IDE agents (Claude Code, Cursor, Windsurf) visually inspect keyframes before encoding.
|
|
17
|
+
- **Automatic Scene Code Synthesis (`src/workflows/assets.js`)**:
|
|
18
|
+
- Automatically invokes `generateSceneCodeWithLLM` when scenes lack pre-rendered media, ensuring all agent-generated videos are true code animations rather than static slideshows.
|
|
19
|
+
|
|
20
|
+
### Enhanced
|
|
21
|
+
- **Full-Canvas Layout & Dynamic Scaling (`src/video-renderer/scene-animator.js`)**:
|
|
22
|
+
- Eliminated bottom glassmorphic rectangle card that caused PowerPoint-style slideshow appearances.
|
|
23
|
+
- Replaced with floating cinematic kinetic typography (subtle top title, bottom caption at `0.92H`, minimal progress line).
|
|
24
|
+
- Centered visual animations at `0.50H` with 1.4×–1.5× scale expansion.
|
|
25
|
+
- **Playwright Dependencies for Production (`package.json`)**:
|
|
26
|
+
- Moved `playwright` and `playwright-core` from `devDependencies` to production `dependencies` so `npx` and standard npm installs run headless Chromium out of the box without falling back to FFmpeg image loops.
|
|
27
|
+
- Expanded `findChromePath` in `src/harness/lib/browser.mjs` and `scripts/render.mjs` to auto-detect Microsoft Edge (`msedge.exe`), Brave, and system Chromium on Windows, macOS, and Linux.
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## [2.3.9] — 2026-11-09
|
|
32
|
+
|
|
33
|
+
### Enhanced
|
|
34
|
+
- **Code-Driven Animation Primitives (`src/video-renderer/scene-animator.js`)**:
|
|
35
|
+
- Expanded `wrapCodeInHarnessHtml` runtime environment with standard animation and easing helpers: `clamp`, `lerp`, `easeIn`, `easeOut`, `easeInOut`, and `boilSeed` (12fps hand-drawn wobble).
|
|
36
|
+
- Explicitly registered `html`, `code`, and `animation_code` in `src/schema/scene.schema.json` for first-class schema validation and tool discovery.
|
|
37
|
+
- Verified pure-function-of-time canvas code execution pipeline: scene code → Playwright virtual clock headless capture → FFmpeg BT.709 H.264/AAC MP4 encode.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
7
41
|
## [2.3.8] — 2026-11-09
|
|
8
42
|
|
|
9
43
|
### Fixed & Enhanced
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@amg555/frameforge-mcp",
|
|
3
|
-
"version": "2.
|
|
4
|
-
"description": "FrameForge — AI video production engine for agentic IDEs. npx @amg555/frameforge-mcp. FFmpeg + TTS run on your machine.
|
|
3
|
+
"version": "2.4.0",
|
|
4
|
+
"description": "FrameForge — AI video production engine for agentic IDEs. npx @amg555/frameforge-mcp. FFmpeg + TTS run on your machine. 21 MCP tools.",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
@@ -40,8 +40,9 @@
|
|
|
40
40
|
"engines": {
|
|
41
41
|
"node": ">=18.0.0"
|
|
42
42
|
},
|
|
43
|
-
"
|
|
44
|
-
"playwright": "^1.64.1"
|
|
43
|
+
"dependencies": {
|
|
44
|
+
"playwright": "^1.64.1",
|
|
45
|
+
"playwright-core": "^1.64.1"
|
|
45
46
|
},
|
|
46
47
|
"keywords": [
|
|
47
48
|
"frameforge",
|
|
@@ -26,7 +26,8 @@ IDE Agent (IBM Bob / Codex / opencode / Cursor / Windsurf / Claude Code)
|
|
|
26
26
|
│
|
|
27
27
|
▼ Cloudflare Remote MCP (recommended) ─────────────────────────────
|
|
28
28
|
│ https://video-mcp.YOUR_SUBDOMAIN.workers.dev/mcp
|
|
29
|
-
│
|
|
29
|
+
│ 21 tools: create_video_from_script · edit_video · render_from_scenes
|
|
30
|
+
│ generate_scene_code · preview_scene
|
|
30
31
|
│ ingest_media · edit_scenes · render_tts · multi_voice_tts
|
|
31
32
|
│ generate_scene_image · assemble_video · publish_video
|
|
32
33
|
│ research_topic · write_script · external_platform_call
|
|
@@ -145,7 +146,8 @@ FrameForge renders full-canvas, broadcast-quality motion graphics (24fps pure fu
|
|
|
145
146
|
|
|
146
147
|
- **Option A — Natural Description:** Provide `visual_description` (e.g. "Earth rotating with 4 satellite orbits and microwave beams intersecting"). FrameForge automatically synthesizes a full-canvas procedural 3D/2D animation entity (orbital mechanics, 3D DNA helix, candlestick charts, neural mesh, 3D wireframe polyhedra).
|
|
147
148
|
- **Option B — Direct Agent Code:** Provide `html` (complete page with `window.DURATION` & `window.seek(t)`) or `code` (canvas drawing instructions for `t`). FrameForge executes your program directly via headless Chromium.
|
|
148
|
-
- **Option C —
|
|
149
|
+
- **Option C — Generate & Inspect (Layer 3 Loop):** Call `generate_scene_code({ visual_description, style })` to synthesize canvas drawing code. Then call `preview_scene({ html, timestamps: [0.1, 1.5, 2.9] })` to render PNG stills to disk, visually verify the scene in the IDE, tweak if needed, and pass into `create_video_from_script`.
|
|
150
|
+
- **Option D — Multilingual & Regional Scripts:** Pass `language: "ml"` (Malayalam), `"hi"` (Hindi), etc. Native DirectWrite/HarfBuzz font stacks (`Nirmala UI`, `Noto Sans Malayalam`) render conjuncts without broken glyphs.
|
|
149
151
|
|
|
150
152
|
```js
|
|
151
153
|
create_video_from_script({
|
|
@@ -317,10 +319,11 @@ marks that production FAILED and moves to the next topic.
|
|
|
317
319
|
| LLM | openai | `OPENAI_API_KEY` | GPT-4o / o1 |
|
|
318
320
|
| LLM | ollama | _(none)_ | Local, OLLAMA_BASE_URL |
|
|
319
321
|
| TTS | edge-tts | _(none)_ | Free, offline, 300+ voices |
|
|
322
|
+
| TTS | voxforge | `VOXFORGE_URL` + `_KEY` | Local neural TTS & cloning (Rust) |
|
|
320
323
|
| TTS | elevenlabs | `ELEVENLABS_API_KEY` | Voice cloning |
|
|
321
324
|
| TTS | azure | `AZURE_SPEECH_KEY` | Enterprise |
|
|
322
325
|
| TTS | openai | `OPENAI_API_KEY` | High quality |
|
|
323
|
-
| Visual| code-driven | _(none)_ | Default —
|
|
326
|
+
| Visual| code-driven | _(none)_ | Default — pure-t canvas animation & kinetic text |
|
|
324
327
|
| Visual| user-media | _(none)_ | Caller `image_path` / `video_path` |
|
|
325
328
|
|
|
326
329
|
---
|
|
@@ -346,9 +349,9 @@ marks that production FAILED and moves to the next topic.
|
|
|
346
349
|
## Key files
|
|
347
350
|
|
|
348
351
|
```
|
|
349
|
-
src/mcp-server/server.js MCP HTTP server (
|
|
350
|
-
src/mcp-server/stdio.js MCP stdio server (
|
|
351
|
-
cloudflare-mcp/src/index.js Cloudflare Workers Remote MCP proxy
|
|
352
|
+
src/mcp-server/server.js MCP HTTP server (21 tools, port 3210)
|
|
353
|
+
src/mcp-server/stdio.js MCP stdio server (21 tools)
|
|
354
|
+
cloudflare-mcp/src/index.js Cloudflare Workers Remote MCP proxy (21 tools)
|
|
352
355
|
src/pipeline/providers.js LLM / TTS / Image abstraction (env-var-switchable providers)
|
|
353
356
|
src/pipeline/state-machine.js File + PostgreSQL production state
|
|
354
357
|
src/pipeline/retry-policy.js Exponential backoff, stage timeouts
|
package/scripts/deep-validate.js
CHANGED
|
@@ -20,12 +20,16 @@ check('batch handling', code.includes('Array.isArray(body)'));
|
|
|
20
20
|
check('ping handler', code.includes("method === 'ping'"));
|
|
21
21
|
check('notifications/init', code.includes("notifications/initialized"));
|
|
22
22
|
check('body size limit', code.includes('MAX_BODY_BYTES'));
|
|
23
|
-
check('SERVER_INFO version 2.
|
|
23
|
+
check('SERVER_INFO version 2.4.0', code.includes("version: '2.4.0'"));
|
|
24
24
|
check('18 tools defined (incl get_job_status)', code.includes('get_job_status'));
|
|
25
25
|
check('export_to_editor tool defined', code.includes('export_to_editor'));
|
|
26
26
|
check('edit_video tool defined', code.includes('edit_video'));
|
|
27
|
+
check('generate_scene_code tool defined', code.includes('generate_scene_code'));
|
|
28
|
+
check('preview_scene tool defined', code.includes('preview_scene'));
|
|
27
29
|
check('video-editor exists', fs.existsSync('./src/video-editor/editor.js'));
|
|
28
30
|
check('voxforge TTS provider', code.includes('voxforge'));
|
|
31
|
+
const pkg = JSON.parse(fs.readFileSync('./package.json', 'utf8'));
|
|
32
|
+
check('playwright in dependencies', Boolean(pkg.dependencies && pkg.dependencies.playwright));
|
|
29
33
|
|
|
30
34
|
// ── No execSync with shell strings in renderer or extractor ───────────────
|
|
31
35
|
console.log('\n=== Shell injection check ===');
|
|
@@ -295,7 +299,7 @@ check('cf-worker has checkRateLimit', cfWorker.includes('checkRateLimit'));
|
|
|
295
299
|
check('cf-worker hashes token (SHA-256)', cfWorker.includes('SHA-256'));
|
|
296
300
|
check('cf-worker uses RATE_LIMITS KV', cfWorker.includes('env.RATE_LIMITS'));
|
|
297
301
|
check('cf-worker returns 429 on limit', cfWorker.includes('429'));
|
|
298
|
-
check('cf-worker version 2.
|
|
302
|
+
check('cf-worker version 2.4.0', cfWorker.includes("'2.4.0'"));
|
|
299
303
|
check('wrangler.toml has kv_namespaces',fs.readFileSync('./cloudflare-mcp/wrangler.toml','utf8').includes('kv_namespaces'));
|
|
300
304
|
|
|
301
305
|
// ── plugins/skills present ────────────────────────────────────────────────
|
|
@@ -14,10 +14,23 @@ export function findChromePath(customPath = null) {
|
|
|
14
14
|
process.env.CHROME_PATH,
|
|
15
15
|
'C:/Program Files/Google/Chrome/Application/chrome.exe',
|
|
16
16
|
'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe',
|
|
17
|
+
'C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe',
|
|
18
|
+
'C:/Program Files/Microsoft/Edge/Application/msedge.exe',
|
|
19
|
+
'C:/Program Files/BraveSoftware/Brave-Browser/Application/brave.exe',
|
|
20
|
+
'C:/Program Files (x86)/BraveSoftware/Brave-Browser/Application/brave.exe',
|
|
17
21
|
process.env.LOCALAPPDATA ? path.join(process.env.LOCALAPPDATA, 'Google/Chrome/Application/chrome.exe') : null,
|
|
22
|
+
process.env.LOCALAPPDATA ? path.join(process.env.LOCALAPPDATA, 'Microsoft/Edge/Application/msedge.exe') : null,
|
|
23
|
+
process.env.LOCALAPPDATA ? path.join(process.env.LOCALAPPDATA, 'BraveSoftware/Brave-Browser/Application/brave.exe') : null,
|
|
18
24
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
25
|
+
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
|
|
26
|
+
'/Applications/Brave Browser.app/Contents/MacOS/Brave Browser',
|
|
27
|
+
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
|
19
28
|
'/usr/bin/google-chrome',
|
|
20
29
|
'/usr/bin/google-chrome-stable',
|
|
30
|
+
'/usr/bin/microsoft-edge',
|
|
31
|
+
'/usr/bin/microsoft-edge-stable',
|
|
32
|
+
'/usr/bin/brave-browser',
|
|
33
|
+
'/usr/bin/brave',
|
|
21
34
|
'/usr/bin/chromium',
|
|
22
35
|
'/usr/bin/chromium-browser',
|
|
23
36
|
'/snap/bin/chromium',
|
|
@@ -78,7 +91,11 @@ export async function launchHarnessBrowser(opts = {}) {
|
|
|
78
91
|
try {
|
|
79
92
|
({ chromium } = await import('playwright'));
|
|
80
93
|
} catch (err) {
|
|
81
|
-
|
|
94
|
+
try {
|
|
95
|
+
({ chromium } = await import('playwright-core'));
|
|
96
|
+
} catch (errCore) {
|
|
97
|
+
throw new Error('Playwright is required. Run: npm install playwright && npx playwright install chromium');
|
|
98
|
+
}
|
|
82
99
|
}
|
|
83
100
|
|
|
84
101
|
const execPath = findChromePath(opts.chromePath);
|
package/src/mcp-server/server.js
CHANGED
|
@@ -1518,13 +1518,122 @@ Features:
|
|
|
1518
1518
|
},
|
|
1519
1519
|
},
|
|
1520
1520
|
|
|
1521
|
+
// ── Tool 20: Generate Scene Code ──────────────────────────────────────────
|
|
1522
|
+
// Writes pure-function-of-t Canvas drawing code using LLM or procedural synthesis
|
|
1523
|
+
generate_scene_code: {
|
|
1524
|
+
description: `Generate pure-function-of-t JavaScript HTML5 Canvas drawing code for a scene.
|
|
1525
|
+
Matches the Claude Opus 5.5 / ClaudeAnimationBase contract where every frame is a pure mathematical function of time t.
|
|
1526
|
+
Returns the raw code and complete wrapped HTML ready for headless browser execution.`,
|
|
1527
|
+
inputSchema: {
|
|
1528
|
+
type: 'object',
|
|
1529
|
+
required: ['visual_description'],
|
|
1530
|
+
properties: {
|
|
1531
|
+
visual_description: { type: 'string', description: 'Visual direction and motion description of the scene' },
|
|
1532
|
+
caption: { type: 'string', description: 'Headline / key concept' },
|
|
1533
|
+
narration: { type: 'string', description: 'Voiceover narration text' },
|
|
1534
|
+
topic: { type: 'string', description: 'Overall video topic' },
|
|
1535
|
+
style: { type: 'string', description: 'Aesthetic / genre (cosmic_space, cyber_tech, cinematic_documentary, finance_executive, retro_synthwave, nature_organic, hype_sports, minimal_luxury, breaking_news, warm_creative)' },
|
|
1536
|
+
language: { type: 'string', default: 'en', description: 'Language code (e.g. en, ml, hi, ta)' },
|
|
1537
|
+
duration: { type: 'number', default: 3, description: 'Duration in seconds' },
|
|
1538
|
+
width: { type: 'number', default: 1920 },
|
|
1539
|
+
height: { type: 'number', default: 1080 },
|
|
1540
|
+
},
|
|
1541
|
+
},
|
|
1542
|
+
handler: async ({ visual_description, caption = '', narration = '', topic = '', style = '', language = 'en', duration = 3, width = 1920, height = 1080 }) => {
|
|
1543
|
+
const { generateSceneCodeWithLLM, wrapCodeInHarnessHtml, generateSceneHtml } = require('../video-renderer/scene-animator');
|
|
1544
|
+
let code = await generateSceneCodeWithLLM({ visualDescription: visual_description, caption, narration, topic, style, language, duration, width, height });
|
|
1545
|
+
let html;
|
|
1546
|
+
if (code) {
|
|
1547
|
+
html = wrapCodeInHarnessHtml({ code, duration, width, height, language, style, topic });
|
|
1548
|
+
} else {
|
|
1549
|
+
html = generateSceneHtml({ visualDescription: visual_description, caption, narration, topic, style, language, duration, width, height });
|
|
1550
|
+
}
|
|
1551
|
+
return { code: code || '// Procedural animation used', html, duration, width, height, success: true };
|
|
1552
|
+
},
|
|
1553
|
+
},
|
|
1554
|
+
|
|
1555
|
+
// ── Tool 21: Preview Scene ───────────────────────────────────────────────
|
|
1556
|
+
// Renders still preview frames at specified timestamps so the agent can inspect them
|
|
1557
|
+
preview_scene: {
|
|
1558
|
+
description: `Render still frame previews (PNGs) of an animated scene at specified timestamps (default: 0.1s, duration/2, duration - 0.1s).
|
|
1559
|
+
Allows the agent in Claude Code, Cursor, or any IDE to inspect the visual output, verify fonts, colors, and compositions, and patch the code before encoding. Satisfies Layer 3 verification loop.`,
|
|
1560
|
+
inputSchema: {
|
|
1561
|
+
type: 'object',
|
|
1562
|
+
properties: {
|
|
1563
|
+
code: { type: 'string', description: 'Raw JavaScript canvas drawing code for window.seek(t)' },
|
|
1564
|
+
html: { type: 'string', description: 'Complete self-contained HTML page conforming to window.seek(t)' },
|
|
1565
|
+
visual_description: { type: 'string', description: 'Visual description (used to generate scene if code/html omitted)' },
|
|
1566
|
+
caption: { type: 'string' },
|
|
1567
|
+
narration: { type: 'string' },
|
|
1568
|
+
topic: { type: 'string' },
|
|
1569
|
+
style: { type: 'string' },
|
|
1570
|
+
language: { type: 'string', default: 'en' },
|
|
1571
|
+
duration: { type: 'number', default: 3 },
|
|
1572
|
+
timestamps: { type: 'array', items: { type: 'number' }, description: 'Timestamps in seconds to capture (default: [0.1, duration/2, duration - 0.1])' },
|
|
1573
|
+
width: { type: 'number', default: 1920 },
|
|
1574
|
+
height: { type: 'number', default: 1080 },
|
|
1575
|
+
},
|
|
1576
|
+
},
|
|
1577
|
+
handler: async ({ code, html, visual_description = '', caption = '', narration = '', topic = '', style = '', language = 'en', duration = 3, timestamps, width = 1920, height = 1080 }) => {
|
|
1578
|
+
const { wrapCodeInHarnessHtml, generateSceneHtml } = require('../video-renderer/scene-animator');
|
|
1579
|
+
const { launchHarnessBrowser } = require('../harness/lib/browser.mjs');
|
|
1580
|
+
const { openPageWithVirtualClock } = require('../harness/lib/capture.mjs');
|
|
1581
|
+
const syncFs = require('fs');
|
|
1582
|
+
const pathMod = require('path');
|
|
1583
|
+
|
|
1584
|
+
let htmlContent = html;
|
|
1585
|
+
if (!htmlContent) {
|
|
1586
|
+
if (code) {
|
|
1587
|
+
htmlContent = wrapCodeInHarnessHtml({ code, duration, width, height, language, style, topic });
|
|
1588
|
+
} else {
|
|
1589
|
+
htmlContent = generateSceneHtml({ visualDescription: visual_description, caption, narration, topic, style, language, duration, width, height });
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
const previewDir = pathMod.join(FILES_DIR, 'previews');
|
|
1594
|
+
syncFs.mkdirSync(previewDir, { recursive: true });
|
|
1595
|
+
const tempHtmlPath = pathMod.join(previewDir, `preview_${Date.now()}_${Math.random().toString(36).slice(2, 6)}.html`);
|
|
1596
|
+
syncFs.writeFileSync(tempHtmlPath, htmlContent, 'utf8');
|
|
1597
|
+
|
|
1598
|
+
const tsList = Array.isArray(timestamps) && timestamps.length > 0
|
|
1599
|
+
? timestamps
|
|
1600
|
+
: [0.1, Number((duration / 2).toFixed(2)), Number((duration - 0.1).toFixed(2))];
|
|
1601
|
+
|
|
1602
|
+
const { browser } = await launchHarnessBrowser();
|
|
1603
|
+
const captures = [];
|
|
1604
|
+
|
|
1605
|
+
try {
|
|
1606
|
+
const page = await openPageWithVirtualClock(browser, tempHtmlPath, { width, height });
|
|
1607
|
+
const { captureFrame } = require('../harness/lib/capture.mjs');
|
|
1608
|
+
for (const t of tsList) {
|
|
1609
|
+
const buf = await captureFrame(page, t, { type: 'image/png' });
|
|
1610
|
+
const outPng = pathMod.join(previewDir, `still_t${t.toString().replace('.', '_')}_${Date.now()}.png`);
|
|
1611
|
+
syncFs.writeFileSync(outPng, buf);
|
|
1612
|
+
captures.push({ timestamp: t, path: outPng, sizeBytes: buf.length });
|
|
1613
|
+
}
|
|
1614
|
+
} finally {
|
|
1615
|
+
await browser.close();
|
|
1616
|
+
try { syncFs.unlinkSync(tempHtmlPath); } catch (_) {}
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
return {
|
|
1620
|
+
previews: captures,
|
|
1621
|
+
count: captures.length,
|
|
1622
|
+
width,
|
|
1623
|
+
height,
|
|
1624
|
+
duration,
|
|
1625
|
+
message: 'Preview stills captured. You can read the image files from disk to inspect the visual composition.'
|
|
1626
|
+
};
|
|
1627
|
+
},
|
|
1628
|
+
},
|
|
1629
|
+
|
|
1521
1630
|
};
|
|
1522
1631
|
|
|
1523
1632
|
// ── MCP protocol handler (MCP 2024-11-05 spec) ───────────────────────────
|
|
1524
1633
|
|
|
1525
1634
|
const SERVER_INFO = {
|
|
1526
1635
|
name: 'video-pipeline',
|
|
1527
|
-
version: '2.
|
|
1636
|
+
version: '2.4.0',
|
|
1528
1637
|
};
|
|
1529
1638
|
|
|
1530
1639
|
const CAPABILITIES = {
|
|
@@ -12,6 +12,9 @@
|
|
|
12
12
|
"video_path": { "type": "string", "description": "Optional path to user-supplied video clip or B-roll" },
|
|
13
13
|
"image_path": { "type": "string", "description": "Optional path to user-supplied static image" },
|
|
14
14
|
"audio_path": { "type": "string", "description": "Optional path to user-supplied pre-recorded voiceover" },
|
|
15
|
+
"html": { "type": "string", "description": "Optional self-contained HTML animation page conforming to window.seek(t)" },
|
|
16
|
+
"code": { "type": "string", "description": "Optional canvas drawing JavaScript code executed for time t in window.seek" },
|
|
17
|
+
"animation_code": { "type": "string", "description": "Alias for code" },
|
|
15
18
|
"visual_description": { "type": "string", "minLength": 1 },
|
|
16
19
|
"character": {
|
|
17
20
|
"type": "object",
|
|
@@ -406,6 +406,21 @@ function sfc32(a, b, c, d) {
|
|
|
406
406
|
}
|
|
407
407
|
const hash = (i) => sfc32(i * 19 + 42, i * 29 + 1337, i * 13 + 7, 99)();
|
|
408
408
|
|
|
409
|
+
// Animation Primitives & Easing
|
|
410
|
+
function clamp(val, min = 0, max = 1) { return Math.max(min, Math.min(max, val)); }
|
|
411
|
+
function lerp(t, a, b, t0 = 0, t1 = 1) {
|
|
412
|
+
if (t <= t0) return a;
|
|
413
|
+
if (t >= t1) return b;
|
|
414
|
+
return a + (b - a) * ((t - t0) / (t1 - t0));
|
|
415
|
+
}
|
|
416
|
+
function easeIn(t) { return clamp(t, 0, 1) ** 2; }
|
|
417
|
+
function easeOut(t) { const c = clamp(t, 0, 1); return 1 - (1 - c) ** 2; }
|
|
418
|
+
function easeInOut(t) { const c = clamp(t, 0, 1); return c < 0.5 ? 2 * c * c : 1 - (-2 * c + 2) ** 2 / 2; }
|
|
419
|
+
function boilSeed(t, seed = 0) {
|
|
420
|
+
const frame12 = Math.floor(t * 12);
|
|
421
|
+
return hash(seed * 1000 + frame12);
|
|
422
|
+
}
|
|
423
|
+
|
|
409
424
|
// Spring dynamics
|
|
410
425
|
function spring(t, delay = 0, tension = 180, friction = 14) {
|
|
411
426
|
const dt = t - delay;
|
|
@@ -670,18 +685,18 @@ function drawBackground(t) {
|
|
|
670
685
|
// ── Full-Canvas Cinematic Visual Directors ─────────────────────────────────────
|
|
671
686
|
function drawMainScene(t) {
|
|
672
687
|
ctx.save();
|
|
673
|
-
const enterSpring = spring(t, 0.05);
|
|
674
|
-
const sceneScale = Math.min(1.
|
|
675
|
-
ctx.translate(W / 2, H * 0.
|
|
688
|
+
const enterSpring = spring(t, 0.05, 170, 13);
|
|
689
|
+
const sceneScale = Math.min(1.04, Math.max(0.88, enterSpring));
|
|
690
|
+
ctx.translate(W / 2, H * 0.50);
|
|
676
691
|
ctx.scale(sceneScale, sceneScale);
|
|
677
|
-
ctx.translate(-W / 2, -H * 0.
|
|
692
|
+
ctx.translate(-W / 2, -H * 0.50);
|
|
678
693
|
|
|
679
694
|
const cx = W / 2;
|
|
680
|
-
const cy = H * 0.
|
|
695
|
+
const cy = H * 0.50;
|
|
681
696
|
|
|
682
697
|
if (COMP_THEME === 'celestial_orbit') {
|
|
683
698
|
// 3D Celestial Body (Earth) with orbiting satellites & signal pulses
|
|
684
|
-
const earthR = Math.min(W, H) * 0.
|
|
699
|
+
const earthR = Math.min(W, H) * 0.22;
|
|
685
700
|
|
|
686
701
|
// Atmospheric halo
|
|
687
702
|
const haloGrad = ctx.createRadialGradient(cx, cy, earthR * 0.9, cx, cy, earthR * 1.45);
|
|
@@ -1153,69 +1168,60 @@ function drawMainScene(t) {
|
|
|
1153
1168
|
ctx.restore();
|
|
1154
1169
|
}
|
|
1155
1170
|
|
|
1156
|
-
// ──
|
|
1171
|
+
// ── Kinetic Typography & Broadcast Subtitle Overlay ─────────────────────────
|
|
1157
1172
|
function drawCinematicOverlay(t) {
|
|
1158
1173
|
ctx.save();
|
|
1159
|
-
const textSpring = spring(t, 0.
|
|
1160
|
-
const
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
// Accent Left Bar
|
|
1177
|
-
ctx.fillStyle = ACCENT;
|
|
1178
|
-
ctx.beginPath();
|
|
1179
|
-
ctx.roundRect(bannerX + 16, bannerY + 16, 6, bannerH - 32, 3);
|
|
1180
|
-
ctx.fill();
|
|
1181
|
-
|
|
1182
|
-
// Tag Pill
|
|
1183
|
-
ctx.font = 'bold 13px monospace';
|
|
1184
|
-
ctx.fillStyle = ACCENT;
|
|
1185
|
-
ctx.textAlign = 'left';
|
|
1186
|
-
ctx.fillText(TAG, bannerX + 38, bannerY + 36);
|
|
1174
|
+
const textSpring = spring(t, 0.12, 190, 15);
|
|
1175
|
+
const headlineAlpha = Math.min(1, Math.max(0, textSpring));
|
|
1176
|
+
ctx.globalAlpha = headlineAlpha;
|
|
1177
|
+
|
|
1178
|
+
// Kinetic Headline at top
|
|
1179
|
+
const capY = H * 0.12 + (1 - textSpring) * -20;
|
|
1180
|
+
|
|
1181
|
+
// Tag Pill - minimal floating badge
|
|
1182
|
+
if (TAG) {
|
|
1183
|
+
ctx.font = 'bold 16px monospace';
|
|
1184
|
+
ctx.fillStyle = ACCENT;
|
|
1185
|
+
ctx.shadowColor = 'rgba(0, 0, 0, 0.85)';
|
|
1186
|
+
ctx.shadowBlur = 10;
|
|
1187
|
+
ctx.textAlign = 'left';
|
|
1188
|
+
ctx.fillText('// ' + TAG, 80, capY - 22);
|
|
1189
|
+
}
|
|
1187
1190
|
|
|
1188
|
-
// Primary Caption Headline
|
|
1189
|
-
ctx.font = FONT_WEIGHT + '
|
|
1191
|
+
// Primary Caption Headline - crisp, heroic typography without opaque cards
|
|
1192
|
+
ctx.font = FONT_WEIGHT + ' 50px ' + FONT_DISPLAY;
|
|
1190
1193
|
ctx.fillStyle = '#ffffff';
|
|
1191
|
-
ctx.shadowColor = 'rgba(0, 0, 0, 0.
|
|
1192
|
-
ctx.shadowBlur =
|
|
1194
|
+
ctx.shadowColor = 'rgba(0, 0, 0, 0.95)';
|
|
1195
|
+
ctx.shadowBlur = 24;
|
|
1193
1196
|
ctx.shadowOffsetY = 4;
|
|
1194
1197
|
ctx.textAlign = 'left';
|
|
1195
|
-
const cleanCap = CAPTION.length >
|
|
1196
|
-
ctx.fillText(cleanCap,
|
|
1198
|
+
const cleanCap = CAPTION.length > 60 ? CAPTION.slice(0, 60) + '...' : CAPTION;
|
|
1199
|
+
ctx.fillText(cleanCap, 80, capY + 30);
|
|
1197
1200
|
|
|
1198
|
-
// Subtitle / Narration
|
|
1201
|
+
// Subtitle / Spoken Narration - bottom center clean broadcast subtitles
|
|
1199
1202
|
if (NARRATION) {
|
|
1200
|
-
|
|
1201
|
-
ctx.
|
|
1202
|
-
ctx.
|
|
1203
|
-
|
|
1204
|
-
ctx.
|
|
1203
|
+
const subSpring = spring(t, 0.22, 180, 16);
|
|
1204
|
+
ctx.globalAlpha = Math.min(1, Math.max(0, subSpring));
|
|
1205
|
+
ctx.font = '500 28px ' + FONT_BODY;
|
|
1206
|
+
ctx.fillStyle = '#f8fafc';
|
|
1207
|
+
ctx.shadowColor = 'rgba(0, 0, 0, 0.95)';
|
|
1208
|
+
ctx.shadowBlur = 18;
|
|
1209
|
+
ctx.shadowOffsetY = 3;
|
|
1210
|
+
ctx.textAlign = 'center';
|
|
1211
|
+
const cleanNarr = NARRATION.length > 115 ? NARRATION.slice(0, 115) + '...' : NARRATION;
|
|
1212
|
+
ctx.fillText(cleanNarr, W / 2, H * 0.92);
|
|
1205
1213
|
}
|
|
1206
1214
|
ctx.restore();
|
|
1207
1215
|
|
|
1208
|
-
// Bottom Progress
|
|
1216
|
+
// Bottom Progress Indicator (minimal line)
|
|
1209
1217
|
ctx.save();
|
|
1210
1218
|
const progress = Math.min(1, Math.max(0, t / window.DURATION));
|
|
1211
|
-
ctx.
|
|
1212
|
-
ctx.fillRect(0, H - 6, W, 6);
|
|
1213
|
-
|
|
1219
|
+
ctx.globalAlpha = 0.85;
|
|
1214
1220
|
const progGrad = ctx.createLinearGradient(0, 0, W, 0);
|
|
1215
1221
|
progGrad.addColorStop(0, ACCENT);
|
|
1216
1222
|
progGrad.addColorStop(1, ACCENT2);
|
|
1217
1223
|
ctx.fillStyle = progGrad;
|
|
1218
|
-
ctx.fillRect(0, H -
|
|
1224
|
+
ctx.fillRect(0, H - 4, W * progress, 4);
|
|
1219
1225
|
ctx.restore();
|
|
1220
1226
|
}
|
|
1221
1227
|
|
|
@@ -1230,9 +1236,68 @@ window.seek = async (t) => {
|
|
|
1230
1236
|
</html>`;
|
|
1231
1237
|
}
|
|
1232
1238
|
|
|
1239
|
+
// ── LLM Creative Code Generator ──────────────────────────────────────────────
|
|
1240
|
+
/**
|
|
1241
|
+
* Asks an LLM (Claude, Groq, Gemini, OpenAI, etc.) to write raw pure-function-of-t
|
|
1242
|
+
* HTML5 Canvas animation code for a given scene brief.
|
|
1243
|
+
* Returns raw JavaScript for window.seek = async (t) => { ... }, or null if unavailable.
|
|
1244
|
+
*/
|
|
1245
|
+
async function generateSceneCodeWithLLM({
|
|
1246
|
+
visualDescription = '',
|
|
1247
|
+
caption = '',
|
|
1248
|
+
narration = '',
|
|
1249
|
+
topic = '',
|
|
1250
|
+
style = '',
|
|
1251
|
+
language = '',
|
|
1252
|
+
duration = 3,
|
|
1253
|
+
width = 1920,
|
|
1254
|
+
height = 1080,
|
|
1255
|
+
} = {}) {
|
|
1256
|
+
try {
|
|
1257
|
+
const { LLMProvider } = require('../pipeline/providers');
|
|
1258
|
+
const llm = new LLMProvider();
|
|
1259
|
+
const system = `You are an elite creative motion graphics developer writing high-end JavaScript Canvas animation code for video production.
|
|
1260
|
+
You write code where EVERY PIXEL IS A PURE FUNCTION OF TIME t (seconds).
|
|
1261
|
+
AVAILABLE VARIABLES:
|
|
1262
|
+
- canvas, ctx, W (${width}), H (${height}), DURATION (${duration}), FONT_DISPLAY, FONT_BODY
|
|
1263
|
+
AVAILABLE HELPERS:
|
|
1264
|
+
- spring(t, delay, tension, friction), lerp(t, a, b, t0, t1), clamp(val, min, max)
|
|
1265
|
+
- easeIn(t), easeOut(t), easeInOut(t), boilSeed(t, seed), hash(i)
|
|
1266
|
+
RULES:
|
|
1267
|
+
1. Every element must animate smoothly over time t [0..DURATION].
|
|
1268
|
+
2. Fill the canvas with rich visual geometry, vectors, animated nodes, characters, particles, or cinematic diagrams matching the prompt.
|
|
1269
|
+
3. NEVER use Math.random(), setTimeout, setInterval, Date.now(), performance.now().
|
|
1270
|
+
4. Output ONLY raw executable JavaScript code for the body of window.seek = async (t) => { ... }.
|
|
1271
|
+
5. NO markdown backticks, NO explanations. Start directly with JavaScript.`;
|
|
1272
|
+
|
|
1273
|
+
const user = `TOPIC: ${topic || 'Visual Explainer'}
|
|
1274
|
+
VISUAL DIRECTION: ${visualDescription || caption || 'Dynamic motion graphics'}
|
|
1275
|
+
KEY PHRASE: ${caption}
|
|
1276
|
+
NARRATION: ${narration}
|
|
1277
|
+
STYLE: ${style || 'cinematic'}
|
|
1278
|
+
DURATION: ${duration}s
|
|
1279
|
+
|
|
1280
|
+
Write the canvas drawing code for time t.`;
|
|
1281
|
+
|
|
1282
|
+
const raw = await llm.chat({ system, user, temperature: 0.7, maxTokens: 4000 });
|
|
1283
|
+
if (!raw) return null;
|
|
1284
|
+
let cleanCode = raw.trim();
|
|
1285
|
+
if (cleanCode.startsWith('```')) {
|
|
1286
|
+
cleanCode = cleanCode.replace(/^```[a-z]*\n?/i, '').replace(/\n?```$/i, '').trim();
|
|
1287
|
+
}
|
|
1288
|
+
if (cleanCode.length > 20 && (cleanCode.includes('ctx.') || cleanCode.includes('fillRect'))) {
|
|
1289
|
+
return cleanCode;
|
|
1290
|
+
}
|
|
1291
|
+
return null;
|
|
1292
|
+
} catch (_) {
|
|
1293
|
+
return null;
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1233
1297
|
module.exports = {
|
|
1234
1298
|
generateSceneHtml,
|
|
1235
1299
|
wrapCodeInHarnessHtml,
|
|
1300
|
+
generateSceneCodeWithLLM,
|
|
1236
1301
|
detectAesthetic,
|
|
1237
1302
|
detectComponentTheme,
|
|
1238
1303
|
getScriptFontStack,
|
|
@@ -182,6 +182,14 @@ async function dispatchJob(job) {
|
|
|
182
182
|
const editor = new VideoEditor({ filesDir: FILES_DIR });
|
|
183
183
|
return editor.editAndAssemble(payload);
|
|
184
184
|
}
|
|
185
|
+
case 'generate_scene_code': {
|
|
186
|
+
const { TOOLS } = require('../mcp-server/server');
|
|
187
|
+
return TOOLS.generate_scene_code.handler(payload);
|
|
188
|
+
}
|
|
189
|
+
case 'preview_scene': {
|
|
190
|
+
const { TOOLS } = require('../mcp-server/server');
|
|
191
|
+
return TOOLS.preview_scene.handler(payload);
|
|
192
|
+
}
|
|
185
193
|
case 'full_pipeline': return runFullPipeline(payload, job.id);
|
|
186
194
|
default: throw new Error(`Unknown job type: ${type}`);
|
|
187
195
|
}
|
package/src/worker/server.js
CHANGED
|
@@ -102,6 +102,16 @@ async function executeJob(job) {
|
|
|
102
102
|
result = await editor.editAndAssemble(job.payload);
|
|
103
103
|
break;
|
|
104
104
|
}
|
|
105
|
+
case 'generate_scene_code': {
|
|
106
|
+
const { TOOLS } = require('../mcp-server/server');
|
|
107
|
+
result = await TOOLS.generate_scene_code.handler(job.payload);
|
|
108
|
+
break;
|
|
109
|
+
}
|
|
110
|
+
case 'preview_scene': {
|
|
111
|
+
const { TOOLS } = require('../mcp-server/server');
|
|
112
|
+
result = await TOOLS.preview_scene.handler(job.payload);
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
105
115
|
default: throw new Error(`Unknown job type: ${job.type}`);
|
|
106
116
|
}
|
|
107
117
|
job.status = 'SUCCEEDED';
|
package/src/workflows/assets.js
CHANGED
|
@@ -73,6 +73,22 @@ async function runAssets({
|
|
|
73
73
|
visualType = 'user_image';
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
+
// Automatically synthesize custom programmatic Canvas animation code with LLM if available
|
|
77
|
+
let sceneCode = scene.code || scene.animation_code || null;
|
|
78
|
+
if (!scene.video_path && !scene.image_path && !scene.html && !sceneCode) {
|
|
79
|
+
try {
|
|
80
|
+
const { generateSceneCodeWithLLM } = require('../video-renderer/scene-animator');
|
|
81
|
+
sceneCode = await generateSceneCodeWithLLM({
|
|
82
|
+
visualDescription: scene.visual_description || scene.narration,
|
|
83
|
+
caption: scene.caption,
|
|
84
|
+
narration: scene.narration,
|
|
85
|
+
topic: scene.topic || topic,
|
|
86
|
+
style: scene.style || style,
|
|
87
|
+
language: scene.language || language,
|
|
88
|
+
});
|
|
89
|
+
} catch (_) {}
|
|
90
|
+
}
|
|
91
|
+
|
|
76
92
|
// ── TTS / User Audio ───────────────────────────────────────────────
|
|
77
93
|
// Priority: audio_path (pre-recorded) > TTS synthesis.
|
|
78
94
|
// --no-tts / skipTts: skip synthesis (scene must have audio_path or silence is used).
|
|
@@ -108,7 +124,7 @@ async function runAssets({
|
|
|
108
124
|
musicMood: scene.music_mood || 'neutral',
|
|
109
125
|
language: scene.language || language || '',
|
|
110
126
|
html: scene.html || null,
|
|
111
|
-
code:
|
|
127
|
+
code: sceneCode,
|
|
112
128
|
startTime: scene.start_time != null ? scene.start_time : scene.trim_start,
|
|
113
129
|
endTime: scene.end_time != null ? scene.end_time : scene.trim_end,
|
|
114
130
|
speed: scene.speed != null ? scene.speed : 1.0,
|