@amg555/frameforge-mcp 0.0.0-stage → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bob/artifacts/enterprise-ai-video-agent-platform-agnostic-implementation-guide.html +464 -0
- package/.claude-plugin/marketplace.json +17 -0
- package/.gitattributes +19 -0
- package/.mcp.json +45 -0
- package/CHANGELOG.md +431 -0
- package/CLAUDE.md +278 -0
- package/CONTRIBUTING.md +112 -0
- package/LICENSE +21 -0
- package/Procfile +1 -0
- package/README.md +359 -2
- package/SECURITY.md +71 -0
- package/START_HERE.md +144 -0
- package/assets/metadata/assets.json +5 -0
- package/assets/metadata/embeddings.json +5 -0
- package/install.sh +64 -0
- package/package.json +58 -4
- package/src/animation/animation-system.js +229 -0
- package/src/asset-library/README.md +188 -0
- package/src/asset-library/asset-library.js +639 -0
- package/src/audio-mixer/mixer.js +187 -0
- package/src/cli/doctor.js +173 -0
- package/src/cli/make.js +655 -0
- package/src/cli/setup-keys.js +378 -0
- package/src/config/load-config.js +72 -0
- package/src/fact-checker/checker.js +384 -0
- package/src/fact-checker/review-gate.js +306 -0
- package/src/fact-checker/validation-set.js +1078 -0
- package/src/mcp-server/server.js +1299 -0
- package/src/mcp-server/stdio.js +87 -0
- package/src/pipeline/artifact-manifest.js +323 -0
- package/src/pipeline/artifact-retention.js +189 -0
- package/src/pipeline/external-adapter.js +366 -0
- package/src/pipeline/providers.js +853 -0
- package/src/pipeline/retry-policy.js +134 -0
- package/src/pipeline/state-machine.js +297 -0
- package/src/pipeline/telemetry.js +162 -0
- package/src/quality-gate/gate.js +229 -0
- package/src/schema/scene.schema.json +101 -0
- package/src/shorts-extractor/extractor.js +343 -0
- package/src/utils/log-redaction.js +274 -0
- package/src/video-renderer/renderer.js +364 -0
- package/src/worker/server-render-free.js +420 -0
- package/src/worker/server.js +328 -0
- package/src/workflows/assets.js +131 -0
- package/src/workflows/ingest.js +299 -0
- package/src/workflows/packaging.js +156 -0
- package/src/workflows/publishing.js +206 -0
- package/src/workflows/render.js +102 -0
- package/src/workflows/research.js +151 -0
- package/src/workflows/script.js +210 -0
- package/src/workflows/shorts.js +113 -0
|
@@ -0,0 +1,1299 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FrameForge MCP Server — MCP 2024-11-05 spec compliant
|
|
3
|
+
*
|
|
4
|
+
* Exposes 14 video production tools via MCP.
|
|
5
|
+
* Works with IBM Bob, Cursor, Windsurf, Codex, opencode, or any MCP-capable IDE.
|
|
6
|
+
* Primary connection: Cloudflare Remote MCP (cloudflare-mcp/).
|
|
7
|
+
* Alternative: stdio (src/mcp-server/stdio.js) or HTTP (this file).
|
|
8
|
+
*
|
|
9
|
+
* Primary path for agentic IDEs:
|
|
10
|
+
* create_video_from_script — IDE writes scenes, FrameForge renders (no internal LLM)
|
|
11
|
+
* multi_voice_tts — per-call TTS platform override
|
|
12
|
+
*
|
|
13
|
+
* Full pipeline path (FrameForge does the research + scripting):
|
|
14
|
+
* research_topic, write_script, generate_scene_image, render_tts,
|
|
15
|
+
* assemble_video, publish_video
|
|
16
|
+
*
|
|
17
|
+
* Edit path:
|
|
18
|
+
* ingest_media, edit_scenes, render_from_scenes, external_platform_call
|
|
19
|
+
*
|
|
20
|
+
* Start: node src/mcp-server/server.js (or: npm start)
|
|
21
|
+
* Port: MCP_PORT env (default 3210)
|
|
22
|
+
*
|
|
23
|
+
* Env (optional — only needed for pipeline-internal LLM calls):
|
|
24
|
+
* PROVIDER_LLM=groq|cerebras|openrouter|anthropic|gemini|mistral|together|openai|ollama|webhook
|
|
25
|
+
* PROVIDER_TTS=edge-tts|elevenlabs|azure|openai|webhook
|
|
26
|
+
* PROVIDER_IMAGE=pollinations|falai|replicate|dalle|webhook
|
|
27
|
+
* MCP_PORT=3210
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
'use strict';
|
|
31
|
+
|
|
32
|
+
const http = require('http');
|
|
33
|
+
const path = require('path');
|
|
34
|
+
const os = require('os');
|
|
35
|
+
|
|
36
|
+
const fs = require('fs').promises;
|
|
37
|
+
const crypto = require('crypto');
|
|
38
|
+
|
|
39
|
+
const { LLMProvider, TTSProvider, ImageProvider } = require('../pipeline/providers');
|
|
40
|
+
const { StateMachine } = require('../pipeline/state-machine');
|
|
41
|
+
const { RetryPolicy } = require('../pipeline/retry-policy');
|
|
42
|
+
const { Telemetry } = require('../pipeline/telemetry');
|
|
43
|
+
const { VideoRenderer } = require('../video-renderer/renderer');
|
|
44
|
+
const { AudioMixer } = require('../audio-mixer/mixer');
|
|
45
|
+
const { QualityGate } = require('../quality-gate/gate');
|
|
46
|
+
const { runPublishing } = require('../workflows/publishing');
|
|
47
|
+
|
|
48
|
+
const PORT = parseInt(process.env.MCP_PORT || '3210', 10);
|
|
49
|
+
const FILES_DIR = process.env.N8N_FILES_DIR || path.join(os.homedir(), '.frameforge-files');
|
|
50
|
+
const STATE_DIR = path.join(FILES_DIR, 'state');
|
|
51
|
+
const USAGE_LOG = path.join(FILES_DIR, 'usage.ndjson');
|
|
52
|
+
|
|
53
|
+
// Singletons
|
|
54
|
+
const llm = new LLMProvider();
|
|
55
|
+
const tts = new TTSProvider();
|
|
56
|
+
const imageGen = new ImageProvider();
|
|
57
|
+
const sm = new StateMachine({ mode: 'file', stateDir: STATE_DIR });
|
|
58
|
+
const retryPol = new RetryPolicy({ maxAttempts: 3, baseDelayMs: 10000 });
|
|
59
|
+
const telemetry = new Telemetry({ mode: 'file', logPath: USAGE_LOG });
|
|
60
|
+
|
|
61
|
+
// ── Tool definitions ──────────────────────────────────────────────────────
|
|
62
|
+
|
|
63
|
+
const TOOLS = {
|
|
64
|
+
|
|
65
|
+
// ── Tool 1: Research ────────────────────────────────────────────────────
|
|
66
|
+
// Delegates to runResearch() which is fully generic — no hardcoded domain.
|
|
67
|
+
research_topic: {
|
|
68
|
+
description: 'Research any topic with fact extraction. Returns verified claims ready for script generation. Works for any content type: tech, cooking, travel, history, tutorials, etc.',
|
|
69
|
+
inputSchema: {
|
|
70
|
+
type: 'object',
|
|
71
|
+
required: ['topic'],
|
|
72
|
+
properties: {
|
|
73
|
+
topic: { type: 'string', description: 'Topic to research' },
|
|
74
|
+
claimCount: { type: 'number', default: 10, description: 'Number of claims to extract' },
|
|
75
|
+
persona: { type: 'string', default: 'video content researcher', description: 'Researcher persona, e.g. "tech journalist", "travel blogger"' },
|
|
76
|
+
format: { type: 'string', default: 'video', description: 'e.g. "short-form reel", "long YouTube video", "explainer"' },
|
|
77
|
+
style: { type: 'string', default: 'informative', description: 'e.g. "educational", "comedic", "documentary"' },
|
|
78
|
+
audience: { type: 'string', default: 'general public', description: 'Target audience' },
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
handler: async ({ topic, claimCount = 10, persona, format, style, audience }) => {
|
|
82
|
+
const { runResearch } = require('../workflows/research');
|
|
83
|
+
return runResearch(topic, { claimCount, persona, format, style, audience });
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
|
|
87
|
+
// ── Tool 2: Script ───────────────────────────────────────────────────────
|
|
88
|
+
// Delegates to runScript() which is fully generic — no hardcoded characters or domain.
|
|
89
|
+
write_script: {
|
|
90
|
+
description: 'Write a video script from research claims. Generic — works for any topic, style, language, or format. Returns structured scene JSON.',
|
|
91
|
+
inputSchema: {
|
|
92
|
+
type: 'object',
|
|
93
|
+
required: ['productionId', 'claims', 'topic'],
|
|
94
|
+
properties: {
|
|
95
|
+
productionId: { type: 'string' },
|
|
96
|
+
claims: { type: 'array' },
|
|
97
|
+
topic: { type: 'string' },
|
|
98
|
+
style: { type: 'string', default: 'informative', description: 'e.g. "educational", "comedic", "documentary", "tutorial", "vlog"' },
|
|
99
|
+
persona: { type: 'string', default: 'video host', description: 'Who the narrator is, e.g. "travel blogger", "tech reviewer"' },
|
|
100
|
+
format: { type: 'string', default: 'YouTube video', description: 'e.g. "YouTube short", "Instagram reel", "long-form tutorial"' },
|
|
101
|
+
language: { type: 'string', default: 'en', description: 'BCP-47 language code for the script' },
|
|
102
|
+
sceneCount: { type: 'number', default: 55, description: 'Target number of scenes' },
|
|
103
|
+
targetDuration: { type: 'number', description: 'Target video duration in seconds (overrides sceneCount)' },
|
|
104
|
+
imageStyle: { type: 'string', default: 'clean flat illustration', description: 'Visual aesthetic for AI image prompts, e.g. "photorealistic", "anime", "watercolour"' },
|
|
105
|
+
},
|
|
106
|
+
},
|
|
107
|
+
handler: async ({ productionId, claims, topic, style, persona, format, language, sceneCount, targetDuration, imageStyle }) => {
|
|
108
|
+
const { runScript } = require('../workflows/script');
|
|
109
|
+
const result = await runScript({ productionId, claims, topic, style, persona, format, language, sceneCount, targetDuration, imageStyle });
|
|
110
|
+
return {
|
|
111
|
+
productionId: result.productionId,
|
|
112
|
+
scenes: result.scenes,
|
|
113
|
+
shorts: result.shorts,
|
|
114
|
+
title: result.title,
|
|
115
|
+
description: result.description,
|
|
116
|
+
tags: result.tags,
|
|
117
|
+
thumbnailPrompt: result.thumbnailPrompt,
|
|
118
|
+
totalWords: result.totalWords,
|
|
119
|
+
sceneCount: result.sceneCount,
|
|
120
|
+
};
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
|
|
124
|
+
// ── Tool 3: Generate scene image ─────────────────────────────────────────
|
|
125
|
+
generate_scene_image: {
|
|
126
|
+
description: 'Generate an image for one scene using the configured image provider. Pass imageStyle to control the visual aesthetic (default: clean flat illustration). No hardcoded art style.',
|
|
127
|
+
inputSchema: {
|
|
128
|
+
type: 'object',
|
|
129
|
+
required: ['productionId', 'sceneIndex', 'visualDescription'],
|
|
130
|
+
properties: {
|
|
131
|
+
productionId: { type: 'string' },
|
|
132
|
+
sceneIndex: { type: 'number' },
|
|
133
|
+
visualDescription:{ type: 'string', description: 'Scene visual description — used directly as image prompt' },
|
|
134
|
+
imageStyle: { type: 'string', description: 'Visual aesthetic, e.g. "flat design", "photorealistic", "anime", "watercolour sketch". Default: clean flat illustration.' },
|
|
135
|
+
negativePrompt: { type: 'string', description: 'What to exclude from the image (optional)' },
|
|
136
|
+
seed: { type: 'number', default: 12345 },
|
|
137
|
+
},
|
|
138
|
+
},
|
|
139
|
+
handler: async ({
|
|
140
|
+
productionId,
|
|
141
|
+
sceneIndex,
|
|
142
|
+
visualDescription,
|
|
143
|
+
// Accept legacy 'visualAction' param name for backwards compat
|
|
144
|
+
visualAction,
|
|
145
|
+
imageStyle = 'clean flat illustration, white background, bold lines',
|
|
146
|
+
negativePrompt = 'watermark, text, logo, blurry, low quality',
|
|
147
|
+
seed = 12345,
|
|
148
|
+
}) => {
|
|
149
|
+
const vdir = path.join(FILES_DIR, 'video_tmp');
|
|
150
|
+
const outputPath = path.join(vdir, `scene_${sceneIndex}.png`);
|
|
151
|
+
await fs.mkdir(vdir, { recursive: true });
|
|
152
|
+
|
|
153
|
+
const description = visualDescription || visualAction || '';
|
|
154
|
+
const prompt = `${description.trim()}, ${imageStyle}`;
|
|
155
|
+
|
|
156
|
+
await retryPol.executeStage('Asset_Generation', async () => {
|
|
157
|
+
return imageGen.generate({ prompt, negativePrompt, outputPath, width: 1920, height: 1080, seed: seed + sceneIndex * 37 });
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
const stat = await fs.stat(outputPath);
|
|
161
|
+
await telemetry.track({ productionId, stage: 'Asset_Generation', provider: process.env.PROVIDER_IMAGE || 'pollinations', operation: 'scene_image_generation', units: 1 });
|
|
162
|
+
|
|
163
|
+
return { productionId, sceneIndex, imagePath: outputPath, sizeBytes: stat.size };
|
|
164
|
+
},
|
|
165
|
+
},
|
|
166
|
+
|
|
167
|
+
// ── Tool 4: TTS ──────────────────────────────────────────────────────────
|
|
168
|
+
render_tts: {
|
|
169
|
+
description: 'Convert narration text to audio using the configured TTS provider. Speech rate is controlled via TTS_RATE env var (default: -5%).',
|
|
170
|
+
inputSchema: {
|
|
171
|
+
type: 'object',
|
|
172
|
+
required: ['productionId', 'sceneIndex', 'text'],
|
|
173
|
+
properties: {
|
|
174
|
+
productionId: { type: 'string' },
|
|
175
|
+
sceneIndex: { type: 'number' },
|
|
176
|
+
text: { type: 'string' },
|
|
177
|
+
voice: { type: 'string', description: 'Voice ID (e.g. en-US-GuyNeural). Overrides TTS_VOICE env var.' },
|
|
178
|
+
},
|
|
179
|
+
},
|
|
180
|
+
handler: async ({ productionId, sceneIndex, text, voice }) => {
|
|
181
|
+
const vdir = path.join(FILES_DIR, 'video_tmp');
|
|
182
|
+
const outputPath = path.join(vdir, `voice_${sceneIndex}.m4a`);
|
|
183
|
+
await fs.mkdir(vdir, { recursive: true });
|
|
184
|
+
|
|
185
|
+
await retryPol.executeStage('TTS', async () => {
|
|
186
|
+
// voice arg overrides env; rate is read from TTS_RATE env inside synthesize()
|
|
187
|
+
return tts.synthesize({ text, voice, outputPath });
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
const stat = await fs.stat(outputPath);
|
|
191
|
+
await telemetry.track({ productionId, stage: 'TTS', provider: process.env.PROVIDER_TTS || 'edge-tts', operation: 'tts_synthesis', units: text.length });
|
|
192
|
+
|
|
193
|
+
return { productionId, sceneIndex, audioPath: outputPath, sizeBytes: stat.size };
|
|
194
|
+
},
|
|
195
|
+
},
|
|
196
|
+
|
|
197
|
+
// ── Tool 5: Assemble video ────────────────────────────────────────────────
|
|
198
|
+
assemble_video: {
|
|
199
|
+
description: 'Assemble rendered scenes into a final MP4. Runs quality gate before returning.',
|
|
200
|
+
inputSchema: {
|
|
201
|
+
type: 'object',
|
|
202
|
+
required: ['productionId'],
|
|
203
|
+
properties: {
|
|
204
|
+
productionId: { type: 'string' },
|
|
205
|
+
sceneCount: { type: 'number', description: 'Number of scenes to assemble' },
|
|
206
|
+
withMusic: { type: 'boolean', default: true },
|
|
207
|
+
musicVolume: { type: 'number', default: 0.07 },
|
|
208
|
+
},
|
|
209
|
+
},
|
|
210
|
+
handler: async ({ productionId, sceneCount, withMusic = true, musicVolume = 0.07 }) => {
|
|
211
|
+
await sm.startStage(productionId, 'Video_Rendering');
|
|
212
|
+
const renderer = new VideoRenderer({ filesDir: FILES_DIR });
|
|
213
|
+
const mixer = new AudioMixer({ filesDir: FILES_DIR });
|
|
214
|
+
const qc = new QualityGate();
|
|
215
|
+
|
|
216
|
+
const masterPath = await retryPol.executeStage('Video_Rendering', async () => {
|
|
217
|
+
return renderer.assembleMain({ productionId, sceneCount, withMusic, musicVolume });
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
const qcReport = await qc.validateMaster(masterPath);
|
|
221
|
+
if (!qcReport.passed) {
|
|
222
|
+
await sm.failStage(productionId, 'Video_Rendering', `QC failed: ${qcReport.checks.filter(c => !c.passed).map(c => c.name).join(', ')}`);
|
|
223
|
+
return { productionId, passed: false, qcReport };
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const hash = crypto.createHash('sha256');
|
|
227
|
+
const stat = await fs.stat(masterPath);
|
|
228
|
+
await sm.completeStage(productionId, 'Video_Rendering', { outputHash: hash.update(`${productionId}:${stat.size}`).digest('hex').slice(0,16) });
|
|
229
|
+
await telemetry.track({ productionId, stage: 'Video_Rendering', provider: 'ffmpeg', operation: 'master_render', units: Math.round(stat.size / 1024 / 1024) });
|
|
230
|
+
|
|
231
|
+
return { productionId, masterPath, passed: true, qcReport, sizeBytes: stat.size };
|
|
232
|
+
},
|
|
233
|
+
},
|
|
234
|
+
|
|
235
|
+
// ── Tool 6: Publish video ─────────────────────────────────────────────────
|
|
236
|
+
publish_video: {
|
|
237
|
+
description: 'Deliver finished video via file copy or HTTP webhook. Channels: "file" (copy to published/) and/or "webhook" (POST JSON summary to PUBLISH_WEBHOOK_URL).',
|
|
238
|
+
inputSchema: {
|
|
239
|
+
type: 'object',
|
|
240
|
+
required: ['productionId', 'masterPath'],
|
|
241
|
+
properties: {
|
|
242
|
+
productionId: { type: 'string' },
|
|
243
|
+
masterPath: { type: 'string' },
|
|
244
|
+
channels: { type: 'array', items: { type: 'string' }, default: ['file'], description: 'Delivery channels: file | webhook' },
|
|
245
|
+
title: { type: 'string', description: 'Video title for metadata and webhook payload' },
|
|
246
|
+
description: { type: 'string', description: 'Video description for metadata and webhook payload' },
|
|
247
|
+
},
|
|
248
|
+
},
|
|
249
|
+
handler: async ({ productionId, masterPath, channels = ['file'], title = '', description = '' }) => {
|
|
250
|
+
await sm.startStage(productionId, 'Publishing');
|
|
251
|
+
|
|
252
|
+
const manifest = {
|
|
253
|
+
master: { path: masterPath },
|
|
254
|
+
metadata: { title, description },
|
|
255
|
+
shorts: [],
|
|
256
|
+
publishDecision: 'AUTO_PUBLISH',
|
|
257
|
+
};
|
|
258
|
+
const pubResult = await runPublishing({ productionId, manifest, channels });
|
|
259
|
+
const results = pubResult.results || {};
|
|
260
|
+
|
|
261
|
+
await sm.completeStage(productionId, 'Publishing', {});
|
|
262
|
+
await sm.transitionProduction(productionId, 'SUCCEEDED').catch(() => {});
|
|
263
|
+
await telemetry.track({ productionId, stage: 'Publishing', provider: 'multi', operation: 'video_delivery', units: 1 });
|
|
264
|
+
|
|
265
|
+
return { productionId, results };
|
|
266
|
+
},
|
|
267
|
+
},
|
|
268
|
+
|
|
269
|
+
// ── Tool 7: Ingest media (edit-only entry point) ──────────────────────────
|
|
270
|
+
// Lets an IDE agent supply video clips, images, and audio without any LLM.
|
|
271
|
+
// Returns a scenes manifest JSON ready for render_from_scenes.
|
|
272
|
+
ingest_media: {
|
|
273
|
+
description: 'Ingest user-supplied video clips, images, and/or audio files into a scenes manifest. No AI generation — edit-only entry point. Returns { productionId, scenesFile, sceneCount }.',
|
|
274
|
+
inputSchema: {
|
|
275
|
+
type: 'object',
|
|
276
|
+
properties: {
|
|
277
|
+
scenes: { type: 'array', description: 'Inline scene array. Each scene: { narration, caption, video_path, image_path, audio_path, camera_hint, music_mood }' },
|
|
278
|
+
scenesFile: { type: 'string', description: 'Path to a scenes.json file on disk' },
|
|
279
|
+
mediaDir: { type: 'string', description: 'Directory to auto-discover clips from' },
|
|
280
|
+
productionId:{ type: 'string', description: 'Reuse an existing productionId (optional)' },
|
|
281
|
+
},
|
|
282
|
+
},
|
|
283
|
+
handler: async ({ scenes, scenesFile, mediaDir, productionId } = {}) => {
|
|
284
|
+
const { runIngest } = require('../workflows/ingest');
|
|
285
|
+
const result = await runIngest({
|
|
286
|
+
productionId,
|
|
287
|
+
rawScenes: scenes,
|
|
288
|
+
scenesFile,
|
|
289
|
+
mediaDir,
|
|
290
|
+
outDir: FILES_DIR,
|
|
291
|
+
});
|
|
292
|
+
return { productionId: result.productionId, scenesFile: result.scenesFile, sceneCount: result.sceneCount, stats: result.stats };
|
|
293
|
+
},
|
|
294
|
+
},
|
|
295
|
+
|
|
296
|
+
// ── Tool 8: Edit scenes (modify a scenes manifest) ────────────────────────
|
|
297
|
+
// Lets an IDE agent read, modify, or annotate an existing scenes manifest.
|
|
298
|
+
edit_scenes: {
|
|
299
|
+
description: 'Read and optionally modify an existing scenes manifest. Use to add/update narration, captions, camera hints, or swap media files before rendering.',
|
|
300
|
+
inputSchema: {
|
|
301
|
+
type: 'object',
|
|
302
|
+
required: ['productionId'],
|
|
303
|
+
properties: {
|
|
304
|
+
productionId: { type: 'string' },
|
|
305
|
+
edits: { type: 'array', description: 'Array of { sceneIndex: number, field: string, value: any } patches to apply' },
|
|
306
|
+
appendScenes: { type: 'array', description: 'New scenes to append to the manifest' },
|
|
307
|
+
},
|
|
308
|
+
},
|
|
309
|
+
handler: async ({ productionId, edits = [], appendScenes = [] }) => {
|
|
310
|
+
const fsp = require('fs').promises;
|
|
311
|
+
const syncFs = require('fs');
|
|
312
|
+
// Guard: productionId must not contain path separators (prevents path traversal)
|
|
313
|
+
if (!productionId || /[/\\]/.test(productionId)) {
|
|
314
|
+
throw new Error(`Invalid productionId: must not contain path separators`);
|
|
315
|
+
}
|
|
316
|
+
// Find the scenes manifest for this productionId
|
|
317
|
+
const scenesPath = path.join(FILES_DIR, `${productionId}-scenes.json`);
|
|
318
|
+
if (!syncFs.existsSync(scenesPath)) {
|
|
319
|
+
throw new Error(`No scenes manifest found for productionId: ${productionId}. Run ingest_media first.`);
|
|
320
|
+
}
|
|
321
|
+
const manifest = JSON.parse(syncFs.readFileSync(scenesPath, 'utf8'));
|
|
322
|
+
const scenes = manifest.scenes || [];
|
|
323
|
+
|
|
324
|
+
// Apply field-level edits
|
|
325
|
+
for (const { sceneIndex, field, value } of edits) {
|
|
326
|
+
if (sceneIndex >= 0 && sceneIndex < scenes.length && field) {
|
|
327
|
+
scenes[sceneIndex][field] = value;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// Append new scenes
|
|
332
|
+
for (const s of appendScenes) {
|
|
333
|
+
scenes.push({ scene_id: scenes.length + 1, schema_version: '1.0.0', ...s });
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
manifest.scenes = scenes;
|
|
337
|
+
manifest.sceneCount = scenes.length;
|
|
338
|
+
manifest.updatedAt = new Date().toISOString();
|
|
339
|
+
|
|
340
|
+
const tmpPath = scenesPath + '.tmp';
|
|
341
|
+
syncFs.writeFileSync(tmpPath, JSON.stringify(manifest, null, 2), 'utf8');
|
|
342
|
+
syncFs.renameSync(tmpPath, scenesPath);
|
|
343
|
+
|
|
344
|
+
return { productionId, scenesFile: scenesPath, sceneCount: scenes.length, message: `Applied ${edits.length} edit(s), appended ${appendScenes.length} scene(s)` };
|
|
345
|
+
},
|
|
346
|
+
},
|
|
347
|
+
|
|
348
|
+
// ── Tool 9: Render from scenes (edit-only render path) ────────────────────
|
|
349
|
+
// Renders a scenes manifest directly — skipping research and script stages.
|
|
350
|
+
render_from_scenes: {
|
|
351
|
+
description: 'Render a video directly from a scenes manifest, skipping research and script generation. Supply a productionId from ingest_media, or pass scenes inline.',
|
|
352
|
+
inputSchema: {
|
|
353
|
+
type: 'object',
|
|
354
|
+
properties: {
|
|
355
|
+
productionId: { type: 'string', description: 'From ingest_media / edit_scenes' },
|
|
356
|
+
scenes: { type: 'array', description: 'Inline scenes array (alternative to productionId)' },
|
|
357
|
+
orientation: { type: 'string', enum: ['landscape', 'portrait'], default: 'landscape', description: 'Output orientation. "landscape" = 1920×1080 (16:9). "portrait" = 1080×1920 (9:16, TikTok/Reels/Shorts).' },
|
|
358
|
+
skipImages: { type: 'boolean', default: false, description: 'Skip AI image generation (use video_path / image_path in scenes)' },
|
|
359
|
+
skipTts: { type: 'boolean', default: false, description: 'Skip TTS synthesis (use audio_path in scenes)' },
|
|
360
|
+
withMusic: { type: 'boolean', default: true },
|
|
361
|
+
musicVolume: { type: 'number', default: 0.07 },
|
|
362
|
+
},
|
|
363
|
+
},
|
|
364
|
+
handler: async ({ productionId, scenes, orientation = 'landscape', skipImages = false, skipTts = false, withMusic = true, musicVolume = 0.07 }) => {
|
|
365
|
+
const syncFs = require('fs');
|
|
366
|
+
const { runAssets } = require('../workflows/assets');
|
|
367
|
+
const { runRender } = require('../workflows/render');
|
|
368
|
+
const { runIngest } = require('../workflows/ingest');
|
|
369
|
+
|
|
370
|
+
// Guard: productionId must not contain path separators (prevents path traversal)
|
|
371
|
+
if (productionId && /[/\\]/.test(productionId)) {
|
|
372
|
+
throw new Error(`Invalid productionId: must not contain path separators`);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// Resolve scenes
|
|
376
|
+
let resolvedScenes = scenes;
|
|
377
|
+
let resolvedPid = productionId;
|
|
378
|
+
|
|
379
|
+
if (!resolvedScenes && productionId) {
|
|
380
|
+
const scenesPath = path.join(FILES_DIR, `${productionId}-scenes.json`);
|
|
381
|
+
if (!syncFs.existsSync(scenesPath)) throw new Error(`No scenes manifest for: ${productionId}. Run ingest_media first.`);
|
|
382
|
+
const m = JSON.parse(syncFs.readFileSync(scenesPath, 'utf8'));
|
|
383
|
+
resolvedScenes = m.scenes;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
if (!resolvedScenes || resolvedScenes.length === 0) {
|
|
387
|
+
throw new Error('No scenes to render. Provide productionId or inline scenes.');
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
if (!resolvedPid) {
|
|
391
|
+
const crypto = require('crypto');
|
|
392
|
+
resolvedPid = `render_${crypto.randomBytes(4).toString('hex')}_${Date.now()}`;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const renderWidth = orientation === 'portrait' ? 1080 : 1920;
|
|
396
|
+
const renderHeight = orientation === 'portrait' ? 1920 : 1080;
|
|
397
|
+
|
|
398
|
+
// Run assets stage (respecting skip flags)
|
|
399
|
+
const assetsResult = await runAssets({
|
|
400
|
+
productionId: resolvedPid,
|
|
401
|
+
scenes: resolvedScenes,
|
|
402
|
+
parallelism: 4,
|
|
403
|
+
skipImages,
|
|
404
|
+
skipTts,
|
|
405
|
+
imageWidth: renderWidth,
|
|
406
|
+
imageHeight: renderHeight,
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
// Run render stage
|
|
410
|
+
const renderResult = await runRender({
|
|
411
|
+
productionId: resolvedPid,
|
|
412
|
+
sceneAssets: assetsResult.sceneAssets,
|
|
413
|
+
withMusic,
|
|
414
|
+
musicVolume,
|
|
415
|
+
renderWidth,
|
|
416
|
+
renderHeight,
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
if (!renderResult.passed) {
|
|
420
|
+
return { productionId: resolvedPid, passed: false, qcReport: renderResult.qcReport };
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
return {
|
|
424
|
+
productionId: resolvedPid,
|
|
425
|
+
masterPath: renderResult.masterPath,
|
|
426
|
+
passed: true,
|
|
427
|
+
sizeBytes: renderResult.sizeBytes,
|
|
428
|
+
sceneCount: renderResult.sceneCount,
|
|
429
|
+
qcReport: renderResult.qcReport,
|
|
430
|
+
};
|
|
431
|
+
},
|
|
432
|
+
},
|
|
433
|
+
|
|
434
|
+
// ── Tool 10: External platform call ────────────────────────────────────────
|
|
435
|
+
// Bridge to VoxForge, Runway, ElevenLabs, Remotion, or any external platform.
|
|
436
|
+
external_platform_call: {
|
|
437
|
+
description: 'Send media or scenes to an external platform (VoxForge, Runway, ElevenLabs, Remotion, etc.). Configured via EXTERNAL_PLATFORM_* env vars or setup-keys wizard.',
|
|
438
|
+
inputSchema: {
|
|
439
|
+
type: 'object',
|
|
440
|
+
required: ['action'],
|
|
441
|
+
properties: {
|
|
442
|
+
action: { type: 'string', enum: ['ingest','render','status','download'], description: 'ingest: upload media files | render: send scenes for rendering | status: poll job | download: fetch output' },
|
|
443
|
+
files: { type: 'array', items: { type: 'string' }, description: 'Local file paths (for action=ingest)' },
|
|
444
|
+
scenes: { type: 'array', description: 'Scene array (for action=render)' },
|
|
445
|
+
productionId:{ type: 'string', description: 'productionId (for action=render)' },
|
|
446
|
+
jobId: { type: 'string', description: 'Job ID to poll (for action=status)' },
|
|
447
|
+
outputUrl: { type: 'string', description: 'URL to download (for action=download)' },
|
|
448
|
+
outputPath: { type: 'string', description: 'Local path to save download (for action=download)' },
|
|
449
|
+
metadata: { type: 'object', description: 'Extra fields forwarded to the platform' },
|
|
450
|
+
},
|
|
451
|
+
},
|
|
452
|
+
handler: async ({ action, files, scenes, productionId, jobId, outputUrl, outputPath, metadata }) => {
|
|
453
|
+
const { ExternalAdapter } = require('../pipeline/external-adapter');
|
|
454
|
+
const adapter = new ExternalAdapter();
|
|
455
|
+
if (!adapter.isConfigured()) {
|
|
456
|
+
throw new Error('External platform not configured. Run: node src/cli/setup-keys.js to set EXTERNAL_PLATFORM_URL and EXTERNAL_PLATFORM_KEY.');
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
switch (action) {
|
|
460
|
+
case 'ingest':
|
|
461
|
+
return adapter.ingestMedia({ files: files || [], metadata: metadata || {} });
|
|
462
|
+
case 'render':
|
|
463
|
+
return adapter.renderScenes({ scenes: scenes || [], productionId, options: metadata || {} });
|
|
464
|
+
case 'status':
|
|
465
|
+
if (!jobId) throw new Error('action=status requires jobId');
|
|
466
|
+
return adapter.checkStatus(jobId);
|
|
467
|
+
case 'download':
|
|
468
|
+
if (!outputUrl || !outputPath) throw new Error('action=download requires outputUrl and outputPath');
|
|
469
|
+
await adapter.downloadOutput(outputUrl, outputPath);
|
|
470
|
+
return { outputPath, downloaded: true };
|
|
471
|
+
default:
|
|
472
|
+
throw new Error(`Unknown action: ${action}. Use ingest|render|status|download`);
|
|
473
|
+
}
|
|
474
|
+
},
|
|
475
|
+
},
|
|
476
|
+
|
|
477
|
+
// ── Tool 11: Agent-first video creation (IDE writes the script, FF renders) ──
|
|
478
|
+
// Primary path for IBM Bob / Claude Code / Cursor users.
|
|
479
|
+
// The IDE agent writes the scenes array in its own context — no internal LLM needed.
|
|
480
|
+
create_video_from_script: {
|
|
481
|
+
description: `Agent-first video creation. YOU (the IDE agent) provide the complete scenes array — FrameForge just renders it. No internal LLM, no research stage. This is the primary path for Claude Code, IBM Bob, Cursor, and any agentic IDE.
|
|
482
|
+
|
|
483
|
+
Example scene:
|
|
484
|
+
{
|
|
485
|
+
"narration": "The text spoken aloud in this scene",
|
|
486
|
+
"caption": "OVERLAY TEXT",
|
|
487
|
+
"visual_description": "A description used as the AI image prompt",
|
|
488
|
+
"camera_hint": "zoom_in",
|
|
489
|
+
"music_mood": "energetic",
|
|
490
|
+
"image_path": "/optional/your/own/image.jpg",
|
|
491
|
+
"audio_path": "/optional/your/own/voice.mp3"
|
|
492
|
+
}`,
|
|
493
|
+
inputSchema: {
|
|
494
|
+
type: 'object',
|
|
495
|
+
required: ['scenes'],
|
|
496
|
+
properties: {
|
|
497
|
+
scenes: { type: 'array', description: 'Complete scenes array written by the agent. Each scene: { narration, caption, visual_description, camera_hint?, music_mood?, image_path?, video_path?, audio_path?, voice? }' },
|
|
498
|
+
title: { type: 'string', description: 'Video title' },
|
|
499
|
+
orientation: { type: 'string', enum: ['landscape', 'portrait'], default: 'landscape', description: 'Output orientation. "landscape" = 1920×1080 (16:9, YouTube/desktop). "portrait" = 1080×1920 (9:16, TikTok/Reels/Shorts).' },
|
|
500
|
+
voice: { type: 'string', description: 'edge-tts voice ID for all scenes, e.g. "ml-IN-MidhunNeural" (Malayalam male), "en-US-BrianNeural" (English). Per-scene voice field overrides this. Full list: edge-tts --list-voices' },
|
|
501
|
+
language: { type: 'string', description: 'BCP-47 language hint, e.g. "ml" (Malayalam), "hi" (Hindi), "ta" (Tamil), "te" (Telugu), "en" (English). Auto-selects a default voice for the language when voice is not set.' },
|
|
502
|
+
skipImages: { type: 'boolean', default: false, description: 'Set true if all scenes already have image_path or video_path' },
|
|
503
|
+
skipTts: { type: 'boolean', default: false, description: 'Set true if all scenes already have audio_path' },
|
|
504
|
+
imageStyle: { type: 'string', description: 'Visual aesthetic for AI-generated images, e.g. "photorealistic", "anime", "flat design". Only used when skipImages is false.' },
|
|
505
|
+
parallelism: { type: 'number', default: 1, description: 'Concurrent image generation requests (default: 1). Increase to 2-4 only when using a reliable image provider like falai or replicate. Keep at 1 for Pollinations to avoid rate limits.' },
|
|
506
|
+
withMusic: { type: 'boolean', default: true },
|
|
507
|
+
musicVolume: { type: 'number', default: 0.07 },
|
|
508
|
+
productionId: { type: 'string', description: 'Optional — reuse an existing productionId' },
|
|
509
|
+
},
|
|
510
|
+
},
|
|
511
|
+
handler: async ({ scenes, title = 'FrameForge Video', orientation = 'landscape', voice, language, skipImages = false, skipTts = false, imageStyle, parallelism = 1, withMusic = true, musicVolume = 0.07, productionId }) => {
|
|
512
|
+
const { runAssets } = require('../workflows/assets');
|
|
513
|
+
const { runRender } = require('../workflows/render');
|
|
514
|
+
|
|
515
|
+
if (!Array.isArray(scenes) || scenes.length === 0) {
|
|
516
|
+
throw new Error('scenes array is required and must not be empty. You (the IDE agent) write the scenes.');
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
// Auto-select voice from language hint when voice is not explicitly provided.
|
|
520
|
+
// Maps BCP-47 language prefix → default edge-tts neural voice.
|
|
521
|
+
const LANGUAGE_VOICES = {
|
|
522
|
+
'ml': 'ml-IN-MidhunNeural', // Malayalam male
|
|
523
|
+
'hi': 'hi-IN-MadhurNeural', // Hindi male
|
|
524
|
+
'ta': 'ta-IN-ValluvarNeural', // Tamil male
|
|
525
|
+
'te': 'te-IN-MohanNeural', // Telugu male
|
|
526
|
+
'kn': 'kn-IN-GaganNeural', // Kannada male
|
|
527
|
+
'bn': 'bn-IN-BashkarNeural', // Bengali male
|
|
528
|
+
'mr': 'mr-IN-ManoharNeural', // Marathi male
|
|
529
|
+
'gu': 'gu-IN-NiranjanNeural', // Gujarati male
|
|
530
|
+
'pa': 'pa-IN-OjasvNeural', // Punjabi male
|
|
531
|
+
'ur': 'ur-PK-AsadNeural', // Urdu male
|
|
532
|
+
'zh': 'zh-CN-YunxiNeural', // Chinese male
|
|
533
|
+
'ja': 'ja-JP-KeitaNeural', // Japanese male
|
|
534
|
+
'ko': 'ko-KR-InJoonNeural', // Korean male
|
|
535
|
+
'ar': 'ar-SA-HamedNeural', // Arabic male
|
|
536
|
+
'fr': 'fr-FR-HenriNeural', // French male
|
|
537
|
+
'de': 'de-DE-ConradNeural', // German male
|
|
538
|
+
'es': 'es-ES-AlvaroNeural', // Spanish male
|
|
539
|
+
'pt': 'pt-BR-AntonioNeural', // Portuguese male
|
|
540
|
+
'it': 'it-IT-DiegoNeural', // Italian male
|
|
541
|
+
'ru': 'ru-RU-DmitryNeural', // Russian male
|
|
542
|
+
};
|
|
543
|
+
const resolvedVoice = voice || (language ? LANGUAGE_VOICES[language.toLowerCase().slice(0, 2)] : null) || null;
|
|
544
|
+
|
|
545
|
+
// Apply imageStyle to visual_description if provided and skipImages is false
|
|
546
|
+
const processedScenes = scenes.map((s, i) => ({
|
|
547
|
+
scene_id: s.scene_id || i + 1,
|
|
548
|
+
schema_version: '2.0.0',
|
|
549
|
+
narration: s.narration || '',
|
|
550
|
+
caption: s.caption || '',
|
|
551
|
+
visual_description: imageStyle && !s.image_path && !s.video_path
|
|
552
|
+
? `${s.visual_description || ''}, ${imageStyle}`.trim()
|
|
553
|
+
: (s.visual_description || ''),
|
|
554
|
+
camera_hint: s.camera_hint || 'auto',
|
|
555
|
+
music_mood: s.music_mood || 'neutral',
|
|
556
|
+
image_path: s.image_path || null,
|
|
557
|
+
video_path: s.video_path || null,
|
|
558
|
+
audio_path: s.audio_path || null,
|
|
559
|
+
voice: s.voice || null, // per-scene voice override
|
|
560
|
+
character: s.character || { character_id: 'none', pose: 'none', emotion: 'neutral', position: { x: 0.5, y: 0.8, scale: 1.0 } },
|
|
561
|
+
focal_point: s.focal_point || { x: 0.5, y: 0.35, weight: 0.8 },
|
|
562
|
+
overlay: s.overlay || { type: 'none' },
|
|
563
|
+
sound_effects: s.sound_effects || [],
|
|
564
|
+
metadata: s.metadata || { importance: 'medium', hook_potential: 0.5, standalone: false },
|
|
565
|
+
}));
|
|
566
|
+
|
|
567
|
+
const pid = productionId || `agent_${crypto.randomBytes(4).toString('hex')}_${Date.now()}`;
|
|
568
|
+
|
|
569
|
+
// Resolve output dimensions from orientation
|
|
570
|
+
const renderWidth = orientation === 'portrait' ? 1080 : 1920;
|
|
571
|
+
const renderHeight = orientation === 'portrait' ? 1920 : 1080;
|
|
572
|
+
|
|
573
|
+
const assetsResult = await runAssets({ productionId: pid, scenes: processedScenes, parallelism, skipImages, skipTts, imageWidth: renderWidth, imageHeight: renderHeight, voice: resolvedVoice });
|
|
574
|
+
const renderResult = await runRender({ productionId: pid, sceneAssets: assetsResult.sceneAssets, withMusic, musicVolume, renderWidth, renderHeight });
|
|
575
|
+
|
|
576
|
+
if (!renderResult.passed) {
|
|
577
|
+
return { productionId: pid, passed: false, qcReport: renderResult.qcReport };
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
return {
|
|
581
|
+
productionId: pid,
|
|
582
|
+
masterPath: renderResult.masterPath,
|
|
583
|
+
passed: true,
|
|
584
|
+
sizeBytes: renderResult.sizeBytes,
|
|
585
|
+
sceneCount: renderResult.sceneCount,
|
|
586
|
+
title,
|
|
587
|
+
message: `Video ready at: ${renderResult.masterPath}`,
|
|
588
|
+
};
|
|
589
|
+
},
|
|
590
|
+
},
|
|
591
|
+
|
|
592
|
+
// ── Tool 12: Multi-platform TTS ──────────────────────────────────────────
|
|
593
|
+
// Call any voice platform per-request without changing global .env
|
|
594
|
+
multi_voice_tts: {
|
|
595
|
+
description: 'Convert text to speech using any supported voice platform per-call. Overrides the global PROVIDER_TTS for this one request. Supports: edge-tts, elevenlabs, azure, openai, custom webhook.',
|
|
596
|
+
inputSchema: {
|
|
597
|
+
type: 'object',
|
|
598
|
+
required: ['text', 'outputPath'],
|
|
599
|
+
properties: {
|
|
600
|
+
text: { type: 'string', description: 'Text to synthesise' },
|
|
601
|
+
outputPath: { type: 'string', description: 'Where to save the audio file (.m4a or .mp3)' },
|
|
602
|
+
platform: { type: 'string', enum: ['edge-tts','elevenlabs','azure','openai','webhook'], description: 'TTS platform. Overrides PROVIDER_TTS for this call.' },
|
|
603
|
+
voice: { type: 'string', description: 'Voice ID. edge-tts: e.g. en-US-GuyNeural. ElevenLabs: voice_id. Azure: short name. OpenAI: alloy/echo/fable/nova/onyx/shimmer.' },
|
|
604
|
+
rate: { type: 'string', description: 'Speaking rate, e.g. "-10%" for slower, "+20%" for faster (edge-tts)' },
|
|
605
|
+
apiKey: { type: 'string', description: 'API key for this call (optional — falls back to env var)' },
|
|
606
|
+
webhookUrl: { type: 'string', description: 'TTS webhook URL (when platform=webhook)' },
|
|
607
|
+
productionId: { type: 'string', description: 'Optional productionId for telemetry' },
|
|
608
|
+
sceneIndex: { type: 'number', description: 'Optional scene index for naming the output file' },
|
|
609
|
+
},
|
|
610
|
+
},
|
|
611
|
+
handler: async ({ text, outputPath, platform, voice, rate, apiKey, webhookUrl, productionId = 'adhoc', sceneIndex }) => {
|
|
612
|
+
const syncFs = require('fs');
|
|
613
|
+
const pathMod = require('path');
|
|
614
|
+
|
|
615
|
+
// Guard: outputPath must not escape FILES_DIR — prevent path traversal
|
|
616
|
+
if (outputPath) {
|
|
617
|
+
const absOutput = pathMod.resolve(outputPath);
|
|
618
|
+
const absFilesDir = pathMod.resolve(FILES_DIR);
|
|
619
|
+
if (!absOutput.startsWith(absFilesDir + pathMod.sep) && !absOutput.startsWith(absFilesDir)) {
|
|
620
|
+
throw new Error(`outputPath must be inside the files directory (${FILES_DIR})`);
|
|
621
|
+
}
|
|
622
|
+
} else {
|
|
623
|
+
throw new Error('outputPath is required');
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// Auto-derive outputPath if sceneIndex given and outputPath is a directory
|
|
627
|
+
let resolvedOutput = outputPath;
|
|
628
|
+
if (sceneIndex !== undefined && syncFs.existsSync(outputPath) && syncFs.statSync(outputPath).isDirectory()) {
|
|
629
|
+
resolvedOutput = pathMod.join(outputPath, `voice_${sceneIndex}.m4a`);
|
|
630
|
+
}
|
|
631
|
+
await fs.mkdir(pathMod.dirname(resolvedOutput), { recursive: true });
|
|
632
|
+
|
|
633
|
+
// Create a TTSProvider with per-call options — no process.env mutation needed.
|
|
634
|
+
// This is concurrency-safe: each invocation has its own provider instance.
|
|
635
|
+
const { TTSProvider } = require('../pipeline/providers');
|
|
636
|
+
const ttsInstance = new TTSProvider({ platform, voice, rate, apiKey, webhookUrl });
|
|
637
|
+
|
|
638
|
+
await retryPol.executeStage('TTS', async () => {
|
|
639
|
+
return ttsInstance.synthesize({ text, voice, outputPath: resolvedOutput });
|
|
640
|
+
});
|
|
641
|
+
|
|
642
|
+
const stat = await fs.stat(resolvedOutput);
|
|
643
|
+
const resolvedPlatform = platform || process.env.PROVIDER_TTS || 'edge-tts';
|
|
644
|
+
await telemetry.track({ productionId, stage: 'TTS', provider: resolvedPlatform, operation: 'multi_voice_tts', units: text.length });
|
|
645
|
+
|
|
646
|
+
return { audioPath: resolvedOutput, sizeBytes: stat.size, platform: resolvedPlatform, voice };
|
|
647
|
+
},
|
|
648
|
+
},
|
|
649
|
+
|
|
650
|
+
// ── Tool 14: Get job status ────────────────────────────────────────────────
|
|
651
|
+
// Poll a job that was queued via queue_full_pipeline (Tool 13).
|
|
652
|
+
// Returns the current status, stage, and result when the job completes.
|
|
653
|
+
get_job_status: {
|
|
654
|
+
description: 'Poll a queued pipeline job by jobId. Returns { status:"RUNNING"|"SUCCEEDED"|"FAILED", stage, result, error }. Call every 30s until SUCCEEDED. Required after queue_full_pipeline.',
|
|
655
|
+
inputSchema: {
|
|
656
|
+
type: 'object',
|
|
657
|
+
required: ['jobId'],
|
|
658
|
+
properties: {
|
|
659
|
+
jobId: { type: 'string', description: 'The jobId returned by queue_full_pipeline' },
|
|
660
|
+
},
|
|
661
|
+
},
|
|
662
|
+
handler: async ({ jobId }) => {
|
|
663
|
+
if (!jobId) throw new Error('jobId is required');
|
|
664
|
+
const targetUrl = process.env.WORKER_API_URL;
|
|
665
|
+
if (!targetUrl) {
|
|
666
|
+
throw new Error('Set WORKER_API_URL in .env (the worker HTTP endpoint, e.g. http://localhost:3211)');
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
const isHttps = targetUrl.startsWith('https');
|
|
670
|
+
const client = require(isHttps ? 'https' : 'http');
|
|
671
|
+
const safePath = `/jobs/${encodeURIComponent(jobId)}`;
|
|
672
|
+
|
|
673
|
+
const internalToken = process.env.INTERNAL_API_TOKEN;
|
|
674
|
+
return new Promise((resolve, reject) => {
|
|
675
|
+
const url = new URL(targetUrl.replace(/\/$/, '') + safePath);
|
|
676
|
+
const req = client.request({
|
|
677
|
+
hostname: url.hostname,
|
|
678
|
+
port: url.port || (isHttps ? 443 : 80),
|
|
679
|
+
path: url.pathname + url.search,
|
|
680
|
+
method: 'GET',
|
|
681
|
+
headers: internalToken ? { Authorization: `Bearer ${internalToken}` } : {},
|
|
682
|
+
}, (res) => {
|
|
683
|
+
const chunks = [];
|
|
684
|
+
res.on('data', d => chunks.push(d));
|
|
685
|
+
res.on('end', () => {
|
|
686
|
+
if (res.statusCode >= 400) return reject(new Error(`Worker returned HTTP ${res.statusCode}`));
|
|
687
|
+
const text = Buffer.concat(chunks).toString('utf8');
|
|
688
|
+
try { resolve(JSON.parse(text)); } catch { resolve({ raw: text }); }
|
|
689
|
+
});
|
|
690
|
+
});
|
|
691
|
+
req.on('error', reject);
|
|
692
|
+
req.setTimeout(15000, () => { req.destroy(); reject(new Error('get_job_status timeout')); });
|
|
693
|
+
req.end();
|
|
694
|
+
});
|
|
695
|
+
},
|
|
696
|
+
},
|
|
697
|
+
|
|
698
|
+
// ── Tool 13: Queue full pipeline ──────────────────────────────────────────
|
|
699
|
+
// Trigger a full pipeline job on the worker API or a generic HTTP endpoint.
|
|
700
|
+
queue_full_pipeline: {
|
|
701
|
+
description: 'Queue a full video pipeline job (research → script → render → publish) on the worker API or any HTTP endpoint. Returns {jobId, status} immediately — poll get_job_status for completion. Set WORKER_API_URL to target the worker, or pass webhookUrl for any custom endpoint.',
|
|
702
|
+
inputSchema: {
|
|
703
|
+
type: 'object',
|
|
704
|
+
required: ['topic'],
|
|
705
|
+
properties: {
|
|
706
|
+
topic: { type: 'string', description: 'Video topic to produce' },
|
|
707
|
+
style: { type: 'string', default: 'informative', description: 'Narrative style, e.g. "educational", "comedic"' },
|
|
708
|
+
language: { type: 'string', default: 'en', description: 'BCP-47 language code for TTS voice selection' },
|
|
709
|
+
channels: { type: 'array', items: { type: 'string' }, default: ['file'], description: 'Delivery channels for the publish step: file | webhook' },
|
|
710
|
+
webhookUrl: { type: 'string', description: 'HTTP endpoint to POST the job to. Overrides WORKER_API_URL env var.' },
|
|
711
|
+
payload: { type: 'object', description: 'Additional fields forwarded to the worker' },
|
|
712
|
+
},
|
|
713
|
+
},
|
|
714
|
+
handler: async ({ topic, style = 'informative', language = 'en', channels = ['file'], webhookUrl, payload = {} }) => {
|
|
715
|
+
const targetUrl = webhookUrl || process.env.WORKER_API_URL;
|
|
716
|
+
if (!targetUrl) {
|
|
717
|
+
throw new Error('Set WORKER_API_URL in .env (the worker HTTP endpoint, e.g. http://localhost:3211) or pass webhookUrl.');
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
// SSRF guard — only when the URL came from agent input (webhookUrl param),
|
|
721
|
+
// not from the operator-configured WORKER_API_URL env var (trusted).
|
|
722
|
+
if (webhookUrl) {
|
|
723
|
+
const { _assertSafeWebhookUrl } = require('../workflows/publishing');
|
|
724
|
+
_assertSafeWebhookUrl(webhookUrl);
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
const isHttps = targetUrl.startsWith('https');
|
|
728
|
+
const client = require(isHttps ? 'https' : 'http');
|
|
729
|
+
const body = JSON.stringify({ type: 'full_pipeline', payload: { topic, style, language, channels, ...payload } });
|
|
730
|
+
const internalToken = process.env.INTERNAL_API_TOKEN;
|
|
731
|
+
const authHeaders = internalToken ? { Authorization: `Bearer ${internalToken}` } : {};
|
|
732
|
+
|
|
733
|
+
const result = await new Promise((resolve, reject) => {
|
|
734
|
+
const url = new URL(`${targetUrl.replace(/\/$/, '')}/jobs`);
|
|
735
|
+
const req = client.request({
|
|
736
|
+
hostname: url.hostname,
|
|
737
|
+
port: url.port || (isHttps ? 443 : 80),
|
|
738
|
+
path: url.pathname + url.search,
|
|
739
|
+
method: 'POST',
|
|
740
|
+
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body), ...authHeaders },
|
|
741
|
+
}, (res) => {
|
|
742
|
+
const chunks = [];
|
|
743
|
+
res.on('data', d => chunks.push(d));
|
|
744
|
+
res.on('end', () => {
|
|
745
|
+
if (res.statusCode >= 400) return reject(new Error(`Worker returned HTTP ${res.statusCode}`));
|
|
746
|
+
const text = Buffer.concat(chunks).toString('utf8');
|
|
747
|
+
try { resolve(JSON.parse(text)); } catch { resolve({ raw: text }); }
|
|
748
|
+
});
|
|
749
|
+
});
|
|
750
|
+
req.on('error', reject);
|
|
751
|
+
req.setTimeout(30000, () => { req.destroy(); reject(new Error('Worker request timeout')); });
|
|
752
|
+
req.write(body);
|
|
753
|
+
req.end();
|
|
754
|
+
});
|
|
755
|
+
|
|
756
|
+
const jobId = result.id || result.jobId || crypto.randomBytes(8).toString('hex');
|
|
757
|
+
return { jobId, status: result.status || 'QUEUED', topic, workerResponse: result };
|
|
758
|
+
},
|
|
759
|
+
},
|
|
760
|
+
// ── Tool 15: Scaffold animation project ──────────────────────────────────
|
|
761
|
+
// Copies the HTML compositor template (reel/engine.js, index.html, scripts/)
|
|
762
|
+
// into a new project folder and generates a starter edit.js from the scenes
|
|
763
|
+
// the agent wrote. The agent then edits edit.js and calls render_animation.
|
|
764
|
+
scaffold_animation_project: {
|
|
765
|
+
description: `Scaffold a code-driven HTML compositor project for high-quality animated video.
|
|
766
|
+
Every frame is a pure function of time t rendered by Playwright + ffmpeg — no AI image generation needed.
|
|
767
|
+
Copies the engine template into outDir and generates a starter edit.js from your scenes.
|
|
768
|
+
|
|
769
|
+
Use this when:
|
|
770
|
+
- You want Remotion-style motion graphics (kinetic text, springs, morphing layouts)
|
|
771
|
+
- Pollinations/image providers are unavailable and you want real visual quality
|
|
772
|
+
- The user asks for animated video, code-rendered video, or "make it look good"
|
|
773
|
+
|
|
774
|
+
After scaffolding, the agent edits edit.js in outDir to animate each scene, then calls render_animation.`,
|
|
775
|
+
inputSchema: {
|
|
776
|
+
type: 'object',
|
|
777
|
+
required: ['outDir'],
|
|
778
|
+
properties: {
|
|
779
|
+
outDir: { type: 'string', description: 'Directory to scaffold the project into (created if absent)' },
|
|
780
|
+
scenes: { type: 'array', description: 'Scene array from create_video_from_script / write_script. Each scene narration/caption/visual_description generates one layer() in edit.js.' },
|
|
781
|
+
title: { type: 'string', description: 'Video title — written into edit.js reel() config' },
|
|
782
|
+
width: { type: 'number', default: 1920, description: '1920 for landscape, 1080 for portrait (Shorts)' },
|
|
783
|
+
height: { type: 'number', default: 1080, description: '1080 for landscape, 1920 for portrait (Shorts)' },
|
|
784
|
+
fps: { type: 'number', default: 30 },
|
|
785
|
+
audioPath: { type: 'string', description: 'Path to the mixed audio file to mux in (from multi_voice_tts or mix_audio.py)' },
|
|
786
|
+
},
|
|
787
|
+
},
|
|
788
|
+
handler: async ({ outDir, scenes = [], title = 'FrameForge Animation', width = 1920, height = 1080, fps = 30, audioPath }) => {
|
|
789
|
+
const syncFs = require('fs');
|
|
790
|
+
const pathMod = require('path');
|
|
791
|
+
|
|
792
|
+
if (!outDir) throw new Error('outDir is required');
|
|
793
|
+
|
|
794
|
+
// Locate the engine template shipped with FrameForge
|
|
795
|
+
const templateDir = pathMod.resolve(__dirname, '../../plugins/video-pipeline/skills/video-pipeline/assets/project');
|
|
796
|
+
if (!syncFs.existsSync(templateDir)) {
|
|
797
|
+
throw new Error(`Engine template not found at ${templateDir}. Ensure the FrameForge plugins/ folder is present.`);
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
// Create output directory
|
|
801
|
+
syncFs.mkdirSync(outDir, { recursive: true });
|
|
802
|
+
syncFs.mkdirSync(pathMod.join(outDir, 'out'), { recursive: true });
|
|
803
|
+
syncFs.mkdirSync(pathMod.join(outDir, 'reel'), { recursive: true });
|
|
804
|
+
syncFs.mkdirSync(pathMod.join(outDir, 'scripts'), { recursive: true });
|
|
805
|
+
|
|
806
|
+
// Copy engine files — all scripts the agent and production pipeline may need
|
|
807
|
+
const engineFiles = [
|
|
808
|
+
['reel/engine.js', 'reel/engine.js'],
|
|
809
|
+
['reel/engine.css', 'reel/engine.css'],
|
|
810
|
+
['reel/virtual-clock.js', 'reel/virtual-clock.js'],
|
|
811
|
+
['scripts/render.mjs', 'scripts/render.mjs'],
|
|
812
|
+
['scripts/look.mjs', 'scripts/look.mjs'],
|
|
813
|
+
['scripts/storyboard-validator.mjs', 'scripts/storyboard-validator.mjs'],
|
|
814
|
+
['scripts/mix_audio.py', 'scripts/mix_audio.py'],
|
|
815
|
+
['scripts/transcribe.py', 'scripts/transcribe.py'],
|
|
816
|
+
['scripts/verify.py', 'scripts/verify.py'],
|
|
817
|
+
['scripts/capture.mjs', 'scripts/capture.mjs'], // website/tweet screenshot b-roll
|
|
818
|
+
['scripts/stock.py', 'scripts/stock.py'], // stock footage search
|
|
819
|
+
['scripts/video_job.py', 'scripts/video_job.py'], // detached job queue with spend caps
|
|
820
|
+
['ANIMATION_GUIDE.md', 'ANIMATION_GUIDE.md'],
|
|
821
|
+
['STORYBOARD.md', 'STORYBOARD.md'],
|
|
822
|
+
// Pipeline subagent briefings (loaded by subagents, not the orchestrator's context)
|
|
823
|
+
['pipeline/briefings/orchestrator.md', 'pipeline/briefings/orchestrator.md'],
|
|
824
|
+
['pipeline/briefings/scene-author.md', 'pipeline/briefings/scene-author.md'],
|
|
825
|
+
['pipeline/briefings/diagram-author.md', 'pipeline/briefings/diagram-author.md'],
|
|
826
|
+
];
|
|
827
|
+
|
|
828
|
+
const copied = [];
|
|
829
|
+
for (const [src, dest] of engineFiles) {
|
|
830
|
+
const srcPath = pathMod.join(templateDir, src);
|
|
831
|
+
const destPath = pathMod.join(outDir, dest);
|
|
832
|
+
if (syncFs.existsSync(srcPath)) {
|
|
833
|
+
syncFs.mkdirSync(pathMod.dirname(destPath), { recursive: true });
|
|
834
|
+
syncFs.copyFileSync(srcPath, destPath);
|
|
835
|
+
copied.push(dest);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
// Copy index.html (compositor page — requires #stage div, engine.css, and edit.js module)
|
|
840
|
+
const indexSrc = pathMod.join(templateDir, 'index.html');
|
|
841
|
+
if (syncFs.existsSync(indexSrc)) {
|
|
842
|
+
syncFs.copyFileSync(indexSrc, pathMod.join(outDir, 'index.html'));
|
|
843
|
+
copied.push('index.html');
|
|
844
|
+
} else {
|
|
845
|
+
// Fallback compositor — correct structure with #stage for engine.js to append layers into
|
|
846
|
+
const minimalHtml = `<!DOCTYPE html>
|
|
847
|
+
<html lang="en"><head><meta charset="utf-8">
|
|
848
|
+
<link rel="stylesheet" href="reel/engine.css">
|
|
849
|
+
<style>
|
|
850
|
+
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
|
851
|
+
html, body { width: 100%; height: 100%; overflow: hidden; background: #000; }
|
|
852
|
+
#stage { position: relative; width: var(--reel-width, 1920px); height: var(--reel-height, 1080px); overflow: hidden; background: #000; }
|
|
853
|
+
#stage > [data-layer] { position: absolute; top: 0; left: 0; width: 100%; height: 100%; pointer-events: none; }
|
|
854
|
+
</style>
|
|
855
|
+
</head><body>
|
|
856
|
+
<div id="stage"></div>
|
|
857
|
+
<script type="module" src="edit.js"></script>
|
|
858
|
+
</body></html>`;
|
|
859
|
+
syncFs.writeFileSync(pathMod.join(outDir, 'index.html'), minimalHtml, 'utf8');
|
|
860
|
+
copied.push('index.html (generated fallback)');
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// Generate starter edit.js from scenes
|
|
864
|
+
// Each scene becomes one layer() with opacity in/out and a text() for its caption.
|
|
865
|
+
// The agent is expected to replace this with real animation code.
|
|
866
|
+
let duration = 0;
|
|
867
|
+
const sceneBlocks = scenes.map((scene, idx) => {
|
|
868
|
+
const wordsPerSec = 2.5;
|
|
869
|
+
const words = (scene.narration || '').split(/\s+/).filter(Boolean).length;
|
|
870
|
+
const dur = Math.max(2, Math.round((words / wordsPerSec) * 10) / 10);
|
|
871
|
+
const start = Math.round(duration * 10) / 10;
|
|
872
|
+
duration = Math.round((duration + dur) * 10) / 10;
|
|
873
|
+
const caption = (scene.caption || `Scene ${idx + 1}`).replace(/'/g, "\\'");
|
|
874
|
+
const visual = (scene.visual_description || scene.narration || '').slice(0, 80).replace(/'/g, "\\'");
|
|
875
|
+
return [
|
|
876
|
+
`// ─── Scene ${idx + 1}: ${start}s–${duration}s ────────────────────────────────`,
|
|
877
|
+
`// Visual: ${visual}`,
|
|
878
|
+
`layer('scene${idx + 1}').on(t => {`,
|
|
879
|
+
` const lt = t - ${start};`,
|
|
880
|
+
` if (lt < 0 || lt > ${dur}) return { opacity: 0 };`,
|
|
881
|
+
` const fadeIn = Math.min(1, lt / 0.3);`,
|
|
882
|
+
` const fadeOut = lt > ${dur} - 0.3 ? 1 - (lt - (${dur} - 0.3)) / 0.3 : 1;`,
|
|
883
|
+
` return { opacity: fadeIn * fadeOut };`,
|
|
884
|
+
`});`,
|
|
885
|
+
`at(${start}, text('${caption}', { font: '48px/1.3 sans-serif', color: '#fff', x: 'center', y: ${height / 2} }));`,
|
|
886
|
+
'',
|
|
887
|
+
].join('\n');
|
|
888
|
+
});
|
|
889
|
+
|
|
890
|
+
const editJs = [
|
|
891
|
+
`/**`,
|
|
892
|
+
` * edit.js — generated by FrameForge scaffold_animation_project`,
|
|
893
|
+
` * Title: ${title}`,
|
|
894
|
+
` *`,
|
|
895
|
+
` * AGENT INSTRUCTIONS:`,
|
|
896
|
+
` * 1. Read ANIMATION_GUIDE.md for the six determinism rules.`,
|
|
897
|
+
` * 2. Validate STORYBOARD.md: node scripts/storyboard-validator.mjs STORYBOARD.md`,
|
|
898
|
+
` * 3. Replace each scene block below with real animation (see reel/engine.js API).`,
|
|
899
|
+
` * 4. Check visuals: node scripts/render.mjs sheet`,
|
|
900
|
+
` * 5. Full render: node scripts/render.mjs full${audioPath ? ` --audio="${audioPath}"` : ''}`,
|
|
901
|
+
` *`,
|
|
902
|
+
` * Iron rule: every style must be a PURE FUNCTION OF t.`,
|
|
903
|
+
` * Never use Math.random(), Date.now(), setTimeout, or module-level mutable counters.`,
|
|
904
|
+
` * Use hash(i) from engine.js for deterministic randomness.`,
|
|
905
|
+
` */`,
|
|
906
|
+
`import { reel, layer, at, text, hash } from './reel/engine.js';`,
|
|
907
|
+
``,
|
|
908
|
+
`reel({ width: ${width}, height: ${height}, fps: ${fps}, duration: ${duration} });`,
|
|
909
|
+
``,
|
|
910
|
+
...sceneBlocks,
|
|
911
|
+
].join('\n');
|
|
912
|
+
|
|
913
|
+
syncFs.writeFileSync(pathMod.join(outDir, 'edit.js'), editJs, 'utf8');
|
|
914
|
+
|
|
915
|
+
return {
|
|
916
|
+
outDir,
|
|
917
|
+
title,
|
|
918
|
+
duration,
|
|
919
|
+
sceneCount: scenes.length,
|
|
920
|
+
copied,
|
|
921
|
+
editJs: pathMod.join(outDir, 'edit.js'),
|
|
922
|
+
nextSteps: [
|
|
923
|
+
`1. Read ANIMATION_GUIDE.md — six determinism rules and four storyboard gates (run from ${outDir})`,
|
|
924
|
+
`2. Edit STORYBOARD.md — every shot needs an event, non-overlapping reads, named transitions, no text`,
|
|
925
|
+
`3. Validate: node scripts/storyboard-validator.mjs STORYBOARD.md (run from ${outDir})`,
|
|
926
|
+
`4. Edit edit.js — replace starter layers with real animation (pure functions of t)`,
|
|
927
|
+
`5. Pre-flight: node scripts/look.mjs (fonts, clipped text, NaN transforms)`,
|
|
928
|
+
`6. Check sheet: node scripts/render.mjs sheet (one sheet per shot)`,
|
|
929
|
+
` Check motion: node scripts/render.mjs strip <start> <end> (every frame of a key motion)`,
|
|
930
|
+
` Check mobile: node scripts/render.mjs crop 9:16 <t>`,
|
|
931
|
+
`7. Full render: node scripts/render.mjs full${audioPath ? ` --audio="${audioPath}"` : ''} (run from ${outDir})`,
|
|
932
|
+
` Requires: npm install playwright && npx playwright install chromium`,
|
|
933
|
+
`8. Unattended/batch: python scripts/video_job.py submit --spec spec.json --max-usd 2.00`,
|
|
934
|
+
],
|
|
935
|
+
message: `Animation project scaffolded at ${outDir}. Read ANIMATION_GUIDE.md, validate STORYBOARD.md, edit edit.js, then render.`,
|
|
936
|
+
};
|
|
937
|
+
},
|
|
938
|
+
},
|
|
939
|
+
|
|
940
|
+
// ── Tool 16: Validate storyboard ─────────────────────────────────────────
|
|
941
|
+
// Hard-gate check on STORYBOARD.md before any scene code is allowed.
|
|
942
|
+
validate_storyboard: {
|
|
943
|
+
description: 'Validate a STORYBOARD.md or storyboard.json against the four hard gates (event, non-overlapping reads, named transitions, no on-screen text). Returns { passed, errors, warnings }. Must pass before writing scene code.',
|
|
944
|
+
inputSchema: {
|
|
945
|
+
type: 'object',
|
|
946
|
+
required: ['storyboardPath'],
|
|
947
|
+
properties: {
|
|
948
|
+
storyboardPath: { type: 'string', description: 'Path to STORYBOARD.md or storyboard.json' },
|
|
949
|
+
},
|
|
950
|
+
},
|
|
951
|
+
handler: async ({ storyboardPath }) => {
|
|
952
|
+
const syncFs = require('fs');
|
|
953
|
+
const pathMod = require('path');
|
|
954
|
+
const { spawnSync } = require('child_process');
|
|
955
|
+
|
|
956
|
+
if (!storyboardPath) throw new Error('storyboardPath is required');
|
|
957
|
+
if (!syncFs.existsSync(storyboardPath)) throw new Error(`Storyboard file not found: ${storyboardPath}`);
|
|
958
|
+
|
|
959
|
+
// Find the validator script relative to the storyboard file's project
|
|
960
|
+
const projectDir = pathMod.dirname(storyboardPath);
|
|
961
|
+
const validatorPath = pathMod.join(projectDir, 'scripts', 'storyboard-validator.mjs');
|
|
962
|
+
const fallbackValidator = pathMod.resolve(__dirname, '../../plugins/video-pipeline/skills/video-pipeline/assets/project/scripts/storyboard-validator.mjs');
|
|
963
|
+
const validator = syncFs.existsSync(validatorPath) ? validatorPath : fallbackValidator;
|
|
964
|
+
|
|
965
|
+
if (!syncFs.existsSync(validator)) {
|
|
966
|
+
throw new Error(`Storyboard validator not found. Run scaffold_animation_project first.`);
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
const result = spawnSync('node', [validator, storyboardPath], { encoding: 'utf8', timeout: 15000 });
|
|
970
|
+
const output = (result.stdout || '') + (result.stderr || '');
|
|
971
|
+
const passed = result.status === 0;
|
|
972
|
+
|
|
973
|
+
// Parse errors and warnings from the validator output
|
|
974
|
+
const errors = [];
|
|
975
|
+
const warnings = [];
|
|
976
|
+
for (const line of output.split('\n')) {
|
|
977
|
+
if (line.includes('❌')) errors.push(line.replace(/\s*❌\s*/, '').trim());
|
|
978
|
+
if (line.includes('⚠️')) warnings.push(line.replace(/\s*⚠️\s*/, '').trim());
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
return {
|
|
982
|
+
passed,
|
|
983
|
+
errors,
|
|
984
|
+
warnings,
|
|
985
|
+
message: passed
|
|
986
|
+
? `All 4 storyboard gates passed${warnings.length ? ` (${warnings.length} warning(s))` : ''}.`
|
|
987
|
+
: `${errors.length} gate violation(s). Fix STORYBOARD.md before writing scene code.`,
|
|
988
|
+
output,
|
|
989
|
+
};
|
|
990
|
+
},
|
|
991
|
+
},
|
|
992
|
+
|
|
993
|
+
// ── Tool 17: Render animation ─────────────────────────────────────────────
|
|
994
|
+
// Triggers render.mjs in a scaffolded project folder.
|
|
995
|
+
// Returns output path and QC results.
|
|
996
|
+
render_animation: {
|
|
997
|
+
description: `Run the HTML compositor renderer on a scaffolded animation project.
|
|
998
|
+
Modes:
|
|
999
|
+
"sheet" — contact sheet (one frame every 0.5s), fast visual check of the whole film
|
|
1000
|
+
"stills" — frames at specific timestamps (pass timestamps array)
|
|
1001
|
+
"strip" — every frame in a time range (motion/transition check); pass timestamps as [start, end]
|
|
1002
|
+
"storyboard" — 4 keyframes at 0%, 33%, 66%, 100% of duration
|
|
1003
|
+
"crop" — focal-point 9:16 mobile crop preview; pass timestamps as [aspect_ratio_string, t]
|
|
1004
|
+
"full" — full render → out/<title>.mp4
|
|
1005
|
+
"mux" — re-mux audio only (no re-render, use after changing audio)
|
|
1006
|
+
"look" — pre-render linter (fonts, clipped text, NaN transforms, missing assets)
|
|
1007
|
+
|
|
1008
|
+
Review budget: at least one "sheet" per shot, one "strip" per key motion/transition, one "crop" per story-carrying face.
|
|
1009
|
+
Returns { outputPath, passed } for "full", or { outputPath } for sheet/strip/storyboard.
|
|
1010
|
+
Requires: npm install playwright && npx playwright install chromium (one-time setup in projectDir).`,
|
|
1011
|
+
inputSchema: {
|
|
1012
|
+
type: 'object',
|
|
1013
|
+
required: ['projectDir', 'mode'],
|
|
1014
|
+
properties: {
|
|
1015
|
+
projectDir: { type: 'string', description: 'The directory created by scaffold_animation_project' },
|
|
1016
|
+
mode: { type: 'string', enum: ['sheet', 'stills', 'strip', 'storyboard', 'crop', 'full', 'mux', 'look'], description: 'Render mode' },
|
|
1017
|
+
timestamps: { type: 'array', items: {}, description: 'For mode="stills": [t1, t2, ...]. For mode="strip": [startSeconds, endSeconds]. For mode="crop": ["9:16", t]. For mode="sheet": omit or pass shot times.' },
|
|
1018
|
+
audioPath: { type: 'string', description: 'Audio file to mux in (for mode="full" or "mux")' },
|
|
1019
|
+
workers: { type: 'number', default: 4, description: 'Parallel Playwright pages for mode="full"' },
|
|
1020
|
+
crf: { type: 'number', default: 18, description: 'FFmpeg CRF quality (lower = better, larger file)' },
|
|
1021
|
+
},
|
|
1022
|
+
},
|
|
1023
|
+
handler: async ({ projectDir, mode, timestamps = [], audioPath, workers = 4, crf = 18 }) => {
|
|
1024
|
+
const syncFs = require('fs');
|
|
1025
|
+
const pathMod = require('path');
|
|
1026
|
+
const { spawnSync } = require('child_process');
|
|
1027
|
+
|
|
1028
|
+
if (!projectDir) throw new Error('projectDir is required');
|
|
1029
|
+
if (!syncFs.existsSync(projectDir)) throw new Error(`Project directory not found: ${projectDir}`);
|
|
1030
|
+
|
|
1031
|
+
const renderScript = pathMod.join(projectDir, 'scripts', 'render.mjs');
|
|
1032
|
+
if (!syncFs.existsSync(renderScript)) {
|
|
1033
|
+
throw new Error(`render.mjs not found in ${projectDir}/scripts/. Run scaffold_animation_project first.`);
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
// ── Validate timestamps ─────────────────────────────────────────────────
|
|
1037
|
+
// For mode="crop" the first item may be an aspect-ratio string like "9:16".
|
|
1038
|
+
// All other items must be numeric (seconds). No item may start with "--"
|
|
1039
|
+
// or contain shell metacharacters that could inject extra CLI flags.
|
|
1040
|
+
const SAFE_TS = /^[0-9]+([.:][0-9]+)*$/; // e.g. "1.5", "9:16", "02:30"
|
|
1041
|
+
const sanitisedTimestamps = timestamps.map((v, i) => {
|
|
1042
|
+
const s = String(v);
|
|
1043
|
+
if (s.startsWith('--') || !SAFE_TS.test(s)) {
|
|
1044
|
+
throw new Error(`timestamps[${i}] is not a valid time value or aspect ratio: "${s}"`);
|
|
1045
|
+
}
|
|
1046
|
+
return s;
|
|
1047
|
+
});
|
|
1048
|
+
|
|
1049
|
+
// ── Validate audioPath ──────────────────────────────────────────────────
|
|
1050
|
+
// Must resolve inside FILES_DIR so an agent cannot write outside the
|
|
1051
|
+
// sandbox by passing "../../etc/passwd" or similar.
|
|
1052
|
+
if (audioPath) {
|
|
1053
|
+
const resolved = pathMod.resolve(audioPath);
|
|
1054
|
+
const filesDir = pathMod.resolve(process.env.N8N_FILES_DIR || pathMod.join(os.homedir(), '.frameforge-files'));
|
|
1055
|
+
if (!resolved.startsWith(filesDir + pathMod.sep) && !resolved.startsWith(pathMod.resolve(projectDir) + pathMod.sep)) {
|
|
1056
|
+
throw new Error(`audioPath must be inside the project directory or FILES_DIR: ${audioPath}`);
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
const args = [renderScript, mode];
|
|
1061
|
+
if ((mode === 'stills' || mode === 'strip' || mode === 'crop') && sanitisedTimestamps.length > 0) {
|
|
1062
|
+
args.push(...sanitisedTimestamps);
|
|
1063
|
+
}
|
|
1064
|
+
if (audioPath) args.push(`--audio=${audioPath}`);
|
|
1065
|
+
if (mode === 'full') {
|
|
1066
|
+
args.push(`--workers=${workers}`, `--crf=${crf}`);
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
const result = spawnSync('node', args, {
|
|
1070
|
+
cwd: projectDir,
|
|
1071
|
+
encoding: 'utf8',
|
|
1072
|
+
timeout: 30 * 60 * 1000, // 30 min ceiling
|
|
1073
|
+
});
|
|
1074
|
+
|
|
1075
|
+
const stdout = result.stdout || '';
|
|
1076
|
+
const stderr = result.stderr || '';
|
|
1077
|
+
const exitCode = result.status;
|
|
1078
|
+
|
|
1079
|
+
if (exitCode !== 0 && mode === 'full') {
|
|
1080
|
+
throw new Error(`render.mjs ${mode} failed (exit ${exitCode}):\n${stderr.slice(-1000)}`);
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
// Locate produced file
|
|
1084
|
+
const outDir = pathMod.join(projectDir, 'out');
|
|
1085
|
+
let outputPath = null;
|
|
1086
|
+
if (syncFs.existsSync(outDir)) {
|
|
1087
|
+
const files = syncFs.readdirSync(outDir);
|
|
1088
|
+
if (mode === 'sheet' || mode === 'storyboard') {
|
|
1089
|
+
const sheet = files.find(f => f.endsWith('.jpg') || f.endsWith('.png'));
|
|
1090
|
+
if (sheet) outputPath = pathMod.join(outDir, sheet);
|
|
1091
|
+
} else if (mode === 'full' || mode === 'mux') {
|
|
1092
|
+
const mp4 = files.find(f => f.endsWith('.mp4') && !f.includes('_preview'));
|
|
1093
|
+
if (mp4) outputPath = pathMod.join(outDir, mp4);
|
|
1094
|
+
} else if (mode === 'stills' || mode === 'strip' || mode === 'crop') {
|
|
1095
|
+
const stills = files.filter(f => f.endsWith('.png') || f.endsWith('.jpg'));
|
|
1096
|
+
if (stills.length > 0) outputPath = stills.map(f => pathMod.join(outDir, f));
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
const passed = exitCode === 0;
|
|
1101
|
+
|
|
1102
|
+
return {
|
|
1103
|
+
mode,
|
|
1104
|
+
passed,
|
|
1105
|
+
exitCode,
|
|
1106
|
+
outputPath,
|
|
1107
|
+
projectDir,
|
|
1108
|
+
stdout: stdout.slice(-2000),
|
|
1109
|
+
stderr: passed ? undefined : stderr.slice(-1000),
|
|
1110
|
+
message: passed
|
|
1111
|
+
? `${mode} complete. Output: ${Array.isArray(outputPath) ? outputPath.join(', ') : (outputPath || outDir)}`
|
|
1112
|
+
: `${mode} failed (exit ${exitCode}). Check stderr for details.`,
|
|
1113
|
+
};
|
|
1114
|
+
},
|
|
1115
|
+
},
|
|
1116
|
+
|
|
1117
|
+
};
|
|
1118
|
+
|
|
1119
|
+
// ── MCP protocol handler (MCP 2024-11-05 spec) ───────────────────────────
|
|
1120
|
+
|
|
1121
|
+
const SERVER_INFO = {
|
|
1122
|
+
name: 'video-pipeline',
|
|
1123
|
+
version: '2.3.0',
|
|
1124
|
+
};
|
|
1125
|
+
|
|
1126
|
+
const CAPABILITIES = {
|
|
1127
|
+
tools: { listChanged: false },
|
|
1128
|
+
};
|
|
1129
|
+
|
|
1130
|
+
async function handleMCPRequest(body) {
|
|
1131
|
+
const { method, params, id } = body;
|
|
1132
|
+
|
|
1133
|
+
// ── MCP lifecycle ─────────────────────────────────────────────────────────
|
|
1134
|
+
|
|
1135
|
+
// initialize — required handshake; IDEs send this first
|
|
1136
|
+
if (method === 'initialize') {
|
|
1137
|
+
return {
|
|
1138
|
+
jsonrpc: '2.0', id,
|
|
1139
|
+
result: {
|
|
1140
|
+
protocolVersion: '2024-11-05',
|
|
1141
|
+
serverInfo: SERVER_INFO,
|
|
1142
|
+
capabilities: CAPABILITIES,
|
|
1143
|
+
},
|
|
1144
|
+
};
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
// notifications/initialized — no response expected
|
|
1148
|
+
if (method === 'notifications/initialized') return null;
|
|
1149
|
+
|
|
1150
|
+
// ping
|
|
1151
|
+
if (method === 'ping') return { jsonrpc: '2.0', id, result: {} };
|
|
1152
|
+
|
|
1153
|
+
// ── Tool calls ────────────────────────────────────────────────────────────
|
|
1154
|
+
|
|
1155
|
+
if (method === 'tools/list') {
|
|
1156
|
+
return {
|
|
1157
|
+
jsonrpc: '2.0', id,
|
|
1158
|
+
result: { tools: Object.entries(TOOLS).map(([name, t]) => ({ name, description: t.description, inputSchema: t.inputSchema })) },
|
|
1159
|
+
};
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
if (method === 'tools/call') {
|
|
1163
|
+
const { name, arguments: args } = params || {};
|
|
1164
|
+
const tool = TOOLS[name];
|
|
1165
|
+
if (!tool) return { jsonrpc: '2.0', id, error: { code: -32601, message: `Unknown tool: ${name}` } };
|
|
1166
|
+
try {
|
|
1167
|
+
const result = await tool.handler(args || {});
|
|
1168
|
+
return {
|
|
1169
|
+
jsonrpc: '2.0', id,
|
|
1170
|
+
result: { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] },
|
|
1171
|
+
};
|
|
1172
|
+
} catch (err) {
|
|
1173
|
+
return {
|
|
1174
|
+
jsonrpc: '2.0', id,
|
|
1175
|
+
error: { code: -32000, message: err.message, data: { errorClass: err.errorClass, stageName: err.stageName } },
|
|
1176
|
+
};
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
return { jsonrpc: '2.0', id, error: { code: -32601, message: `Unknown method: ${method}` } };
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
// ── HTTP server ───────────────────────────────────────────────────────────
|
|
1184
|
+
|
|
1185
|
+
const CORS_HEADERS = {
|
|
1186
|
+
'Access-Control-Allow-Origin': '*',
|
|
1187
|
+
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
|
|
1188
|
+
'Access-Control-Allow-Headers': 'Content-Type, Authorization',
|
|
1189
|
+
};
|
|
1190
|
+
|
|
1191
|
+
// 4 MB body limit — prevents OOM on Render free tier from oversized payloads
|
|
1192
|
+
const MAX_BODY_BYTES = 4 * 1024 * 1024;
|
|
1193
|
+
|
|
1194
|
+
async function readBodyLimited(req) {
|
|
1195
|
+
return new Promise((resolve, reject) => {
|
|
1196
|
+
let size = 0;
|
|
1197
|
+
const chunks = [];
|
|
1198
|
+
req.on('data', chunk => {
|
|
1199
|
+
size += chunk.length;
|
|
1200
|
+
if (size > MAX_BODY_BYTES) {
|
|
1201
|
+
req.destroy();
|
|
1202
|
+
return reject(Object.assign(new Error('Request body too large'), { statusCode: 413 }));
|
|
1203
|
+
}
|
|
1204
|
+
chunks.push(chunk);
|
|
1205
|
+
});
|
|
1206
|
+
req.on('end', () => resolve(Buffer.concat(chunks).toString()));
|
|
1207
|
+
req.on('error', reject);
|
|
1208
|
+
});
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
// Timing-safe token comparison — prevents timing oracle on the MCP auth token.
|
|
1212
|
+
function _checkMCPAuth(req) {
|
|
1213
|
+
const expected = process.env.MCP_AUTH_TOKEN;
|
|
1214
|
+
if (!expected) return true; // auth disabled — local dev only
|
|
1215
|
+
const header = req.headers['authorization'] || '';
|
|
1216
|
+
const token = header.replace(/^Bearer\s+/i, '').trim();
|
|
1217
|
+
const bufA = Buffer.from(token);
|
|
1218
|
+
const bufB = Buffer.from(expected);
|
|
1219
|
+
if (bufA.length !== bufB.length) {
|
|
1220
|
+
// Compare equal-length buffers to avoid short-circuit timing leak
|
|
1221
|
+
crypto.timingSafeEqual(bufB, bufB);
|
|
1222
|
+
return false;
|
|
1223
|
+
}
|
|
1224
|
+
return crypto.timingSafeEqual(bufA, bufB);
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
async function start() {
|
|
1228
|
+
await sm.init();
|
|
1229
|
+
await telemetry.init();
|
|
1230
|
+
|
|
1231
|
+
// Warn on startup if auth is disabled (intentional for local dev, risky otherwise)
|
|
1232
|
+
if (!process.env.MCP_AUTH_TOKEN) {
|
|
1233
|
+
console.warn('[MCP] WARNING: MCP_AUTH_TOKEN not set — HTTP MCP endpoint is unauthenticated. Set MCP_AUTH_TOKEN in .env for production use.');
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
const server = http.createServer(async (req, res) => {
|
|
1237
|
+
// CORS pre-flight (required by browser-based IDE extensions)
|
|
1238
|
+
if (req.method === 'OPTIONS') {
|
|
1239
|
+
res.writeHead(204, CORS_HEADERS);
|
|
1240
|
+
return res.end();
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
// Health check — omit tool list to avoid leaking internal API surface
|
|
1244
|
+
if (req.method === 'GET' && req.url === '/health') {
|
|
1245
|
+
res.writeHead(200, { ...CORS_HEADERS, 'Content-Type': 'application/json' });
|
|
1246
|
+
return res.end(JSON.stringify({ status: 'ok', serverInfo: SERVER_INFO }));
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
if (req.method !== 'POST') {
|
|
1250
|
+
res.writeHead(405, CORS_HEADERS); return res.end();
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
// Auth check — enforce MCP_AUTH_TOKEN when set
|
|
1254
|
+
if (!_checkMCPAuth(req)) {
|
|
1255
|
+
res.writeHead(401, { ...CORS_HEADERS, 'Content-Type': 'application/json' });
|
|
1256
|
+
return res.end(JSON.stringify({ jsonrpc: '2.0', error: { code: -32001, message: 'Unauthorized — set Authorization: Bearer <MCP_AUTH_TOKEN>' } }));
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
let rawBody;
|
|
1260
|
+
try {
|
|
1261
|
+
rawBody = await readBodyLimited(req);
|
|
1262
|
+
} catch (err) {
|
|
1263
|
+
const status = err.statusCode || 400;
|
|
1264
|
+
res.writeHead(status, { ...CORS_HEADERS, 'Content-Type': 'application/json' });
|
|
1265
|
+
return res.end(JSON.stringify({ jsonrpc: '2.0', error: { code: -32700, message: err.message } }));
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
let body;
|
|
1269
|
+
try { body = JSON.parse(rawBody); } catch {
|
|
1270
|
+
res.writeHead(400, { ...CORS_HEADERS, 'Content-Type': 'application/json' });
|
|
1271
|
+
return res.end(JSON.stringify({ jsonrpc: '2.0', error: { code: -32700, message: 'Parse error' } }));
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
// Handle JSON-RPC batch (array) or single request
|
|
1275
|
+
let response;
|
|
1276
|
+
if (Array.isArray(body)) {
|
|
1277
|
+
const results = await Promise.all(body.map(handleMCPRequest));
|
|
1278
|
+
response = results.filter(r => r !== null); // drop notifications
|
|
1279
|
+
} else {
|
|
1280
|
+
response = await handleMCPRequest(body);
|
|
1281
|
+
if (response === null) { res.writeHead(204, CORS_HEADERS); return res.end(); }
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
res.writeHead(200, { ...CORS_HEADERS, 'Content-Type': 'application/json' });
|
|
1285
|
+
res.end(JSON.stringify(response));
|
|
1286
|
+
});
|
|
1287
|
+
|
|
1288
|
+
server.listen(PORT, '127.0.0.1', () => {
|
|
1289
|
+
console.log(`[video-pipeline MCP] listening on http://127.0.0.1:${PORT}`);
|
|
1290
|
+
console.log(`[video-pipeline MCP] protocol: MCP 2024-11-05`);
|
|
1291
|
+
console.log(`[video-pipeline MCP] tools: ${Object.keys(TOOLS).join(', ')}`);
|
|
1292
|
+
});
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
if (require.main === module) {
|
|
1296
|
+
start().catch(err => { console.error('[video-agent MCP] startup failed:', err); process.exit(1); });
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
module.exports = { handleMCPRequest, TOOLS, SERVER_INFO, start };
|