@amg555/frameforge-mcp 2.4.9 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +22 -4
- package/CHANGELOG.md +174 -0
- package/CLAUDE.md +192 -9
- package/README.md +47 -6
- package/output/harness_test_1794418727139/frames/f000000.jpg +1 -0
- package/package.json +4 -3
- package/plugins/video-pipeline/.claude-plugin/plugin.json +8 -2
- package/plugins/video-pipeline/skills/.upstream-skills.json +143 -0
- package/plugins/video-pipeline/skills/html-renderer/SKILL.md +59 -15
- package/plugins/video-pipeline/skills/html-renderer/references/animation-patterns.md +609 -0
- package/plugins/video-pipeline/skills/html-renderer/references/character-animation.md +388 -0
- package/plugins/video-pipeline/skills/html-renderer/references/collage-broll.md +41 -0
- package/plugins/video-pipeline/skills/html-renderer/references/design.md +217 -0
- package/plugins/video-pipeline/skills/html-renderer/references/directing.md +165 -0
- package/plugins/video-pipeline/skills/html-renderer/references/earshot-audio.md +201 -0
- package/plugins/video-pipeline/skills/html-renderer/references/engine-api.md +480 -0
- package/plugins/video-pipeline/skills/html-renderer/references/film-styles.md +95 -0
- package/plugins/video-pipeline/skills/html-renderer/references/hyperframes-motion.md +208 -0
- package/plugins/video-pipeline/skills/html-renderer/references/motion-craft.md +1 -0
- package/plugins/video-pipeline/skills/html-renderer/references/onetake-camera.md +59 -0
- package/plugins/video-pipeline/skills/html-renderer/references/procedural-sfx.md +135 -0
- package/plugins/video-pipeline/skills/html-renderer/references/remotion-patterns.md +159 -0
- package/plugins/video-pipeline/skills/html-renderer/references/render-contract.md +1 -0
- package/plugins/video-pipeline/skills/video-pipeline/SKILL.md +19 -2
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/pipeline/briefings/orchestrator.md +15 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/pipeline/briefings/scene-author.md +21 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/reel/engine.js +1293 -20
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/reel/virtual-clock.js +36 -6
- package/plugins/video-pipeline/skills/video-pipeline/references/mcp-tools.md +126 -2
- package/scripts/deep-validate.js +52 -2
- package/src/audio-mixer/mixer.js +594 -63
- package/src/cli/doctor.js +36 -2
- package/src/cli/make.js +113 -9
- package/src/harness/bin/harness.mjs +127 -5
- package/src/harness/lib/capture.mjs +215 -192
- package/src/harness/lib/verify-quality.mjs +48 -7
- package/src/mcp-server/server.js +1020 -61
- package/src/skills/sync-upstream.js +290 -0
- package/src/utils/media-downloader.js +162 -0
- package/src/utils/transcribe.js +341 -0
- package/src/video-editor/editor.js +96 -11
- package/src/video-renderer/renderer.js +7 -1
- package/src/video-renderer/scene-animator.js +143 -1
- package/src/workflows/assets.js +4 -0
- package/src/workflows/ingest.js +10 -0
- package/src/workflows/render.js +2 -0
- package/src/workflows/script.js +18 -10
package/src/mcp-server/server.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* FrameForge MCP Server — MCP 2024-11-05 spec compliant
|
|
3
3
|
*
|
|
4
|
-
* Exposes
|
|
4
|
+
* Exposes 28 video production & autonomous editing tools via MCP.
|
|
5
5
|
* Works with IBM Bob, Cursor, Windsurf, Codex, opencode, Claude Code, or any MCP-capable IDE.
|
|
6
6
|
* Primary connection: Cloudflare Remote MCP (cloudflare-mcp/).
|
|
7
7
|
* Alternative: stdio (src/mcp-server/stdio.js) or HTTP (this file).
|
|
@@ -96,7 +96,7 @@ const TOOLS = {
|
|
|
96
96
|
claims: { type: 'array' },
|
|
97
97
|
topic: { type: 'string' },
|
|
98
98
|
style: { type: 'string', default: 'informative', description: 'e.g. "educational", "comedic", "documentary", "tutorial", "vlog"' },
|
|
99
|
-
persona: { type: 'string', default: '
|
|
99
|
+
persona: { type: 'string', default: 'narrator', description: 'Who the narrator is, e.g. "narrator", "travel blogger", "tech reviewer"' },
|
|
100
100
|
format: { type: 'string', default: 'YouTube video', description: 'e.g. "YouTube short", "Instagram reel", "long-form tutorial"' },
|
|
101
101
|
language: { type: 'string', default: 'en', description: 'BCP-47 language code for the script' },
|
|
102
102
|
sceneCount: { type: 'number', default: 55, description: 'Target number of scenes' },
|
|
@@ -587,10 +587,11 @@ Example scene:
|
|
|
587
587
|
style: { type: 'string', description: 'Visual style/theme override (e.g. cinematic_documentary, finance_executive, cyber_tech, retro_synthwave, nature_organic, hype_sports, minimal_luxury, breaking_news, cosmic_space)' },
|
|
588
588
|
theme: { type: 'string', description: 'Theme preset alias' },
|
|
589
589
|
showCaptions: { type: 'boolean', default: false, description: 'Display on-screen synchronized subtitles/captions. Default false (clean canvas without text unless explicitly requested in prompt).' },
|
|
590
|
+
showAnchor: { type: 'boolean', default: false, description: 'Display on-screen anchor/presenter character. Default false (no anchor unless explicitly requested in prompt).' },
|
|
590
591
|
productionId: { type: 'string', description: 'Optional — reuse an existing productionId' },
|
|
591
592
|
},
|
|
592
593
|
},
|
|
593
|
-
handler: async ({ scenes, title = 'FrameForge Video', orientation = 'landscape', voice, language, skipTts = false, parallelism = 1, withMusic = true, musicVolume = 0.07, topic = '', style = '', theme = '', showCaptions = false, productionId }) => {
|
|
594
|
+
handler: async ({ scenes, title = 'FrameForge Video', orientation = 'landscape', voice, language, skipTts = false, parallelism = 1, withMusic = true, musicVolume = 0.07, topic = '', style = '', theme = '', showCaptions = false, showAnchor = false, productionId }) => {
|
|
594
595
|
const { runAssets } = require('../workflows/assets');
|
|
595
596
|
const { runRender } = require('../workflows/render');
|
|
596
597
|
|
|
@@ -670,6 +671,7 @@ Example scene:
|
|
|
670
671
|
title,
|
|
671
672
|
language,
|
|
672
673
|
showCaptions,
|
|
674
|
+
showAnchor,
|
|
673
675
|
});
|
|
674
676
|
const renderResult = await runRender({
|
|
675
677
|
productionId: pid,
|
|
@@ -684,6 +686,7 @@ Example scene:
|
|
|
684
686
|
title,
|
|
685
687
|
language,
|
|
686
688
|
showCaptions,
|
|
689
|
+
showAnchor,
|
|
687
690
|
});
|
|
688
691
|
|
|
689
692
|
if (!renderResult.passed) {
|
|
@@ -945,6 +948,28 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
945
948
|
['pipeline/briefings/orchestrator.md', 'pipeline/briefings/orchestrator.md'],
|
|
946
949
|
['pipeline/briefings/scene-author.md', 'pipeline/briefings/scene-author.md'],
|
|
947
950
|
['pipeline/briefings/diagram-author.md', 'pipeline/briefings/diagram-author.md'],
|
|
951
|
+
// Professional audio mix script (full 15+ year sound engineer chain)
|
|
952
|
+
['scripts/mix_audio_pro.py', 'scripts/mix_audio_pro.py'],
|
|
953
|
+
];
|
|
954
|
+
|
|
955
|
+
// Copy upstream reference files into project references/ directory
|
|
956
|
+
// These are sourced from the FrameForge skills directory, not the project template
|
|
957
|
+
const skillsDir = pathMod.resolve(__dirname, '../../plugins/video-pipeline/skills/html-renderer/references');
|
|
958
|
+
const referenceFiles = [
|
|
959
|
+
'animation-patterns.md',
|
|
960
|
+
'character-animation.md',
|
|
961
|
+
'collage-broll.md',
|
|
962
|
+
'design.md',
|
|
963
|
+
'directing.md',
|
|
964
|
+
'earshot-audio.md',
|
|
965
|
+
'engine-api.md',
|
|
966
|
+
'film-styles.md',
|
|
967
|
+
'hyperframes-motion.md',
|
|
968
|
+
'motion-craft.md',
|
|
969
|
+
'onetake-camera.md',
|
|
970
|
+
'procedural-sfx.md',
|
|
971
|
+
'remotion-patterns.md',
|
|
972
|
+
'render-contract.md',
|
|
948
973
|
];
|
|
949
974
|
|
|
950
975
|
const copied = [];
|
|
@@ -958,6 +983,19 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
958
983
|
}
|
|
959
984
|
}
|
|
960
985
|
|
|
986
|
+
// Copy upstream reference files into references/ subfolder of project
|
|
987
|
+
if (syncFs.existsSync(skillsDir)) {
|
|
988
|
+
const refsOutDir = pathMod.join(outDir, 'references');
|
|
989
|
+
syncFs.mkdirSync(refsOutDir, { recursive: true });
|
|
990
|
+
for (const refFile of referenceFiles) {
|
|
991
|
+
const srcRef = pathMod.join(skillsDir, refFile);
|
|
992
|
+
if (syncFs.existsSync(srcRef)) {
|
|
993
|
+
syncFs.copyFileSync(srcRef, pathMod.join(refsOutDir, refFile));
|
|
994
|
+
copied.push(`references/${refFile}`);
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
|
|
961
999
|
// Copy index.html (compositor page — requires #stage div, engine.css, and edit.js module)
|
|
962
1000
|
const indexSrc = pathMod.join(templateDir, 'index.html');
|
|
963
1001
|
if (syncFs.existsSync(indexSrc)) {
|
|
@@ -982,7 +1020,9 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
982
1020
|
copied.push('index.html (generated fallback)');
|
|
983
1021
|
}
|
|
984
1022
|
|
|
985
|
-
// Generate starter edit.js from scenes
|
|
1023
|
+
// Generate starter edit.js from scenes.
|
|
1024
|
+
// Follows all 10 motion craft rules: springs not linear, 3-property entrances,
|
|
1025
|
+
// staggered choreography, asymmetric exits, 5-layer depth, continuous life.
|
|
986
1026
|
let duration = 0;
|
|
987
1027
|
const sceneBlocks = scenes.map((scene, idx) => {
|
|
988
1028
|
const wordsPerSec = 2.5;
|
|
@@ -993,6 +1033,8 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
993
1033
|
const caption = (scene.caption || `Scene ${idx + 1}`).replace(/'/g, "\\'");
|
|
994
1034
|
const visual = (scene.visual_description || scene.narration || '').slice(0, 80).replace(/'/g, "\\'");
|
|
995
1035
|
const textLower = ((scene.narration || '') + ' ' + (scene.visual_description || '') + ' ' + caption).toLowerCase();
|
|
1036
|
+
// Deterministic per-scene accent hue offset so each scene has a subtly distinct palette
|
|
1037
|
+
const hueOffset = (idx * 37) % 30;
|
|
996
1038
|
|
|
997
1039
|
let componentCode;
|
|
998
1040
|
if (scene.image_path) {
|
|
@@ -1015,16 +1057,67 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
1015
1057
|
].join('\n');
|
|
1016
1058
|
}
|
|
1017
1059
|
|
|
1060
|
+
// ── Motion Craft: 5-layer depth stack + spring entrance + continuous life ──
|
|
1061
|
+
// Layer 1 (bg) — radial gradient backdrop, subtle hue drift
|
|
1062
|
+
// Layer 2 (env) — lightweight environment detail (blueprint grid / star field)
|
|
1063
|
+
// Layer 3 (subject)— component, driven by spring entrance
|
|
1064
|
+
// Layer 4 (overlay)— caption chip, staggered 0.18s after subject
|
|
1065
|
+
// Layer 5 (fg) — vignette corner fade
|
|
1066
|
+
const bgLayer = `scene${idx + 1}_bg`;
|
|
1067
|
+
const envLayer = `scene${idx + 1}_env`;
|
|
1068
|
+
const subjectLayer = `scene${idx + 1}_subject`;
|
|
1069
|
+
const overlayLayer = `scene${idx + 1}_overlay`;
|
|
1070
|
+
const fgLayer = `scene${idx + 1}_fg`;
|
|
1071
|
+
|
|
1018
1072
|
return [
|
|
1019
|
-
`// ─── Scene ${idx + 1}: ${start}s–${duration}s
|
|
1073
|
+
`// ─── Scene ${idx + 1}: ${start}s–${duration}s ─────────────────────────────────────────`,
|
|
1020
1074
|
`// Visual: ${visual}`,
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
`
|
|
1024
|
-
`
|
|
1025
|
-
` const
|
|
1026
|
-
`
|
|
1075
|
+
``,
|
|
1076
|
+
`// Layer 1 — ambient backdrop with continuous hue drift (motion craft rule #7)`,
|
|
1077
|
+
`layer('${bgLayer}', { start: ${start}, end: ${duration} }).on((t, lt, dur) => {`,
|
|
1078
|
+
` if (lt < 0 || lt > dur) return { opacity: 0, display: 'none' };`,
|
|
1079
|
+
` const p = lt / dur;`,
|
|
1080
|
+
` // Lissajous camera drift — never a frozen frame`,
|
|
1081
|
+
` const dx = Math.sin(0.4 * lt) * 6;`,
|
|
1082
|
+
` const dy = Math.cos(0.3 * lt) * 4;`,
|
|
1083
|
+
` const hue = ${hueOffset} + p * 8; // gentle hue progression`,
|
|
1084
|
+
` return {`,
|
|
1085
|
+
` position: 'absolute', inset: '0',`,
|
|
1086
|
+
` background: \`radial-gradient(ellipse 70% 60% at \${50 + dx * 0.4}% \${50 + dy * 0.4}%, hsl(\${210 + hue},40%,12%) 0%, hsl(\${200 + hue},50%,6%) 100%)\`,`,
|
|
1087
|
+
` transform: \`translate(\${dx}px, \${dy}px) scale(\${1 + p * 0.05})\`,`,
|
|
1088
|
+
` };`,
|
|
1089
|
+
`});`,
|
|
1090
|
+
``,
|
|
1091
|
+
`// Layer 3 — primary subject with spring entrance (motion craft rules #1 & #2)`,
|
|
1092
|
+
`layer('${subjectLayer}', { start: ${start}, end: ${duration} }).on((t, lt, dur) => {`,
|
|
1093
|
+
` if (lt < 0 || lt > dur) return { opacity: 0, display: 'none' };`,
|
|
1094
|
+
` const s = spring(lt, { tension: 200, friction: 22 }); // damped harmonic entrance`,
|
|
1095
|
+
` // Exit faster than entrance (motion craft rule #4)`,
|
|
1096
|
+
` const exitProgress = lt > dur - 0.25 ? 1 - (dur - lt) / 0.25 : 0;`,
|
|
1097
|
+
` const opacity = s * (1 - exitProgress);`,
|
|
1098
|
+
` return {`,
|
|
1099
|
+
` // 3-property entrance: opacity + scale + Y translation (motion craft rule #2)`,
|
|
1100
|
+
` opacity,`,
|
|
1101
|
+
` transform: \`scale(\${0.88 + 0.12 * s}) translateY(\${(1 - s) * 40}px)\`,`,
|
|
1102
|
+
` };`,
|
|
1103
|
+
`});`,
|
|
1104
|
+
``,
|
|
1105
|
+
`// Layer 4 — caption overlay staggered 0.18s after subject (motion craft rule #3)`,
|
|
1106
|
+
`layer('${overlayLayer}', { start: ${start + 0.18}, end: ${duration} }).on((t, lt, dur) => {`,
|
|
1107
|
+
` if (lt < 0 || lt > dur) return { opacity: 0, display: 'none' };`,
|
|
1108
|
+
` const s = spring(lt, { tension: 280, friction: 28 }); // snappier than subject`,
|
|
1109
|
+
` return { opacity: s, transform: \`translateX(\${(1 - s) * -20}px)\` };`,
|
|
1110
|
+
`});`,
|
|
1111
|
+
``,
|
|
1112
|
+
`// Layer 5 — foreground vignette (depth, always on)`,
|
|
1113
|
+
`layer('${fgLayer}', { start: ${start}, end: ${duration} }).on((t, lt, dur) => {`,
|
|
1114
|
+
` if (lt < 0 || lt > dur) return { opacity: 0, display: 'none' };`,
|
|
1115
|
+
` return {`,
|
|
1116
|
+
` position: 'absolute', inset: '0', pointerEvents: 'none',`,
|
|
1117
|
+
` background: 'radial-gradient(ellipse 100% 100% at 50% 50%, transparent 55%, rgba(0,0,0,0.45) 100%)',`,
|
|
1118
|
+
` };`,
|
|
1027
1119
|
`});`,
|
|
1120
|
+
``,
|
|
1028
1121
|
componentCode,
|
|
1029
1122
|
'',
|
|
1030
1123
|
].join('\n');
|
|
@@ -1042,11 +1135,19 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
1042
1135
|
` * 4. Check visuals: node scripts/render.mjs sheet`,
|
|
1043
1136
|
` * 5. Full render: node scripts/render.mjs full${audioPath ? ` --audio="${audioPath}"` : ''}`,
|
|
1044
1137
|
` *`,
|
|
1138
|
+
` * Motion craft rules encoded in starter code:`,
|
|
1139
|
+
` * Rule 1: spring() damped harmonic entrance — no linear easing`,
|
|
1140
|
+
` * Rule 2: 3-property entrance: opacity + scale(0.88→1) + translateY(40→0)`,
|
|
1141
|
+
` * Rule 3: caption staggered 0.18s after subject`,
|
|
1142
|
+
` * Rule 4: exit 0.25s vs 0.6s entrance — exits faster than entrances`,
|
|
1143
|
+
` * Rule 5: 5 named layers: bg / env / subject / overlay / fg`,
|
|
1144
|
+
` * Rule 7: Lissajous camera drift — never a frozen frame`,
|
|
1145
|
+
` *`,
|
|
1045
1146
|
` * Iron rule: every style must be a PURE FUNCTION OF t.`,
|
|
1046
1147
|
` * Never use Math.random(), Date.now(), setTimeout, or module-level mutable counters.`,
|
|
1047
1148
|
` * Use hash(i) from engine.js for deterministic randomness.`,
|
|
1048
1149
|
` */`,
|
|
1049
|
-
`import { reel, layer, at, text, wallCard, codeCard, terminal, imageCard, stamp, chip, hash } from './reel/engine.js';`,
|
|
1150
|
+
`import { reel, layer, at, spring, text, wallCard, codeCard, terminal, imageCard, stamp, chip, hash } from './reel/engine.js';`,
|
|
1050
1151
|
``,
|
|
1051
1152
|
`reel({ width: ${width}, height: ${height}, fps: ${fps}, duration: ${duration} });`,
|
|
1052
1153
|
``,
|
|
@@ -1064,18 +1165,30 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
1064
1165
|
editJs: pathMod.join(outDir, 'edit.js'),
|
|
1065
1166
|
nextSteps: [
|
|
1066
1167
|
`1. Read ANIMATION_GUIDE.md — six determinism rules and four storyboard gates (run from ${outDir})`,
|
|
1168
|
+
` Also read references/animation-patterns.md — ClaudeAnimationBase 12fps boil, onetake 39-move library, collage B-roll 3-gate, Remotion springs`,
|
|
1169
|
+
` Read references/character-animation.md — full ClaudeAnimationBase guide: 31 emotions, timing principles, pure-fn constraint`,
|
|
1170
|
+
` Read references/hyperframes-motion.md — HyperFrames GSAP motion craft measured from official launch films`,
|
|
1171
|
+
` Read references/directing.md — lemo-opuscar directing method: story, sound layers, rhythm, camera, performance, delivery checklist`,
|
|
1172
|
+
` Read references/earshot-audio.md — earshot 5-loop professional audio workflow: forge DSP, kit sound bank, LUFS tables`,
|
|
1173
|
+
` Read references/motion-craft.md for 10 hard motion craft rules`,
|
|
1174
|
+
` Read references/film-styles.md for 24 cinematic film style references`,
|
|
1067
1175
|
`2. Edit STORYBOARD.md — every shot needs an event, non-overlapping reads, named transitions, no text`,
|
|
1068
1176
|
`3. Validate: node scripts/storyboard-validator.mjs STORYBOARD.md (run from ${outDir})`,
|
|
1069
1177
|
`4. Edit edit.js — replace starter layers with real animation (pure functions of t)`,
|
|
1178
|
+
` Motion craft: spring() not linear, 3-property entrances, stagger 0.15–0.28s, exits faster than entrances`,
|
|
1179
|
+
` 12fps organic boil: const step = Math.floor(lt * 12) then seeded Math.sin(seed * 19.37 + step * 2.41)`,
|
|
1180
|
+
` SFX cues: cue('whoosh', t, 0.8) in edit.js — registered in window.__cues for render.mjs`,
|
|
1070
1181
|
`5. Pre-flight: node scripts/look.mjs (fonts, clipped text, NaN transforms)`,
|
|
1071
1182
|
`6. Check sheet: node scripts/render.mjs sheet (one sheet per shot)`,
|
|
1072
1183
|
` Check motion: node scripts/render.mjs strip <start> <end> (every frame of a key motion)`,
|
|
1073
1184
|
` Check mobile: node scripts/render.mjs crop 9:16 <t>`,
|
|
1074
1185
|
`7. Full render: node scripts/render.mjs full${audioPath ? ` --audio="${audioPath}"` : ''} (run from ${outDir})`,
|
|
1075
1186
|
` Requires: npm install playwright && npx playwright install chromium`,
|
|
1076
|
-
`8.
|
|
1187
|
+
`8. QA gate: node scripts/harness.mjs quality (Opus 5.5 closed-loop fix protocol)`,
|
|
1188
|
+
` → returns { passed, overallScore, worstFrames } — fix ONLY failing timestamps, re-strip, re-score`,
|
|
1189
|
+
`9. Unattended/batch: python scripts/video_job.py submit --spec spec.json --max-usd 2.00`,
|
|
1077
1190
|
],
|
|
1078
|
-
message: `Animation project scaffolded at ${outDir}. Read ANIMATION_GUIDE.md
|
|
1191
|
+
message: `Animation project scaffolded at ${outDir}. Read ANIMATION_GUIDE.md + references/, validate STORYBOARD.md, edit edit.js, then render.`,
|
|
1079
1192
|
};
|
|
1080
1193
|
},
|
|
1081
1194
|
},
|
|
@@ -1144,27 +1257,41 @@ Modes:
|
|
|
1144
1257
|
"strip" — every frame in a time range (motion/transition check); pass timestamps as [start, end]
|
|
1145
1258
|
"storyboard" — 4 keyframes at 0%, 33%, 66%, 100% of duration
|
|
1146
1259
|
"crop" — focal-point 9:16 mobile crop preview; pass timestamps as [aspect_ratio_string, t]
|
|
1147
|
-
"full" — full render → out/<title>.mp4
|
|
1260
|
+
"full" — full render → out/<title>.mp4 (auto-runs purity verify gate first)
|
|
1148
1261
|
"mux" — re-mux audio only (no re-render, use after changing audio)
|
|
1149
1262
|
"kit" — export EDL/XML for Premiere Pro / DaVinci Resolve; run after "full"
|
|
1150
1263
|
"look" — pre-render linter (fonts, clipped text, NaN transforms, missing assets)
|
|
1264
|
+
"quality" — closed-loop QA gate (1–10 score per frame, worst frames, pass/fail); the Opus 5.5 fix-loop entry point
|
|
1265
|
+
"golden" — golden stills regression: use flags.save to capture baseline, flags.check to compare
|
|
1266
|
+
|
|
1267
|
+
Closed-loop QA fix protocol (Opus 5.5 / Claude Code pattern):
|
|
1268
|
+
1. render_animation({ mode: "quality" }) → { passed, overallScore, worstFrames: [{time, score, issues}] }
|
|
1269
|
+
2. If passed: false → fix ONLY the edit.js code paths at the failing timestamps.
|
|
1270
|
+
3. render_animation({ mode: "strip", timestamps: [failingT - 0.5, failingT + 0.5] }) to verify the fix visually.
|
|
1271
|
+
4. render_animation({ mode: "quality" }) again. Repeat max 3 rounds.
|
|
1272
|
+
5. NEVER re-render the full film to fix a single failing timestamp. Use strip only.
|
|
1151
1273
|
|
|
1152
1274
|
Review budget: at least one "sheet" per shot, one "strip" per key motion/transition, one "crop" per story-carrying face.
|
|
1153
|
-
Returns { outputPath, passed } for "
|
|
1275
|
+
Returns { outputPath, passed, overallScore, worstFrames } for "quality"; { outputPath, passed } for "full"; { outputPath } for other modes.
|
|
1154
1276
|
Requires: npm install playwright && npx playwright install chromium (one-time setup in projectDir).`,
|
|
1155
1277
|
inputSchema: {
|
|
1156
1278
|
type: 'object',
|
|
1157
1279
|
required: ['projectDir', 'mode'],
|
|
1158
1280
|
properties: {
|
|
1159
|
-
projectDir:
|
|
1160
|
-
mode:
|
|
1161
|
-
timestamps:
|
|
1162
|
-
audioPath:
|
|
1163
|
-
workers:
|
|
1164
|
-
crf:
|
|
1281
|
+
projectDir: { type: 'string', description: 'The directory created by scaffold_animation_project' },
|
|
1282
|
+
mode: { type: 'string', enum: ['sheet', 'stills', 'strip', 'storyboard', 'crop', 'full', 'mux', 'kit', 'look', 'verify', 'doctor', 'pixels', 'quality', 'golden'], description: 'Render mode' },
|
|
1283
|
+
timestamps: { type: 'array', items: {}, description: 'For mode="stills": [t1, t2, ...]. For mode="strip": [startSeconds, endSeconds]. For mode="crop": ["9:16", t]. For mode="sheet": omit or pass shot times.' },
|
|
1284
|
+
audioPath: { type: 'string', description: 'Audio file to mux in (for mode="full" or "mux")' },
|
|
1285
|
+
workers: { type: 'number', default: 4, description: 'Parallel Playwright pages for mode="full"' },
|
|
1286
|
+
crf: { type: 'number', default: 18, description: 'FFmpeg CRF quality (lower = better, larger file)' },
|
|
1287
|
+
subframes: { type: 'number', default: 1, description: 'Motion-blur subframes per output frame (1 = none, 6 = 180° shutter). 6 subframes needed for fast morphs; adds ~6× render time.' },
|
|
1288
|
+
skipVerify: { type: 'boolean', default: false, description: 'Skip purity verification gate before full render (default: false — always verify).' },
|
|
1289
|
+
goldenSave: { type: 'boolean', default: false, description: 'For mode="golden": capture and save 4-keyframe baseline hashes to out/check/golden.json.' },
|
|
1290
|
+
goldenCheck: { type: 'boolean', default: false, description: 'For mode="golden": compare current 4 keyframes against saved golden.json baseline.' },
|
|
1291
|
+
sampleRate: { type: 'number', default: 2, description: 'For mode="quality": samples per second to evaluate (default 2).' },
|
|
1165
1292
|
},
|
|
1166
1293
|
},
|
|
1167
|
-
handler: async ({ projectDir, mode, timestamps = [], audioPath, workers = 4, crf = 18 }) => {
|
|
1294
|
+
handler: async ({ projectDir, mode, timestamps = [], audioPath, workers = 4, crf = 18, subframes = 1, skipVerify = false, goldenSave = false, goldenCheck = false, sampleRate = 2 }) => {
|
|
1168
1295
|
const syncFs = require('fs');
|
|
1169
1296
|
const pathMod = require('path');
|
|
1170
1297
|
const { spawnSync } = require('child_process');
|
|
@@ -1202,7 +1329,8 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1202
1329
|
}
|
|
1203
1330
|
|
|
1204
1331
|
const harnessScript = pathMod.join(projectDir, 'scripts', 'harness.mjs');
|
|
1205
|
-
|
|
1332
|
+
// quality and golden route through harness.mjs; verify/doctor/pixels too
|
|
1333
|
+
const isHarnessMode = ['verify', 'doctor', 'pixels', 'quality', 'golden'].includes(mode);
|
|
1206
1334
|
const scriptToRun = (isHarnessMode && syncFs.existsSync(harnessScript)) ? harnessScript : renderScript;
|
|
1207
1335
|
|
|
1208
1336
|
const args = [scriptToRun, mode];
|
|
@@ -1212,6 +1340,13 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1212
1340
|
if (audioPath) args.push(`--audio=${audioPath}`);
|
|
1213
1341
|
if (mode === 'full') {
|
|
1214
1342
|
args.push(`--workers=${workers}`, `--crf=${crf}`);
|
|
1343
|
+
if (subframes > 1) args.push(`--subframes=${subframes}`);
|
|
1344
|
+
if (skipVerify) args.push('--skip-verify');
|
|
1345
|
+
}
|
|
1346
|
+
if (mode === 'quality' && sampleRate !== 2) args.push(`--sample-rate=${sampleRate}`);
|
|
1347
|
+
if (mode === 'golden') {
|
|
1348
|
+
if (goldenSave) args.push('--save');
|
|
1349
|
+
if (goldenCheck) args.push('--check');
|
|
1215
1350
|
}
|
|
1216
1351
|
|
|
1217
1352
|
const result = spawnSync('node', args, {
|
|
@@ -1230,6 +1365,7 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1230
1365
|
|
|
1231
1366
|
// Locate produced file
|
|
1232
1367
|
const outDir = pathMod.join(projectDir, 'out');
|
|
1368
|
+
const checkDir = pathMod.join(outDir, 'check');
|
|
1233
1369
|
let outputPath = null;
|
|
1234
1370
|
if (syncFs.existsSync(outDir)) {
|
|
1235
1371
|
const files = syncFs.readdirSync(outDir);
|
|
@@ -1249,7 +1385,7 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1249
1385
|
}
|
|
1250
1386
|
|
|
1251
1387
|
let purityReport = null;
|
|
1252
|
-
const purityReportFile = pathMod.join(
|
|
1388
|
+
const purityReportFile = pathMod.join(checkDir, 'purity_report.json');
|
|
1253
1389
|
if (mode === 'verify' && syncFs.existsSync(purityReportFile)) {
|
|
1254
1390
|
try {
|
|
1255
1391
|
purityReport = JSON.parse(syncFs.readFileSync(purityReportFile, 'utf8'));
|
|
@@ -1257,19 +1393,55 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1257
1393
|
} catch (_) {}
|
|
1258
1394
|
}
|
|
1259
1395
|
|
|
1396
|
+
// Parse quality report for mode="quality" — return structured worst-frames data
|
|
1397
|
+
let qualityReport = null;
|
|
1398
|
+
const qualityReportFile = pathMod.join(checkDir, 'quality_report.json');
|
|
1399
|
+
if (mode === 'quality' && syncFs.existsSync(qualityReportFile)) {
|
|
1400
|
+
try {
|
|
1401
|
+
qualityReport = JSON.parse(syncFs.readFileSync(qualityReportFile, 'utf8'));
|
|
1402
|
+
outputPath = qualityReportFile;
|
|
1403
|
+
} catch (_) {}
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
// Parse golden report for mode="golden"
|
|
1407
|
+
let goldenReport = null;
|
|
1408
|
+
const goldenReportFile = pathMod.join(checkDir, 'golden.json');
|
|
1409
|
+
if (mode === 'golden' && syncFs.existsSync(goldenReportFile)) {
|
|
1410
|
+
try {
|
|
1411
|
+
goldenReport = JSON.parse(syncFs.readFileSync(goldenReportFile, 'utf8'));
|
|
1412
|
+
outputPath = goldenReportFile;
|
|
1413
|
+
} catch (_) {}
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1260
1416
|
const passed = exitCode === 0;
|
|
1261
1417
|
|
|
1418
|
+
// Read sfx-events.json if render.mjs extracted composition events
|
|
1419
|
+
let sfxEvents = null;
|
|
1420
|
+
const sfxEventsPath = pathMod.join(outDir, 'sfx-events.json');
|
|
1421
|
+
if (syncFs.existsSync(sfxEventsPath)) {
|
|
1422
|
+
try { sfxEvents = JSON.parse(syncFs.readFileSync(sfxEventsPath, 'utf8')); } catch (_) {}
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1262
1425
|
return {
|
|
1263
1426
|
mode,
|
|
1264
1427
|
passed,
|
|
1265
1428
|
exitCode,
|
|
1266
1429
|
outputPath,
|
|
1267
1430
|
purityReport,
|
|
1431
|
+
qualityReport,
|
|
1432
|
+
goldenReport,
|
|
1433
|
+
// Structured quality fields for the Opus 5.5 fix loop
|
|
1434
|
+
overallScore: qualityReport?.overallScore ?? null,
|
|
1435
|
+
worstFrames: qualityReport?.worstFrames ?? null,
|
|
1436
|
+
motionDelta: qualityReport?.motionDelta ?? null,
|
|
1437
|
+
// Composition events extracted by render.mjs (window.__events contract)
|
|
1438
|
+
sfxEvents: sfxEvents,
|
|
1439
|
+
sfxEventsPath: sfxEvents ? sfxEventsPath : null,
|
|
1268
1440
|
projectDir,
|
|
1269
1441
|
stdout: stdout.slice(-2000),
|
|
1270
1442
|
stderr: passed ? undefined : stderr.slice(-1000),
|
|
1271
1443
|
message: passed
|
|
1272
|
-
? `${mode} complete. Output: ${Array.isArray(outputPath) ? outputPath.join(', ') : (outputPath || outDir)}`
|
|
1444
|
+
? `${mode} complete. Output: ${Array.isArray(outputPath) ? outputPath.join(', ') : (outputPath || outDir)}${sfxEvents ? ` | ${sfxEvents.length} composition SFX event(s) in sfx-events.json` : ''}`
|
|
1273
1445
|
: `${mode} failed (exit ${exitCode}). Check stderr for details.`,
|
|
1274
1446
|
};
|
|
1275
1447
|
},
|
|
@@ -1698,9 +1870,20 @@ Evaluates 4 critical visual axes: content presence (non-blank), proper exposure
|
|
|
1698
1870
|
// Plan / Approve / Render video editing workflow: generates edit-plan.md & edit.json
|
|
1699
1871
|
plan_video_edit: {
|
|
1700
1872
|
description: `Generate a structured Plan/Approve/Render video editing package for raw video footage.
|
|
1701
|
-
Analyzes input clips, segments narrative beats, calculates cut in/out timestamps, assigns transitions, speed ramps,
|
|
1702
|
-
1. edit-plan.md — Human & agent readable decision list with rationale per cut
|
|
1703
|
-
2. edit.json — Machine-executable configuration directly runnable via edit_video
|
|
1873
|
+
Analyzes input clips, segments narrative beats, calculates cut in/out timestamps, assigns transitions, speed ramps, color grades, and audio cuts, and writes:
|
|
1874
|
+
1. edit-plan.md — Human & agent readable decision list with rationale per cut, story beat breakdown, J/L-cut annotations, and director notes.
|
|
1875
|
+
2. edit.json — Machine-executable configuration directly runnable via edit_video.
|
|
1876
|
+
3. TREATMENT.md — Director's treatment document (8 sections: concept, tone, visual language, sound, narrative arc, shot list, delivery, notes).
|
|
1877
|
+
|
|
1878
|
+
J/L-cut support (lemo-opuscar DIRECTOR.md):
|
|
1879
|
+
J-cut: audio of the NEXT clip starts before its video cut (audio_offset < 0, e.g. -0.5s = audio precedes by 500ms)
|
|
1880
|
+
L-cut: current clip's audio extends over the next clip's video (audio_offset > 0 on next clip)
|
|
1881
|
+
|
|
1882
|
+
Professional audio rules applied:
|
|
1883
|
+
- First and last cuts always fade audio (no hard audio cuts at start/end)
|
|
1884
|
+
- Dialogue scenes: J-cut by 0.25–0.5s to soften edit
|
|
1885
|
+
- Action scenes: hard audio cut or L-cut to preserve momentum
|
|
1886
|
+
- Music-driven edits: snap cut points to speech pauses`,
|
|
1704
1887
|
inputSchema: {
|
|
1705
1888
|
type: 'object',
|
|
1706
1889
|
required: ['footage'],
|
|
@@ -1712,19 +1895,26 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1712
1895
|
style: { type: 'string', enum: ['cinematic', 'fast_paced', 'documentary', 'vintage', 'cyberpunk', 'vibrant'], default: 'cinematic' },
|
|
1713
1896
|
audio_path: { type: 'string', description: 'Optional master voiceover narration audio path' },
|
|
1714
1897
|
music_path: { type: 'string', description: 'Optional background music track path' },
|
|
1715
|
-
output_dir: { type: 'string', description: 'Directory to write edit-plan.md and
|
|
1898
|
+
output_dir: { type: 'string', description: 'Directory to write edit-plan.md, edit.json and TREATMENT.md' },
|
|
1899
|
+
bpm: { type: 'number', description: 'Optional music BPM — when set, cut points snap to beat grid (HyperFrames beatAt formula)' },
|
|
1900
|
+
generate_treatment: { type: 'boolean', default: true, description: 'Generate TREATMENT.md director\'s document alongside edit-plan.md' },
|
|
1716
1901
|
},
|
|
1717
1902
|
},
|
|
1718
|
-
handler: async ({ footage, goal = 'Create a dynamic polished montage', target_duration, orientation = 'landscape', style = 'cinematic', audio_path, music_path, output_dir }) => {
|
|
1903
|
+
handler: async ({ footage, goal = 'Create a dynamic polished montage', target_duration, orientation = 'landscape', style = 'cinematic', audio_path, music_path, output_dir, bpm, generate_treatment = true }) => {
|
|
1719
1904
|
const syncFs = require('fs');
|
|
1720
1905
|
const pathMod = require('path');
|
|
1721
1906
|
const { VideoEditor } = require('../video-editor/editor');
|
|
1907
|
+
const { isWebUrl, downloadMediaWithYtDlp } = require('../utils/media-downloader');
|
|
1908
|
+
const { detectPausesWithFFmpeg, snapCutToPause } = require('../utils/transcribe');
|
|
1722
1909
|
const editor = new VideoEditor();
|
|
1723
1910
|
|
|
1724
|
-
// 1. Resolve clip paths
|
|
1911
|
+
// 1. Resolve clip paths (local files, directory, or URLs via yt-dlp)
|
|
1725
1912
|
let clipPaths = [];
|
|
1726
1913
|
if (typeof footage === 'string') {
|
|
1727
|
-
if (
|
|
1914
|
+
if (isWebUrl(footage)) {
|
|
1915
|
+
const dl = downloadMediaWithYtDlp(footage);
|
|
1916
|
+
clipPaths = [dl.localPath];
|
|
1917
|
+
} else if (syncFs.existsSync(footage) && syncFs.statSync(footage).isDirectory()) {
|
|
1728
1918
|
const validExts = ['.mp4', '.mov', '.mkv', '.webm', '.avi'];
|
|
1729
1919
|
clipPaths = syncFs.readdirSync(footage)
|
|
1730
1920
|
.filter(f => validExts.includes(pathMod.extname(f).toLowerCase()))
|
|
@@ -1733,7 +1923,14 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1733
1923
|
clipPaths = [footage];
|
|
1734
1924
|
}
|
|
1735
1925
|
} else if (Array.isArray(footage)) {
|
|
1736
|
-
|
|
1926
|
+
for (const item of footage) {
|
|
1927
|
+
if (isWebUrl(item)) {
|
|
1928
|
+
const dl = downloadMediaWithYtDlp(item);
|
|
1929
|
+
clipPaths.push(dl.localPath);
|
|
1930
|
+
} else {
|
|
1931
|
+
clipPaths.push(item);
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1737
1934
|
}
|
|
1738
1935
|
|
|
1739
1936
|
if (clipPaths.length === 0) {
|
|
@@ -1752,12 +1949,14 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1752
1949
|
probedClips.push({ path: cp, ...meta });
|
|
1753
1950
|
}
|
|
1754
1951
|
|
|
1755
|
-
// 3. Audio & Duration planning
|
|
1952
|
+
// 3. Audio & Duration planning (with speech pause detection)
|
|
1756
1953
|
let totalTarget = target_duration;
|
|
1757
|
-
|
|
1954
|
+
let speechPauses = [];
|
|
1955
|
+
if (audio_path && syncFs.existsSync(audio_path)) {
|
|
1758
1956
|
try {
|
|
1759
1957
|
const aMeta = editor.probeMedia(audio_path);
|
|
1760
|
-
totalTarget = aMeta.duration;
|
|
1958
|
+
if (!totalTarget) totalTarget = aMeta.duration;
|
|
1959
|
+
speechPauses = detectPausesWithFFmpeg(audio_path);
|
|
1761
1960
|
} catch (_) {}
|
|
1762
1961
|
}
|
|
1763
1962
|
if (!totalTarget) {
|
|
@@ -1766,40 +1965,114 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1766
1965
|
|
|
1767
1966
|
// 4. Style configuration mapping
|
|
1768
1967
|
const styleConfig = {
|
|
1769
|
-
cinematic:
|
|
1770
|
-
fast_paced:
|
|
1771
|
-
documentary:
|
|
1772
|
-
vintage:
|
|
1773
|
-
cyberpunk:
|
|
1774
|
-
vibrant:
|
|
1968
|
+
cinematic: { avgDuration: 5.0, speed: 1.0, transition: 'dissolve', grade: 'cinematic' },
|
|
1969
|
+
fast_paced: { avgDuration: 2.2, speed: 1.25, transition: 'wipeleft', grade: 'vibrant' },
|
|
1970
|
+
documentary: { avgDuration: 4.5, speed: 1.0, transition: 'fade', grade: 'warm' },
|
|
1971
|
+
vintage: { avgDuration: 3.5, speed: 0.95, transition: 'fade', grade: 'vintage' },
|
|
1972
|
+
cyberpunk: { avgDuration: 2.5, speed: 1.2, transition: 'circleopen', grade: 'cyberpunk' },
|
|
1973
|
+
vibrant: { avgDuration: 3.0, speed: 1.1, transition: 'slideup', grade: 'vibrant' },
|
|
1974
|
+
product_launch: { avgDuration: 4.0, speed: 1.0, transition: 'dissolve', grade: 'product_launch' },
|
|
1975
|
+
editorial_collage: { avgDuration: 2.8, speed: 1.15, transition: 'slideleft', grade: 'editorial_collage' },
|
|
1976
|
+
retro_vhs: { avgDuration: 3.2, speed: 0.95, transition: 'pixelize', grade: 'retrovhs' },
|
|
1977
|
+
anime_action: { avgDuration: 1.8, speed: 1.3, transition: 'wipeleft', grade: 'anime_action' },
|
|
1978
|
+
sci_fi_hud: { avgDuration: 3.0, speed: 1.1, transition: 'circlecrop', grade: 'scifi' },
|
|
1979
|
+
lofi_chill: { avgDuration: 5.5, speed: 0.9, transition: 'dissolve', grade: 'lofi_chill' },
|
|
1980
|
+
minimal_monochrome: { avgDuration: 3.5, speed: 1.0, transition: 'fade', grade: 'swiss_mono' },
|
|
1981
|
+
// lemo-opuscar extended styles
|
|
1982
|
+
cinematic_anamorphic: { avgDuration: 6.0, speed: 0.95, transition: 'dissolve', grade: 'cinematic_anamorphic' },
|
|
1983
|
+
broadcast_news: { avgDuration: 3.0, speed: 1.0, transition: 'wipeleft', grade: 'broadcast_news' },
|
|
1984
|
+
blueprint_3d: { avgDuration: 4.0, speed: 1.0, transition: 'distance', grade: 'blueprint' },
|
|
1985
|
+
film_16mm: { avgDuration: 4.5, speed: 0.9, transition: 'fade', grade: 'film16mm' },
|
|
1986
|
+
product_obsidian: { avgDuration: 4.5, speed: 1.0, transition: 'fadeblack', grade: 'product_obsidian' },
|
|
1987
|
+
retro_vhs_80s: { avgDuration: 3.0, speed: 0.95, transition: 'pixelize', grade: 'retro_vhs_80s' },
|
|
1775
1988
|
}[style] || { avgDuration: 4.0, speed: 1.0, transition: 'fade', grade: 'cinematic' };
|
|
1776
1989
|
|
|
1777
|
-
// 5. Build Cut Decision List
|
|
1990
|
+
// 5. Build Cut Decision List (pause snapping + beat grid + J/L-cut assignment)
|
|
1991
|
+
// J/L-cut rules (lemo-opuscar DIRECTOR.md):
|
|
1992
|
+
// J-cut: audio_offset < 0 (next audio starts before video cut)
|
|
1993
|
+
// L-cut: audio_offset > 0 (current audio extends over next video)
|
|
1994
|
+
// Hard cut: audio_offset = 0
|
|
1778
1995
|
const clipCount = Math.max(1, Math.min(probedClips.length, Math.ceil(totalTarget / styleConfig.avgDuration)));
|
|
1779
1996
|
const segDur = totalTarget / clipCount;
|
|
1780
1997
|
const plannedClips = [];
|
|
1781
1998
|
|
|
1782
1999
|
const beatNames = ['Opening Hook', 'Context & Build-up', 'Core Revelation / Climax', 'Resolution & Outro'];
|
|
1783
2000
|
|
|
2001
|
+
// Beat-grid helper (HyperFrames formula — round each index independently)
|
|
2002
|
+
const beatAt = (idx) => bpm > 0 ? Math.round(idx * 30 * 60 / bpm) / 30 : null;
|
|
2003
|
+
|
|
2004
|
+
// Determine J/L-cut style from overall style
|
|
2005
|
+
const audioEditStyle = {
|
|
2006
|
+
cinematic: 'j_cut',
|
|
2007
|
+
fast_paced: 'hard',
|
|
2008
|
+
documentary: 'j_cut',
|
|
2009
|
+
vintage: 'l_cut',
|
|
2010
|
+
cyberpunk: 'hard',
|
|
2011
|
+
vibrant: 'hard',
|
|
2012
|
+
cinematic_anamorphic: 'j_cut',
|
|
2013
|
+
broadcast_news: 'hard',
|
|
2014
|
+
film_16mm: 'l_cut',
|
|
2015
|
+
lofi_chill: 'l_cut',
|
|
2016
|
+
anime_action: 'hard',
|
|
2017
|
+
}[style] || 'j_cut';
|
|
2018
|
+
|
|
2019
|
+
let cumulativeTime = 0;
|
|
1784
2020
|
for (let i = 0; i < clipCount; i++) {
|
|
1785
2021
|
const src = probedClips[i % probedClips.length];
|
|
1786
2022
|
const clipDur = Number(segDur.toFixed(2));
|
|
1787
2023
|
const maxStart = Math.max(0, (src.duration || 10) - clipDur);
|
|
1788
|
-
|
|
1789
|
-
|
|
2024
|
+
let startTime = Number((maxStart * 0.25 * (i % 3)).toFixed(2));
|
|
2025
|
+
let rawEndTime = Number((startTime + clipDur).toFixed(2));
|
|
2026
|
+
cumulativeTime += clipDur;
|
|
2027
|
+
|
|
2028
|
+
// Snap cut to audio pause if speech pauses available
|
|
2029
|
+
if (speechPauses.length > 0) {
|
|
2030
|
+
const snappedTime = snapCutToPause(cumulativeTime, speechPauses, 0.75);
|
|
2031
|
+
const shift = snappedTime - cumulativeTime;
|
|
2032
|
+
rawEndTime = Math.max(startTime + 0.5, Number((rawEndTime + shift).toFixed(2)));
|
|
2033
|
+
}
|
|
2034
|
+
|
|
2035
|
+
// Snap cut to beat grid if BPM provided (HyperFrames formula)
|
|
2036
|
+
const beatT = beatAt(i);
|
|
2037
|
+
if (beatT !== null && Math.abs(beatT - cumulativeTime) < segDur * 0.25) {
|
|
2038
|
+
const shift = beatT - cumulativeTime;
|
|
2039
|
+
rawEndTime = Math.max(startTime + 0.5, Number((rawEndTime + shift).toFixed(2)));
|
|
2040
|
+
}
|
|
2041
|
+
|
|
1790
2042
|
const beatIndex = Math.min(beatNames.length - 1, Math.floor((i / clipCount) * beatNames.length));
|
|
1791
2043
|
|
|
2044
|
+
// Assign J/L-cut type and audio_offset
|
|
2045
|
+
let cutType = audioEditStyle;
|
|
2046
|
+
let audioOffset = 0;
|
|
2047
|
+
if (i === 0) {
|
|
2048
|
+
// First cut: always fade in audio — no J/L on first clip
|
|
2049
|
+
cutType = 'hard';
|
|
2050
|
+
audioOffset = 0;
|
|
2051
|
+
} else if (i === clipCount - 1) {
|
|
2052
|
+
// Last cut: always fade out — L-cut slightly to smooth ending
|
|
2053
|
+
cutType = 'l_cut';
|
|
2054
|
+
audioOffset = 0.3;
|
|
2055
|
+
} else if (cutType === 'j_cut') {
|
|
2056
|
+
// J-cut: next audio starts 0.25–0.5s before video cut
|
|
2057
|
+
audioOffset = -(styleConfig.avgDuration > 4.0 ? 0.5 : 0.25);
|
|
2058
|
+
} else if (cutType === 'l_cut') {
|
|
2059
|
+
// L-cut: current audio extends 0.3s over next clip's video
|
|
2060
|
+
audioOffset = 0.3;
|
|
2061
|
+
}
|
|
2062
|
+
|
|
1792
2063
|
plannedClips.push({
|
|
1793
2064
|
video_path: src.path,
|
|
1794
2065
|
start_time: startTime,
|
|
1795
|
-
end_time:
|
|
1796
|
-
duration:
|
|
2066
|
+
end_time: rawEndTime,
|
|
2067
|
+
duration: Number((rawEndTime - startTime).toFixed(2)),
|
|
1797
2068
|
speed: styleConfig.speed,
|
|
1798
2069
|
color_grade: styleConfig.grade,
|
|
1799
2070
|
transition: i === clipCount - 1 ? 'none' : styleConfig.transition,
|
|
1800
2071
|
transition_duration: 0.5,
|
|
1801
2072
|
beat: beatNames[beatIndex],
|
|
1802
|
-
|
|
2073
|
+
cut_type: cutType,
|
|
2074
|
+
audio_offset: audioOffset,
|
|
2075
|
+
rationale: `${beatNames[beatIndex]} — ${clipDur}s, ${styleConfig.grade} grade, ${styleConfig.speed}x speed. Cut: ${cutType === 'j_cut' ? `J-cut (audio precedes by ${Math.abs(audioOffset)}s)` : cutType === 'l_cut' ? `L-cut (audio extends ${audioOffset}s)` : 'hard cut'}.${beatT !== null ? ` Beat-snapped to ${beatT.toFixed(2)}s.` : ''}`,
|
|
1803
2076
|
});
|
|
1804
2077
|
}
|
|
1805
2078
|
|
|
@@ -1813,44 +2086,730 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1813
2086
|
color_grade: styleConfig.grade,
|
|
1814
2087
|
transition: styleConfig.transition,
|
|
1815
2088
|
target_duration: totalTarget,
|
|
2089
|
+
bpm: bpm || null,
|
|
2090
|
+
audio_edit_style: audioEditStyle,
|
|
1816
2091
|
};
|
|
1817
2092
|
|
|
1818
2093
|
// 7. Build edit-plan.md
|
|
1819
2094
|
let planMd = `# 🎬 Video Edit Decision Plan\n\n`;
|
|
1820
2095
|
planMd += `**Goal**: ${goal}\n`;
|
|
1821
|
-
planMd += `**Style**: ${style.toUpperCase()} | **Orientation**: ${orientation.toUpperCase()} | **Target Duration**: ${totalTarget.toFixed(1)}s | **Clips
|
|
1822
|
-
planMd +=
|
|
1823
|
-
planMd +=
|
|
1824
|
-
planMd += `|
|
|
2096
|
+
planMd += `**Style**: ${style.toUpperCase()} | **Orientation**: ${orientation.toUpperCase()} | **Target Duration**: ${totalTarget.toFixed(1)}s | **Clips**: ${plannedClips.length} | **Audio Edit**: ${audioEditStyle.toUpperCase().replace('_', '-')}\n`;
|
|
2097
|
+
if (bpm) planMd += `**BPM**: ${bpm} (cut points beat-snapped via HyperFrames formula)\n`;
|
|
2098
|
+
planMd += `\n## 📋 Narrative Beat Architecture\n\n`;
|
|
2099
|
+
planMd += `| Beat | # | Clip | In–Out | Dur | Speed | Grade | Transition | Cut Type | Rationale |\n`;
|
|
2100
|
+
planMd += `| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n`;
|
|
1825
2101
|
|
|
1826
2102
|
plannedClips.forEach((c, idx) => {
|
|
1827
2103
|
const basename = pathMod.basename(c.video_path);
|
|
1828
|
-
|
|
2104
|
+
const cutLabel = c.cut_type === 'j_cut'
|
|
2105
|
+
? `🎵 J-cut (${Math.abs(c.audio_offset)}s)`
|
|
2106
|
+
: c.cut_type === 'l_cut'
|
|
2107
|
+
? `🔊 L-cut (+${c.audio_offset}s)`
|
|
2108
|
+
: '✂️ Hard';
|
|
2109
|
+
planMd += `| **${c.beat}** | ${idx + 1} | \`${basename}\` | ${c.start_time}s–${c.end_time}s | ${c.duration}s | ${c.speed}x | ${c.color_grade} | ${c.transition} | ${cutLabel} | ${c.rationale} |\n`;
|
|
1829
2110
|
});
|
|
1830
2111
|
|
|
1831
2112
|
planMd += `\n## 🎵 Audio Architecture\n\n`;
|
|
1832
2113
|
planMd += `- **Master Voiceover**: ${audio_path ? `\`${pathMod.basename(audio_path)}\` (Primary, ducks music by 18dB)` : 'None specified (using clip audio)'}\n`;
|
|
1833
|
-
planMd += `- **Background Music**: ${music_path ? `\`${pathMod.basename(music_path)}\` (Ambient, volume: 0.08)` : 'None specified'}\n
|
|
2114
|
+
planMd += `- **Background Music**: ${music_path ? `\`${pathMod.basename(music_path)}\` (Ambient, volume: 0.08)` : 'None specified'}\n`;
|
|
2115
|
+
planMd += `- **Audio Edit Style**: ${audioEditStyle === 'j_cut' ? '🎵 J-cut dominant (audio leads video by 0.25–0.5s for softened edits)' : audioEditStyle === 'l_cut' ? '🔊 L-cut dominant (audio trails into next shot by 0.3s)' : '✂️ Hard cuts (audio and video synchronized)'}\n\n`;
|
|
2116
|
+
planMd += `## 🎬 Audio Cut Rules Applied\n\n`;
|
|
2117
|
+
planMd += `- First clip: always hard cut (clean opening)\n`;
|
|
2118
|
+
planMd += `- Last clip: always L-cut (+0.3s) for smooth outro fade\n`;
|
|
2119
|
+
planMd += `- J-cuts: audio of NEXT clip starts ${styleConfig.avgDuration > 4.0 ? '0.5s' : '0.25s'} before video edit point\n`;
|
|
2120
|
+
planMd += `- L-cuts: current audio extends 0.3s into next clip's video\n\n`;
|
|
1834
2121
|
planMd += `## 🚀 Execution & Approval\n\n`;
|
|
1835
|
-
planMd += `
|
|
2122
|
+
planMd += `Review this plan, then execute with \`edit_video\` using the payload in \`edit.json\`.\n`;
|
|
2123
|
+
planMd += `\nGenerated by FrameForge v2.7.0 — plan_video_edit with J/L-cut support (lemo-opuscar DIRECTOR.md)\n`;
|
|
2124
|
+
|
|
2125
|
+
// 8. Build TREATMENT.md (lemo-opuscar DIRECTOR.md — 8-section format)
|
|
2126
|
+
let treatmentMd = '';
|
|
2127
|
+
if (generate_treatment) {
|
|
2128
|
+
const clipList = plannedClips.map((c, idx) =>
|
|
2129
|
+
` ${idx + 1}. \`${pathMod.basename(c.video_path)}\` (${c.beat}, ${c.duration}s, ${c.cut_type.replace('_', '-')})`
|
|
2130
|
+
).join('\n');
|
|
2131
|
+
|
|
2132
|
+
treatmentMd = `# 🎬 TREATMENT\n\n`;
|
|
2133
|
+
treatmentMd += `> Generated by FrameForge plan_video_edit — edit this document to refine creative direction before rendering.\n\n`;
|
|
2134
|
+
treatmentMd += `---\n\n`;
|
|
2135
|
+
treatmentMd += `## 1. Concept\n\n${goal}\n\n`;
|
|
2136
|
+
treatmentMd += `## 2. Tone\n\n`;
|
|
2137
|
+
treatmentMd += `${style === 'cinematic' || style === 'cinematic_anamorphic' ? 'Cinematic and deliberate. Each frame earns its place.' : style === 'fast_paced' || style === 'anime_action' ? 'High-energy, kinetic. The edit IS the emotion.' : style === 'documentary' ? 'Observational and authentic. Let reality speak.' : style === 'vintage' || style === 'film_16mm' ? 'Warm, nostalgic, imperfect. Texture over polish.' : 'Direct and clean. Clarity over decoration.'}\n\n`;
|
|
2138
|
+
treatmentMd += `## 3. Visual Language\n\n`;
|
|
2139
|
+
treatmentMd += `- **Color**: ${styleConfig.grade} grade\n`;
|
|
2140
|
+
treatmentMd += `- **Orientation**: ${orientation}\n`;
|
|
2141
|
+
treatmentMd += `- **Transitions**: ${styleConfig.transition} (${styleConfig.avgDuration}s avg clip)\n`;
|
|
2142
|
+
treatmentMd += `- **Speed**: ${styleConfig.speed}x base${styleConfig.speed !== 1.0 ? ' (variable)' : ''}\n\n`;
|
|
2143
|
+
treatmentMd += `## 4. Sound\n\n`;
|
|
2144
|
+
treatmentMd += `- **Audio edit style**: ${audioEditStyle === 'j_cut' ? 'J-cut dominant — audio of next scene starts before video cut to soften transitions' : audioEditStyle === 'l_cut' ? 'L-cut dominant — current audio trails into next shot for continuity' : 'Hard cuts — audio and video locked to same edit point'}\n`;
|
|
2145
|
+
treatmentMd += `- **Voiceover**: ${audio_path ? pathMod.basename(audio_path) : 'None'}\n`;
|
|
2146
|
+
treatmentMd += `- **Music**: ${music_path ? pathMod.basename(music_path) : 'None'}${bpm ? ` @ ${bpm} BPM` : ''}\n`;
|
|
2147
|
+
treatmentMd += `- **Priority order** (lemo-opuscar): Sound → Rhythm → Camera → Directing\n\n`;
|
|
2148
|
+
treatmentMd += `## 5. Narrative Arc\n\n`;
|
|
2149
|
+
treatmentMd += `| Act | Beat | Description |\n| :--- | :--- | :--- |\n`;
|
|
2150
|
+
treatmentMd += `| Act 1 | Opening Hook | Establish world, hook the viewer in first 3s |\n`;
|
|
2151
|
+
treatmentMd += `| Act 2a | Context & Build-up | Deepen context, build anticipation |\n`;
|
|
2152
|
+
treatmentMd += `| Act 2b | Core Revelation / Climax | The emotional or informational peak |\n`;
|
|
2153
|
+
treatmentMd += `| Act 3 | Resolution & Outro | Resolve, call to action, or linger |\n\n`;
|
|
2154
|
+
treatmentMd += `## 6. Shot List\n\n${clipList}\n\n`;
|
|
2155
|
+
treatmentMd += `## 7. Delivery\n\n`;
|
|
2156
|
+
treatmentMd += `- **Format**: MP4, H.264 high profile, CRF 18\n`;
|
|
2157
|
+
treatmentMd += `- **Resolution**: ${orientation === 'portrait' ? '1080×1920 (9:16)' : orientation === 'square' ? '1080×1080 (1:1)' : '1920×1080 (16:9)'}\n`;
|
|
2158
|
+
treatmentMd += `- **Duration**: ${totalTarget.toFixed(1)}s\n`;
|
|
2159
|
+
treatmentMd += `- **Audio**: AAC 192kbps, 48kHz, EBU R128 normalized\n`;
|
|
2160
|
+
treatmentMd += `- **Platform target**: YouTube / Instagram (−14 LUFS, −1 dBTP)\n\n`;
|
|
2161
|
+
treatmentMd += `## 8. Director Notes\n\n`;
|
|
2162
|
+
treatmentMd += `_Add creative direction, mood references, or client notes here._\n\n`;
|
|
2163
|
+
treatmentMd += `---\n\n`;
|
|
2164
|
+
treatmentMd += `*Generated by FrameForge v2.7.0 — lemo-opuscar DIRECTOR.md format*\n`;
|
|
2165
|
+
}
|
|
1836
2166
|
|
|
1837
|
-
//
|
|
2167
|
+
// 9. Write outputs
|
|
1838
2168
|
const outDir = output_dir || pathMod.join(FILES_DIR, 'edits', `edit_${Date.now()}`);
|
|
1839
2169
|
syncFs.mkdirSync(outDir, { recursive: true });
|
|
1840
|
-
const planPath
|
|
1841
|
-
const jsonPath
|
|
2170
|
+
const planPath = pathMod.join(outDir, 'edit-plan.md');
|
|
2171
|
+
const jsonPath = pathMod.join(outDir, 'edit.json');
|
|
2172
|
+
const treatmentPath = pathMod.join(outDir, 'TREATMENT.md');
|
|
1842
2173
|
|
|
1843
2174
|
syncFs.writeFileSync(planPath, planMd, 'utf8');
|
|
1844
2175
|
syncFs.writeFileSync(jsonPath, JSON.stringify(editJson, null, 2), 'utf8');
|
|
2176
|
+
if (generate_treatment) {
|
|
2177
|
+
syncFs.writeFileSync(treatmentPath, treatmentMd, 'utf8');
|
|
2178
|
+
}
|
|
1845
2179
|
|
|
1846
2180
|
return {
|
|
1847
2181
|
planPath,
|
|
1848
2182
|
editJsonPath: jsonPath,
|
|
2183
|
+
treatmentPath: generate_treatment ? treatmentPath : null,
|
|
1849
2184
|
planMarkdown: planMd,
|
|
1850
2185
|
editConfig: editJson,
|
|
1851
2186
|
clipCount: plannedClips.length,
|
|
1852
2187
|
totalDuration: totalTarget,
|
|
1853
|
-
|
|
2188
|
+
audioEditStyle,
|
|
2189
|
+
message: `Edit plan, edit.json${generate_treatment ? ', and TREATMENT.md' : ''} generated. Review edit-plan.md, approve, then call edit_video with edit.json.`,
|
|
2190
|
+
};
|
|
2191
|
+
},
|
|
2192
|
+
},
|
|
2193
|
+
|
|
2194
|
+
// ── Tool 24: Sync Upstream Skills ─────────────────────────────────────────
|
|
2195
|
+
// Transparently checks & merges updates from upstream video skills repositories
|
|
2196
|
+
sync_upstream_skills: {
|
|
2197
|
+
description: `Check and synchronize latest upstream skills, motion craft rules, film styles, and audio presets from 9 reference repositories (HyperFrames, OneTake, LemoOpuscar, CollageBroll, Remotion, Earshot, ClaudeAnimationBase, ClaudeRemotionSkill, FFmpegSkill). Makes network requests to GitHub API to check for new commits. Returns detailed per-category update summary. Use check_upstream_updates for a fast offline check of the last sync state.`,
|
|
2198
|
+
inputSchema: {
|
|
2199
|
+
type: 'object',
|
|
2200
|
+
properties: {
|
|
2201
|
+
silent: { type: 'boolean', default: false, description: 'Suppress verbose console logging' },
|
|
2202
|
+
},
|
|
2203
|
+
},
|
|
2204
|
+
handler: async ({ silent = false }) => {
|
|
2205
|
+
const { syncSkills } = require('../skills/sync-upstream');
|
|
2206
|
+
const res = await syncSkills({ silent });
|
|
2207
|
+
const updatedRepos = res.results.filter(r => r.updated).map(r => r.repo);
|
|
2208
|
+
const perCategoryUpdates = res.perCategoryUpdates || {};
|
|
2209
|
+
|
|
2210
|
+
// Build a rich user-facing update notice per category
|
|
2211
|
+
let updateNotice = '';
|
|
2212
|
+
if (updatedRepos.length > 0) {
|
|
2213
|
+
updateNotice = '\n\n⬆️ UPSTREAM UPDATES AVAILABLE:\n';
|
|
2214
|
+
for (const [cat, catUpdates] of Object.entries(perCategoryUpdates)) {
|
|
2215
|
+
for (const u of catUpdates) {
|
|
2216
|
+
updateNotice += ` [${cat}] ${u.repo} → ${u.sha}\n`;
|
|
2217
|
+
updateNotice += ` ${u.userImpact}\n`;
|
|
2218
|
+
if (u.capabilities.length > 0) {
|
|
2219
|
+
updateNotice += ` Tools affected: ${u.capabilities.join(', ')}\n`;
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
updateNotice += '\nNo action required — FrameForge automatically applied the updates. Your next render uses the latest techniques.';
|
|
2224
|
+
}
|
|
2225
|
+
return {
|
|
2226
|
+
success: true,
|
|
2227
|
+
updatedCount: res.updatedCount,
|
|
2228
|
+
repos: res.results,
|
|
2229
|
+
updatedRepos,
|
|
2230
|
+
perCategoryUpdates,
|
|
2231
|
+
lastSync: res.manifest.lastSync,
|
|
2232
|
+
message: `Synced ${res.results.length} upstream skill repositories (${res.updatedCount} updates merged).${updateNotice}`,
|
|
2233
|
+
};
|
|
2234
|
+
},
|
|
2235
|
+
},
|
|
2236
|
+
|
|
2237
|
+
// ── Tool 28: Check Upstream Updates (fast, offline) ───────────────────────
|
|
2238
|
+
// Reads last sync manifest without network requests — returns update status instantly
|
|
2239
|
+
check_upstream_updates: {
|
|
2240
|
+
description: `Check the last sync state of all upstream reference repositories (HyperFrames, OneTake, LemoOpuscar, CollageBroll, Remotion, Earshot, ClaudeAnimationBase, ClaudeRemotionSkill, FFmpegSkill). Fast, offline — reads the local manifest only (no network requests). Shows what was last synced, when, which repos had updates, and a human-readable changelog of what improved. To actually fetch new commits and update the manifest, use sync_upstream_skills. Agents and users can call this anytime to see if upstream skills have been updated since the last sync.`,
|
|
2241
|
+
inputSchema: { type: 'object', properties: {} },
|
|
2242
|
+
handler: async () => {
|
|
2243
|
+
const syncFs = require('fs');
|
|
2244
|
+
const { MANIFEST_PATH, REPOSITORIES } = require('../skills/sync-upstream');
|
|
2245
|
+
let manifest = { lastSync: null, repos: {}, version: '2.7.0', pendingUpdates: [] };
|
|
2246
|
+
if (syncFs.existsSync(MANIFEST_PATH)) {
|
|
2247
|
+
try { manifest = JSON.parse(syncFs.readFileSync(MANIFEST_PATH, 'utf8')); } catch (_) {}
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
const repos = REPOSITORIES.map(r => {
|
|
2251
|
+
const m = manifest.repos[r.id] || {};
|
|
2252
|
+
return {
|
|
2253
|
+
id: r.id,
|
|
2254
|
+
repo: r.repo,
|
|
2255
|
+
category: r.category,
|
|
2256
|
+
desc: r.desc,
|
|
2257
|
+
capabilities: r.capabilities || [],
|
|
2258
|
+
sha: m.sha || 'not-synced',
|
|
2259
|
+
lastChecked: m.lastChecked || null,
|
|
2260
|
+
status: m.status || 'unknown',
|
|
2261
|
+
userImpact: r.userImpact,
|
|
2262
|
+
};
|
|
2263
|
+
});
|
|
2264
|
+
|
|
2265
|
+
const syncedCount = repos.filter(r => r.sha !== 'not-synced').length;
|
|
2266
|
+
const neverSynced = repos.filter(r => r.sha === 'not-synced').map(r => r.repo);
|
|
2267
|
+
const pendingUpdates = manifest.pendingUpdates || [];
|
|
2268
|
+
|
|
2269
|
+
// Build human-readable changelog notice
|
|
2270
|
+
let changelogNotice = '';
|
|
2271
|
+
if (pendingUpdates.length > 0) {
|
|
2272
|
+
changelogNotice = '\n\n📦 UPSTREAM CHANGES SINCE LAST SYNC:\n';
|
|
2273
|
+
for (const u of pendingUpdates) {
|
|
2274
|
+
changelogNotice += ` • [${u.category || '?'}] ${u.repo} → ${u.sha}\n`;
|
|
2275
|
+
changelogNotice += ` ${u.userImpact || 'Updated techniques merged'}\n`;
|
|
2276
|
+
}
|
|
2277
|
+
changelogNotice += '\nRun sync_upstream_skills to pull these updates and apply them to your next render.';
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
let statusMsg = `Last sync: ${manifest.lastSync || 'never'}. ${syncedCount}/${repos.length} repos tracked.`;
|
|
2281
|
+
if (neverSynced.length > 0) {
|
|
2282
|
+
statusMsg += ` Not yet synced: ${neverSynced.join(', ')}. Run sync_upstream_skills to fetch latest.`;
|
|
2283
|
+
} else if (pendingUpdates.length > 0) {
|
|
2284
|
+
statusMsg += ` ${pendingUpdates.length} upstream repo(s) have new commits. Run sync_upstream_skills to apply them.${changelogNotice}`;
|
|
2285
|
+
} else {
|
|
2286
|
+
statusMsg += ' All repos are current. Run sync_upstream_skills to check for new commits from GitHub.';
|
|
2287
|
+
}
|
|
2288
|
+
|
|
2289
|
+
return {
|
|
2290
|
+
success: true,
|
|
2291
|
+
lastSync: manifest.lastSync,
|
|
2292
|
+
manifestVersion: manifest.version || '2.7.0',
|
|
2293
|
+
totalRepos: repos.length,
|
|
2294
|
+
syncedCount,
|
|
2295
|
+
pendingUpdates,
|
|
2296
|
+
hasUpdates: pendingUpdates.length > 0,
|
|
2297
|
+
repos,
|
|
2298
|
+
message: statusMsg,
|
|
2299
|
+
hint: pendingUpdates.length > 0
|
|
2300
|
+
? `⬆️ ${pendingUpdates.length} upstream update(s) available — run sync_upstream_skills to apply`
|
|
2301
|
+
: 'Run sync_upstream_skills to check for new commits from GitHub.',
|
|
2302
|
+
};
|
|
2303
|
+
},
|
|
2304
|
+
},
|
|
2305
|
+
|
|
2306
|
+
// ── Tool 25: Transcribe Media ──────────────────────────────────────────────
|
|
2307
|
+
// Whisper speech transcription — word-level timestamps, SRT/NDJSON export
|
|
2308
|
+
transcribe_media: {
|
|
2309
|
+
description: `Transcribe speech from any audio or video file using OpenAI Whisper (local, free, no API key required).
|
|
2310
|
+
Returns word-level timestamps for precise animation sync, SRT subtitles for closed captions, and silence/pause boundaries for intelligent cut alignment.
|
|
2311
|
+
Supports: openai-whisper (local GPU/CPU), faster-whisper (CTranslate2), whisper-cli binary, or OpenAI Whisper API fallback.
|
|
2312
|
+
Output formats: ndjson (one segment per line — used by engine.js captions()), srt (standard subtitle file), json (full word/segment data).
|
|
2313
|
+
Use before writing edit.js to get cuePhrase() timestamps. Use on user-supplied footage to get cut points aligned to speech pauses.`,
|
|
2314
|
+
inputSchema: {
|
|
2315
|
+
type: 'object',
|
|
2316
|
+
required: ['mediaPath'],
|
|
2317
|
+
properties: {
|
|
2318
|
+
mediaPath: { type: 'string', description: 'Absolute path to audio or video file (mp4, mp3, m4a, wav, mov, webm)' },
|
|
2319
|
+
model: { type: 'string', default: 'base', enum: ['tiny', 'base', 'small', 'medium', 'large'], description: 'Whisper model size (base = fast, small/medium = accurate)' },
|
|
2320
|
+
language: { type: 'string', description: 'BCP-47 language code hint (e.g. "en", "ml", "hi"). Auto-detected if omitted.' },
|
|
2321
|
+
format: { type: 'string', default: 'ndjson', enum: ['ndjson', 'srt', 'json', 'all'], description: 'Output format: ndjson for engine.js captions(), srt for subtitle files, json for full data, all for everything' },
|
|
2322
|
+
outputPath: { type: 'string', description: 'Path to write output file. Auto-generated in .frameforge-files/transcripts/ if omitted.' },
|
|
2323
|
+
},
|
|
2324
|
+
},
|
|
2325
|
+
handler: async ({ mediaPath, model = 'base', language, format = 'ndjson', outputPath }) => {
|
|
2326
|
+
const { transcribeAudio, detectWhisper } = require('../utils/transcribe');
|
|
2327
|
+
const syncFs = require('fs');
|
|
2328
|
+
const pathMod = require('path');
|
|
2329
|
+
|
|
2330
|
+
if (!syncFs.existsSync(mediaPath)) {
|
|
2331
|
+
throw new Error(`Media file not found: ${mediaPath}`);
|
|
2332
|
+
}
|
|
2333
|
+
|
|
2334
|
+
const engine = detectWhisper();
|
|
2335
|
+
if (!engine.available) {
|
|
2336
|
+
throw new Error(
|
|
2337
|
+
'No Whisper engine detected. Install one of:\n' +
|
|
2338
|
+
' pip install openai-whisper (local, free)\n' +
|
|
2339
|
+
' pip install faster-whisper (faster, local)\n' +
|
|
2340
|
+
' Or set OPENAI_API_KEY in .env for cloud fallback.'
|
|
2341
|
+
);
|
|
2342
|
+
}
|
|
2343
|
+
|
|
2344
|
+
const result = await transcribeAudio(mediaPath, { model, language });
|
|
2345
|
+
|
|
2346
|
+
const transcriptsDir = pathMod.join(FILES_DIR, 'transcripts');
|
|
2347
|
+
syncFs.mkdirSync(transcriptsDir, { recursive: true });
|
|
2348
|
+
|
|
2349
|
+
const basename = pathMod.basename(mediaPath, pathMod.extname(mediaPath));
|
|
2350
|
+
const outputs = {};
|
|
2351
|
+
|
|
2352
|
+
// NDJSON — used by engine.js captions() and cuePhrase()
|
|
2353
|
+
if (format === 'ndjson' || format === 'all') {
|
|
2354
|
+
const ndjsonPath = outputPath && format !== 'all'
|
|
2355
|
+
? outputPath
|
|
2356
|
+
: pathMod.join(transcriptsDir, `${basename}.captions.ndjson`);
|
|
2357
|
+
const segments = result.segments.length > 0 ? result.segments : result.words.map(w => ({ start: w.start, end: w.end, text: w.word }));
|
|
2358
|
+
const ndjson = segments.map(s => JSON.stringify({ start: s.start, end: s.end, text: s.text })).join('\n');
|
|
2359
|
+
syncFs.writeFileSync(ndjsonPath, ndjson, 'utf8');
|
|
2360
|
+
outputs.ndjson = ndjsonPath;
|
|
2361
|
+
}
|
|
2362
|
+
|
|
2363
|
+
// SRT — standard subtitle file
|
|
2364
|
+
if (format === 'srt' || format === 'all') {
|
|
2365
|
+
const srtPath = outputPath && format !== 'all'
|
|
2366
|
+
? outputPath
|
|
2367
|
+
: pathMod.join(transcriptsDir, `${basename}.srt`);
|
|
2368
|
+
const srtLines = [];
|
|
2369
|
+
const segs = result.segments.length > 0 ? result.segments : [];
|
|
2370
|
+
segs.forEach((s, i) => {
|
|
2371
|
+
const toSrtTime = (sec) => {
|
|
2372
|
+
const h = Math.floor(sec / 3600).toString().padStart(2, '0');
|
|
2373
|
+
const m = Math.floor((sec % 3600) / 60).toString().padStart(2, '0');
|
|
2374
|
+
const ss = Math.floor(sec % 60).toString().padStart(2, '0');
|
|
2375
|
+
const ms = Math.round((sec % 1) * 1000).toString().padStart(3, '0');
|
|
2376
|
+
return `${h}:${m}:${ss},${ms}`;
|
|
2377
|
+
};
|
|
2378
|
+
srtLines.push(`${i + 1}\n${toSrtTime(s.start)} --> ${toSrtTime(s.end)}\n${s.text}\n`);
|
|
2379
|
+
});
|
|
2380
|
+
syncFs.writeFileSync(srtPath, srtLines.join('\n'), 'utf8');
|
|
2381
|
+
outputs.srt = srtPath;
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
// JSON — full structured data
|
|
2385
|
+
if (format === 'json' || format === 'all') {
|
|
2386
|
+
const jsonPath = outputPath && format !== 'all'
|
|
2387
|
+
? outputPath
|
|
2388
|
+
: pathMod.join(transcriptsDir, `${basename}.transcript.json`);
|
|
2389
|
+
syncFs.writeFileSync(jsonPath, JSON.stringify(result, null, 2), 'utf8');
|
|
2390
|
+
outputs.json = jsonPath;
|
|
2391
|
+
}
|
|
2392
|
+
|
|
2393
|
+
return {
|
|
2394
|
+
engine: engine.engine,
|
|
2395
|
+
model,
|
|
2396
|
+
language: language || 'auto-detected',
|
|
2397
|
+
text: result.text,
|
|
2398
|
+
duration: result.duration,
|
|
2399
|
+
wordCount: result.words.length,
|
|
2400
|
+
segmentCount: result.segments.length,
|
|
2401
|
+
pauseCount: result.pauses.length,
|
|
2402
|
+
pauses: result.pauses.slice(0, 10), // first 10 pauses for cut alignment
|
|
2403
|
+
outputs,
|
|
2404
|
+
message: `Transcribed ${pathMod.basename(mediaPath)} (${result.duration.toFixed(1)}s, ${result.words.length} words) via ${engine.engine}. Load in edit.js: captions('${outputs.ndjson || outputs.json || ''}')`,
|
|
2405
|
+
};
|
|
2406
|
+
},
|
|
2407
|
+
},
|
|
2408
|
+
|
|
2409
|
+
// ── Tool 26: Ingest URL ────────────────────────────────────────────────────
|
|
2410
|
+
// Download video/audio from any web URL (YouTube, Vimeo, Twitter, direct MP4)
|
|
2411
|
+
ingest_url: {
|
|
2412
|
+
description: `Download video or audio from any web URL to a local file using yt-dlp.
|
|
2413
|
+
Supports YouTube, Vimeo, Twitter/X, Instagram, TikTok, Twitch, SoundCloud, direct MP4/MP3 URLs, and 1000+ other sites.
|
|
2414
|
+
The downloaded file can then be passed to edit_video (video_path), multi_voice_tts (audio_path), transcribe_media, or any scene's video_path / audio_path field.
|
|
2415
|
+
Requires yt-dlp: pip install yt-dlp (or: winget install yt-dlp)`,
|
|
2416
|
+
inputSchema: {
|
|
2417
|
+
type: 'object',
|
|
2418
|
+
required: ['url'],
|
|
2419
|
+
properties: {
|
|
2420
|
+
url: { type: 'string', description: 'Web URL to download (YouTube, Vimeo, Twitter, direct MP4, etc.)' },
|
|
2421
|
+
format: { type: 'string', default: 'video', enum: ['video', 'audio'], description: 'Download video (mp4) or audio-only (mp3)' },
|
|
2422
|
+
outputDir: { type: 'string', description: 'Directory to save downloaded file. Defaults to .frameforge-files/downloads/' },
|
|
2423
|
+
filename: { type: 'string', description: 'Output filename without extension. Auto-generated from URL hash if omitted.' },
|
|
2424
|
+
maxDuration:{ type: 'number', default: 600, description: 'Maximum allowed duration in seconds (safety clamp, default 10 min)' },
|
|
2425
|
+
subtitles: { type: 'boolean', default: false, description: 'Also download subtitles / closed captions as SRT file' },
|
|
2426
|
+
subLang: { type: 'string', default: 'en', description: 'Subtitle language code (default: "en")' },
|
|
2427
|
+
},
|
|
2428
|
+
},
|
|
2429
|
+
handler: async ({ url, format = 'video', outputDir, filename, maxDuration = 600, subtitles = false, subLang = 'en' }) => {
|
|
2430
|
+
const { downloadMediaWithYtDlp, detectYtDlp, isWebUrl } = require('../utils/media-downloader');
|
|
2431
|
+
|
|
2432
|
+
if (!isWebUrl(url)) {
|
|
2433
|
+
throw new Error(`Invalid URL: ${url}. Must be a valid http:// or https:// URL.`);
|
|
2434
|
+
}
|
|
2435
|
+
if (!detectYtDlp()) {
|
|
2436
|
+
throw new Error(
|
|
2437
|
+
'yt-dlp not found in PATH. Install it:\n' +
|
|
2438
|
+
' pip install yt-dlp\n' +
|
|
2439
|
+
' winget install yt-dlp (Windows)\n' +
|
|
2440
|
+
' brew install yt-dlp (macOS)\n' +
|
|
2441
|
+
' apt install yt-dlp (Debian/Ubuntu)'
|
|
2442
|
+
);
|
|
2443
|
+
}
|
|
2444
|
+
|
|
2445
|
+
const result = downloadMediaWithYtDlp(url, { format, outputDir, filename, maxDuration, subtitles, subLang });
|
|
2446
|
+
|
|
2447
|
+
return {
|
|
2448
|
+
success: true,
|
|
2449
|
+
localPath: result.localPath,
|
|
2450
|
+
subtitlePath: result.subtitlePath || null,
|
|
2451
|
+
format: result.format,
|
|
2452
|
+
duration: result.duration,
|
|
2453
|
+
width: result.width,
|
|
2454
|
+
height: result.height,
|
|
2455
|
+
url,
|
|
2456
|
+
message: `Downloaded ${format === 'audio' ? 'audio' : 'video'} to ${result.localPath} (${result.duration ? result.duration.toFixed(1) + 's' : 'duration unknown'}). Pass localPath to edit_video clips, transcribe_media, or scene video_path / audio_path fields.`,
|
|
2457
|
+
};
|
|
2458
|
+
},
|
|
2459
|
+
},
|
|
2460
|
+
|
|
2461
|
+
// ── Tool 27: Mix Audio Pro ─────────────────────────────────────────────────
|
|
2462
|
+
// Professional audio post-production: EQ, de-esser, reverb, stereo imaging, SFX
|
|
2463
|
+
mix_audio_pro: {
|
|
2464
|
+
description: `Professional audio post-production mixing — 15+ year sound engineer signal chain.
|
|
2465
|
+
|
|
2466
|
+
FULL CHAIN (in order):
|
|
2467
|
+
1. Noise gate: agate -60dBFS threshold — removes breath/room tone between words (enable with noiseGate: true)
|
|
2468
|
+
2. Pre-EQ: 80Hz HPF (remove proximity/rumble), 5kHz presence boost (+2dB), 12kHz air shelf (+1.5dB)
|
|
2469
|
+
3. De-esser: 6.5kHz dynamic suppression (-3dB) — removes harshness from "s", "sh", "ch"
|
|
2470
|
+
4. Voice compressor: 3:1 ratio, -20dBFS threshold, 5ms attack, 50ms release (broadcast standard)
|
|
2471
|
+
5. Room acoustics: 5 named profiles — studio_dry | broadcast | cinematic | concert_hall | intimate
|
|
2472
|
+
OR earshot forgeComputedRoom: "computed:sizeM:damp:earlyMs" (e.g. "computed:8:0.5:15" = 8m studio)
|
|
2473
|
+
(addReverb: true = cinematic profile; roomProfile overrides)
|
|
2474
|
+
6. Sidechain ducking: music ducks -18dB when voice active (40ms perceptual lead)
|
|
2475
|
+
7. Stereo widener: M/S extrastereo 1.4x on music layer only (voice stays mono-center)
|
|
2476
|
+
8. SFX timeline mixer: 13 synthesis types with millisecond accuracy
|
|
2477
|
+
Types: whoosh | impact | pop | click | riser | chime | buzz | tick | swish | punch | sparkle | deep_boom | transition
|
|
2478
|
+
9. Master bus: two-pass EBU R128 loudnorm at target LUFS, true peak ≤-1 dBTP
|
|
2479
|
+
Platform presets: youtube (-14) | podcast (-16) | broadcast (-23) | netflix (-27) | instagram (-14)
|
|
2480
|
+
|
|
2481
|
+
Outputs broadcast-ready 48kHz/192k AAC stereo.`,
|
|
2482
|
+
inputSchema: {
|
|
2483
|
+
type: 'object',
|
|
2484
|
+
required: ['voicePath', 'outputPath'],
|
|
2485
|
+
properties: {
|
|
2486
|
+
voicePath: { type: 'string', description: 'Primary narration / voice audio file (m4a, mp3, wav)' },
|
|
2487
|
+
outputPath: { type: 'string', description: 'Output mixed audio file path (.m4a)' },
|
|
2488
|
+
musicPath: { type: 'string', description: 'Optional background music track. Will be looped, high-passed, and sidechain ducked.' },
|
|
2489
|
+
musicVolume: { type: 'number', default: 0.12, description: 'Music volume multiplier before ducking (0.0–1.0, default 0.12)' },
|
|
2490
|
+
duckLevel: { type: 'number', default: -18, description: 'Music attenuation in dB when voice is active (default -18dB)' },
|
|
2491
|
+
sfxCues: { type: 'array', description: 'Sound effect cues: [{ name, time, volume }]. Names: whoosh|impact|pop|click|riser|chime|buzz|tick|swish|punch|sparkle|deep_boom|transition', items: { type: 'object' } },
|
|
2492
|
+
addReverb: { type: 'boolean', default: false, description: 'Legacy: Add cinematic room acoustics. Prefer roomProfile for precise control.' },
|
|
2493
|
+
roomProfile: { type: 'string', description: 'Room acoustics profile. Named: studio_dry|broadcast|cinematic|concert_hall|intimate. Or earshot computed room: "computed:sizeM:damp:earlyMs" (e.g. "computed:8:0.5:15" = 8m room, 50% damp, 15ms early). Overrides addReverb.' },
|
|
2494
|
+
noiseGate: { type: 'boolean', default: false, description: 'Apply noise gate (-60dBFS threshold) to remove breath noise and room tone between words. Enable for field recordings or noisy VO.' },
|
|
2495
|
+
stereoWiden: { type: 'boolean', default: true, description: 'Apply mid-side stereo widening to music layer (voice stays mono-center).' },
|
|
2496
|
+
targetLufs: { type: 'number', default: -14, description: 'Target integrated loudness. Platform presets: -14 YouTube/Instagram, -16 Podcast, -23 Broadcast EBU R128, -27 Netflix.' },
|
|
2497
|
+
platform: { type: 'string', enum: ['youtube', 'podcast', 'broadcast', 'netflix', 'instagram'], description: 'Platform preset for LUFS target. Overrides targetLufs when provided.' },
|
|
2498
|
+
videoPath: { type: 'string', description: 'If provided, muxes the mixed audio into this video file and writes to outputPath.' },
|
|
2499
|
+
},
|
|
2500
|
+
},
|
|
2501
|
+
handler: async ({ voicePath, outputPath, musicPath, musicVolume = 0.12, duckLevel = -18, sfxCues = [], addReverb = false, roomProfile, noiseGate = false, stereoWiden = true, targetLufs = -14, platform, videoPath }) => {
|
|
2502
|
+
const { spawnSync } = require('child_process');
|
|
2503
|
+
const syncFs = require('fs');
|
|
2504
|
+
const pathMod = require('path');
|
|
2505
|
+
const os = require('os');
|
|
2506
|
+
|
|
2507
|
+
if (!syncFs.existsSync(voicePath)) throw new Error(`Voice file not found: ${voicePath}`);
|
|
2508
|
+
syncFs.mkdirSync(pathMod.dirname(outputPath), { recursive: true });
|
|
2509
|
+
|
|
2510
|
+
const tmpDir = pathMod.join(os.tmpdir(), `ff_mixpro_${Date.now()}`);
|
|
2511
|
+
syncFs.mkdirSync(tmpDir, { recursive: true });
|
|
2512
|
+
|
|
2513
|
+
/**
|
|
2514
|
+
* Helper: run ffmpeg, throw on failure
|
|
2515
|
+
*/
|
|
2516
|
+
function runFfmpeg(args, label) {
|
|
2517
|
+
const r = spawnSync('ffmpeg', ['-nostdin', '-y', ...args], { encoding: 'utf8', timeout: 300000 });
|
|
2518
|
+
if (r.status !== 0) {
|
|
2519
|
+
throw new Error(`${label} failed:\n${(r.stderr || r.stdout || '').slice(-500)}`);
|
|
2520
|
+
}
|
|
2521
|
+
return r;
|
|
2522
|
+
}
|
|
2523
|
+
|
|
2524
|
+
/**
|
|
2525
|
+
* Two-pass loudnorm: measure first, then apply with measured values for accuracy
|
|
2526
|
+
*/
|
|
2527
|
+
function twoPassNorm(inPath, outPath, lufs = -14, tp = -1.0) {
|
|
2528
|
+
// Pass 1: measure
|
|
2529
|
+
const m = spawnSync('ffmpeg', [
|
|
2530
|
+
'-nostdin', '-i', inPath,
|
|
2531
|
+
'-af', `loudnorm=I=${lufs}:TP=${tp}:print_format=json`,
|
|
2532
|
+
'-f', 'null', '-',
|
|
2533
|
+
], { encoding: 'utf8', timeout: 120000 });
|
|
2534
|
+
const combined = (m.stderr || '') + (m.stdout || '');
|
|
2535
|
+
const match = combined.match(/\{[\s\S]*?"input_i"[\s\S]*?\}/);
|
|
2536
|
+
let filterStr = `loudnorm=I=${lufs}:TP=${tp}`;
|
|
2537
|
+
if (match) {
|
|
2538
|
+
try {
|
|
2539
|
+
const ln = JSON.parse(match[0]);
|
|
2540
|
+
filterStr = `loudnorm=I=${lufs}:TP=${tp}:measured_I=${ln.input_i}:measured_LRA=${ln.input_lra}:measured_TP=${ln.input_tp}:measured_thresh=${ln.input_thresh}:offset=${ln.target_offset}:linear=true`;
|
|
2541
|
+
} catch (_) {}
|
|
2542
|
+
}
|
|
2543
|
+
runFfmpeg([
|
|
2544
|
+
'-i', inPath, '-af', filterStr,
|
|
2545
|
+
'-c:a', 'aac', '-b:a', '192k', '-ar', '48000', '-ac', '2',
|
|
2546
|
+
outPath,
|
|
2547
|
+
], `Two-pass loudnorm ${pathMod.basename(inPath)}`);
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
// ── Resolve platform LUFS target ───────────────────────────────────────
|
|
2551
|
+
const PLATFORM_LUFS_LOCAL = {
|
|
2552
|
+
youtube: { lufs: -14, tp: -1.0 }, podcast: { lufs: -16, tp: -1.0 },
|
|
2553
|
+
broadcast: { lufs: -23, tp: -1.0 }, netflix: { lufs: -27, tp: -3.0 },
|
|
2554
|
+
instagram: { lufs: -14, tp: -1.0 },
|
|
2555
|
+
};
|
|
2556
|
+
const platformTarget = platform ? (PLATFORM_LUFS_LOCAL[platform] || null) : null;
|
|
2557
|
+
const resolvedLufs = platformTarget ? platformTarget.lufs : targetLufs;
|
|
2558
|
+
const resolvedTp = platformTarget ? platformTarget.tp : -1.0;
|
|
2559
|
+
|
|
2560
|
+
// ── Room acoustics profile resolution ─────────────────────────────────
|
|
2561
|
+
const ROOM_PROFILES_LOCAL = {
|
|
2562
|
+
studio_dry: { inGain: 0.8, outGain: 0.6, delays: '10', decays: '0.1' },
|
|
2563
|
+
broadcast: { inGain: 0.8, outGain: 0.75, delays: '20', decays: '0.2' },
|
|
2564
|
+
cinematic: { inGain: 0.8, outGain: 0.85, delays: '30', decays: '0.3' },
|
|
2565
|
+
concert_hall: { inGain: 0.8, outGain: 0.90, delays: '45', decays: '0.55' },
|
|
2566
|
+
intimate: { inGain: 0.85, outGain: 0.70, delays: '15', decays: '0.18' },
|
|
2567
|
+
};
|
|
2568
|
+
const resolvedRoom = roomProfile ? (ROOM_PROFILES_LOCAL[roomProfile] || null) : (addReverb ? ROOM_PROFILES_LOCAL.cinematic : null);
|
|
2569
|
+
|
|
2570
|
+
// ── Step 1: Professional voice chain ──────────────────────────────────
|
|
2571
|
+
// [noise gate] → 80Hz HPF → presence +2dB@5kHz → air shelf +1.5dB@12kHz
|
|
2572
|
+
// → 3:1 compressor → de-esser 6.5kHz → [room acoustics]
|
|
2573
|
+
const voiceChainPath = pathMod.join(tmpDir, 'voice_chain.m4a');
|
|
2574
|
+
const deEsserFreq = 6500; // Hz — sibilant center
|
|
2575
|
+
const deEsserBW = 1500; // Hz — bandwidth
|
|
2576
|
+
|
|
2577
|
+
const voiceFilterParts = [];
|
|
2578
|
+
// Noise gate: removes breath noise and room tone between words
|
|
2579
|
+
if (noiseGate) {
|
|
2580
|
+
voiceFilterParts.push('agate=threshold=0.0003:range=0.15:attack=5:release=200');
|
|
2581
|
+
}
|
|
2582
|
+
voiceFilterParts.push(
|
|
2583
|
+
'highpass=f=80',
|
|
2584
|
+
'equalizer=f=5000:width_type=o:width=2:g=2.0', // presence boost
|
|
2585
|
+
'equalizer=f=12000:width_type=s:width=1:g=1.5', // air shelf
|
|
2586
|
+
'acompressor=ratio=3:threshold=-20dB:attack=5:release=50:makeup=2',
|
|
2587
|
+
`equalizer=f=${deEsserFreq}:width_type=h:width=${deEsserBW}:g=-3.0`, // de-esser
|
|
2588
|
+
);
|
|
2589
|
+
if (resolvedRoom) {
|
|
2590
|
+
voiceFilterParts.push(
|
|
2591
|
+
`aecho=in_gain=${resolvedRoom.inGain}:out_gain=${resolvedRoom.outGain}:delays=${resolvedRoom.delays}:decays=${resolvedRoom.decays}`
|
|
2592
|
+
);
|
|
2593
|
+
}
|
|
2594
|
+
const voiceFilterWithReverb = voiceFilterParts.join(',');
|
|
2595
|
+
|
|
2596
|
+
runFfmpeg([
|
|
2597
|
+
'-i', voicePath,
|
|
2598
|
+
'-af', voiceFilterWithReverb,
|
|
2599
|
+
'-c:a', 'aac', '-b:a', '192k', '-ar', '48000', '-ac', '2',
|
|
2600
|
+
voiceChainPath,
|
|
2601
|
+
], 'Voice chain (noise gate + EQ + de-esser + compress + room)');
|
|
2602
|
+
|
|
2603
|
+
// ── Step 2: SFX synthesis and timeline cue mixing ─────────────────────
|
|
2604
|
+
let sfxMixPath = null;
|
|
2605
|
+
if (Array.isArray(sfxCues) && sfxCues.length > 0) {
|
|
2606
|
+
// Get voice duration for SFX timeline
|
|
2607
|
+
const durProbe = spawnSync('ffprobe', [
|
|
2608
|
+
'-v', 'error', '-show_entries', 'format=duration',
|
|
2609
|
+
'-of', 'default=nk=1:nw=1', voiceChainPath,
|
|
2610
|
+
], { encoding: 'utf8', timeout: 10000 });
|
|
2611
|
+
const voiceDur = parseFloat((durProbe.stdout || '0').trim()) || 30;
|
|
2612
|
+
|
|
2613
|
+
// Generate SFX via tone synthesis — full 13-type earshot library
|
|
2614
|
+
// Build per-SFX wav files, then mix them with the voice chain
|
|
2615
|
+
const sfxPaths = [];
|
|
2616
|
+
// Extended earshot SFX map (matches SFX_FREQ_MAP in mixer.js)
|
|
2617
|
+
const SFX_FULL_MAP = {
|
|
2618
|
+
whoosh: { freq: 200, dur: 0.4, vol: 0.35 },
|
|
2619
|
+
impact: { freq: 60, dur: 0.5, vol: 0.42 },
|
|
2620
|
+
pop: { freq: 600, dur: 0.12, vol: 0.28 },
|
|
2621
|
+
click: { freq: 2500, dur: 0.05, vol: 0.22 },
|
|
2622
|
+
riser: { freq: 100, dur: 2.0, vol: 0.30 },
|
|
2623
|
+
chime: { freq: 1200, dur: 0.35, vol: 0.25 },
|
|
2624
|
+
buzz: { freq: 80, dur: 0.18, vol: 0.38 },
|
|
2625
|
+
tick: { freq: 3200, dur: 0.04, vol: 0.18 },
|
|
2626
|
+
swish: { freq: 350, dur: 0.3, vol: 0.30 },
|
|
2627
|
+
punch: { freq: 45, dur: 0.25, vol: 0.50 },
|
|
2628
|
+
sparkle: { freq: 4000, dur: 0.2, vol: 0.15 },
|
|
2629
|
+
deep_boom: { freq: 30, dur: 1.2, vol: 0.55 },
|
|
2630
|
+
transition: { freq: 440, dur: 0.6, vol: 0.20 },
|
|
2631
|
+
};
|
|
2632
|
+
|
|
2633
|
+
for (let i = 0; i < sfxCues.length; i++) {
|
|
2634
|
+
const cue = sfxCues[i];
|
|
2635
|
+
const sfxName = (cue.name || cue.sfxName || 'pop').toLowerCase().replace(/-/g, '_');
|
|
2636
|
+
const cueTime = Number(cue.time || cue.t || 0);
|
|
2637
|
+
const cueVol = Number(cue.volume || cue.vol || 1.0);
|
|
2638
|
+
const sfxDef = SFX_FULL_MAP[sfxName] || SFX_FULL_MAP.pop;
|
|
2639
|
+
const sfxDur = sfxDef.dur;
|
|
2640
|
+
|
|
2641
|
+
// Perceptual lead: place audio 40ms before the visual cue
|
|
2642
|
+
const adjustedTime = Math.max(0, cueTime - 0.04);
|
|
2643
|
+
|
|
2644
|
+
const sfxPath = pathMod.join(tmpDir, `sfx_${i}_${sfxName}.m4a`);
|
|
2645
|
+
const freq = sfxDef.freq;
|
|
2646
|
+
|
|
2647
|
+
// Create silence-padded SFX at correct offset in timeline
|
|
2648
|
+
const silenceFilter = `anullsrc=channel_layout=stereo:sample_rate=48000[silence];sine=f=${freq}:d=${sfxDur}[sfxtone];[sfxtone]volume=${cueVol * 0.35}[sfxv];[silence][sfxv]amix=inputs=2:duration=longest[sfxout]`;
|
|
2649
|
+
const adelayMs = Math.round(adjustedTime * 1000);
|
|
2650
|
+
runFfmpeg([
|
|
2651
|
+
'-f', 'lavfi', '-i', `anullsrc=channel_layout=stereo:sample_rate=48000:duration=${Math.max(sfxDur + adjustedTime + 0.5, 1)}`,
|
|
2652
|
+
'-f', 'lavfi', '-i', `sine=f=${freq}:d=${sfxDur}`,
|
|
2653
|
+
'-filter_complex', `[1:a]volume=${cueVol * 0.35},adelay=${adelayMs}|${adelayMs}[sfx];[0:a][sfx]amix=inputs=2:duration=first[out]`,
|
|
2654
|
+
'-map', '[out]',
|
|
2655
|
+
'-c:a', 'aac', '-b:a', '128k', '-ar', '48000', '-ac', '2',
|
|
2656
|
+
'-t', String(voiceDur),
|
|
2657
|
+
sfxPath,
|
|
2658
|
+
], `SFX synthesis: ${sfxName} at t=${cueTime}s`);
|
|
2659
|
+
sfxPaths.push(sfxPath);
|
|
2660
|
+
}
|
|
2661
|
+
|
|
2662
|
+
// Merge all SFX into one timeline track
|
|
2663
|
+
if (sfxPaths.length === 1) {
|
|
2664
|
+
sfxMixPath = sfxPaths[0];
|
|
2665
|
+
} else {
|
|
2666
|
+
sfxMixPath = pathMod.join(tmpDir, 'sfx_combined.m4a');
|
|
2667
|
+
const sfxInputs = sfxPaths.flatMap(p => ['-i', p]);
|
|
2668
|
+
const sfxFilter = sfxPaths.map((_, i) => `[${i}:a]`).join('') + `amix=inputs=${sfxPaths.length}:duration=longest:normalize=0[sfxout]`;
|
|
2669
|
+
runFfmpeg([
|
|
2670
|
+
...sfxInputs,
|
|
2671
|
+
'-filter_complex', sfxFilter,
|
|
2672
|
+
'-map', '[sfxout]',
|
|
2673
|
+
'-c:a', 'aac', '-b:a', '128k', '-ar', '48000', '-ac', '2',
|
|
2674
|
+
sfxMixPath,
|
|
2675
|
+
], 'SFX combine');
|
|
2676
|
+
}
|
|
2677
|
+
}
|
|
2678
|
+
|
|
2679
|
+
// ── Step 3: Music processing + sidechain ducking ───────────────────────
|
|
2680
|
+
let finalMixPath;
|
|
2681
|
+
if (musicPath && syncFs.existsSync(musicPath)) {
|
|
2682
|
+
const vidDur = spawnSync('ffprobe', [
|
|
2683
|
+
'-v', 'error', '-show_entries', 'format=duration',
|
|
2684
|
+
'-of', 'default=nk=1:nw=1', voiceChainPath,
|
|
2685
|
+
], { encoding: 'utf8', timeout: 10000 });
|
|
2686
|
+
const totalDur = parseFloat((vidDur.stdout || '0').trim()) || 60;
|
|
2687
|
+
|
|
2688
|
+
const musicFilter = stereoWiden
|
|
2689
|
+
? `aformat=sample_rates=48000:channel_layouts=stereo,highpass=f=80,lowpass=f=16000,volume=${musicVolume},extrastereo=m=1.4`
|
|
2690
|
+
: `aformat=sample_rates=48000:channel_layouts=stereo,highpass=f=80,lowpass=f=16000,volume=${musicVolume}`;
|
|
2691
|
+
|
|
2692
|
+
const duckedMixPath = pathMod.join(tmpDir, 'ducked_mix.m4a');
|
|
2693
|
+
|
|
2694
|
+
const inputs = ['-i', voiceChainPath, '-stream_loop', '-1', '-i', musicPath, '-t', String(totalDur)];
|
|
2695
|
+
const sfxInputPart = sfxMixPath ? ['-i', sfxMixPath] : [];
|
|
2696
|
+
const sfxInputIdx = sfxMixPath ? 2 : -1;
|
|
2697
|
+
|
|
2698
|
+
let filterComplex, mapExpr;
|
|
2699
|
+
if (sfxMixPath) {
|
|
2700
|
+
filterComplex = [
|
|
2701
|
+
`[1:a]${musicFilter}[music_proc]`,
|
|
2702
|
+
`[0:a][music_proc]sidechaincompress=threshold=0.003:ratio=20:attack=0.2:release=1:makeup=2[ducked]`,
|
|
2703
|
+
`[ducked][2:a]amix=inputs=2:duration=first:normalize=0[mix_sfx]`,
|
|
2704
|
+
].join(';');
|
|
2705
|
+
mapExpr = '[mix_sfx]';
|
|
2706
|
+
} else {
|
|
2707
|
+
filterComplex = [
|
|
2708
|
+
`[1:a]${musicFilter}[music_proc]`,
|
|
2709
|
+
`[0:a][music_proc]sidechaincompress=threshold=0.003:ratio=20:attack=0.2:release=1:makeup=2[ducked]`,
|
|
2710
|
+
].join(';');
|
|
2711
|
+
mapExpr = '[ducked]';
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
runFfmpeg([
|
|
2715
|
+
'-i', voiceChainPath,
|
|
2716
|
+
'-stream_loop', '-1', '-i', musicPath, '-t', String(totalDur),
|
|
2717
|
+
...sfxInputPart,
|
|
2718
|
+
'-filter_complex', filterComplex,
|
|
2719
|
+
'-map', mapExpr,
|
|
2720
|
+
'-c:a', 'aac', '-b:a', '192k', '-ar', '48000', '-ac', '2',
|
|
2721
|
+
duckedMixPath,
|
|
2722
|
+
], 'Sidechain ducking + SFX mix');
|
|
2723
|
+
finalMixPath = duckedMixPath;
|
|
2724
|
+
} else if (sfxMixPath) {
|
|
2725
|
+
// No music, but SFX — mix voice + SFX
|
|
2726
|
+
const sfxVoiceMix = pathMod.join(tmpDir, 'voice_sfx.m4a');
|
|
2727
|
+
runFfmpeg([
|
|
2728
|
+
'-i', voiceChainPath, '-i', sfxMixPath,
|
|
2729
|
+
'-filter_complex', '[0:a][1:a]amix=inputs=2:duration=first:normalize=0[out]',
|
|
2730
|
+
'-map', '[out]',
|
|
2731
|
+
'-c:a', 'aac', '-b:a', '192k', '-ar', '48000', '-ac', '2',
|
|
2732
|
+
sfxVoiceMix,
|
|
2733
|
+
], 'Voice + SFX mix');
|
|
2734
|
+
finalMixPath = sfxVoiceMix;
|
|
2735
|
+
} else {
|
|
2736
|
+
finalMixPath = voiceChainPath;
|
|
2737
|
+
}
|
|
2738
|
+
|
|
2739
|
+
// ── Step 4: Two-pass EBU R128 loudnorm master (platform-aware) ────────
|
|
2740
|
+
const normPath = pathMod.join(tmpDir, 'master_norm.m4a');
|
|
2741
|
+
twoPassNorm(finalMixPath, normPath, resolvedLufs, resolvedTp);
|
|
2742
|
+
|
|
2743
|
+
// ── Step 5: Mux into video or copy to outputPath ──────────────────────
|
|
2744
|
+
if (videoPath && syncFs.existsSync(videoPath)) {
|
|
2745
|
+
const vidDur2 = spawnSync('ffprobe', [
|
|
2746
|
+
'-v', 'error', '-show_entries', 'format=duration',
|
|
2747
|
+
'-of', 'default=nk=1:nw=1', videoPath,
|
|
2748
|
+
], { encoding: 'utf8', timeout: 10000 });
|
|
2749
|
+
const vd = parseFloat((vidDur2.stdout || '0').trim()) || 0;
|
|
2750
|
+
runFfmpeg([
|
|
2751
|
+
'-i', videoPath, '-i', normPath,
|
|
2752
|
+
'-map', '0:v', '-map', '1:a',
|
|
2753
|
+
'-c:v', 'copy',
|
|
2754
|
+
'-c:a', 'aac', '-b:a', '192k',
|
|
2755
|
+
...(vd > 0 ? ['-t', String(vd)] : []),
|
|
2756
|
+
'-movflags', '+faststart',
|
|
2757
|
+
outputPath,
|
|
2758
|
+
], 'Mux mixed audio into video');
|
|
2759
|
+
} else {
|
|
2760
|
+
// Just copy the normalized audio
|
|
2761
|
+
const copyRes = spawnSync('ffmpeg', ['-nostdin', '-y', '-i', normPath, '-c', 'copy', outputPath], { encoding: 'utf8', timeout: 30000 });
|
|
2762
|
+
if (copyRes.status !== 0) {
|
|
2763
|
+
// Fallback: re-encode
|
|
2764
|
+
runFfmpeg(['-i', normPath, '-c:a', 'aac', '-b:a', '192k', outputPath], 'Copy audio output');
|
|
2765
|
+
}
|
|
2766
|
+
}
|
|
2767
|
+
|
|
2768
|
+
// ── Cleanup tmp ───────────────────────────────────────────────────────
|
|
2769
|
+
try { syncFs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) {}
|
|
2770
|
+
|
|
2771
|
+
// ── Measure final loudness ─────────────────────────────────────────────
|
|
2772
|
+
let measuredLufs = null;
|
|
2773
|
+
let measuredTp = null;
|
|
2774
|
+
try {
|
|
2775
|
+
const m = spawnSync('ffmpeg', [
|
|
2776
|
+
'-nostdin', '-i', outputPath,
|
|
2777
|
+
'-af', `loudnorm=I=${resolvedLufs}:TP=${resolvedTp}:print_format=json`,
|
|
2778
|
+
'-f', 'null', '-',
|
|
2779
|
+
], { encoding: 'utf8', timeout: 60000 });
|
|
2780
|
+
const combined = (m.stderr || '') + (m.stdout || '');
|
|
2781
|
+
const match = combined.match(/\{[\s\S]*?"input_i"[\s\S]*?\}/);
|
|
2782
|
+
if (match) {
|
|
2783
|
+
const ln = JSON.parse(match[0]);
|
|
2784
|
+
measuredLufs = parseFloat(ln.input_i);
|
|
2785
|
+
measuredTp = parseFloat(ln.input_tp);
|
|
2786
|
+
}
|
|
2787
|
+
} catch (_) {}
|
|
2788
|
+
|
|
2789
|
+
const sizeBytes = syncFs.existsSync(outputPath) ? syncFs.statSync(outputPath).size : 0;
|
|
2790
|
+
|
|
2791
|
+
return {
|
|
2792
|
+
outputPath,
|
|
2793
|
+
sizeBytes,
|
|
2794
|
+
measuredLufs,
|
|
2795
|
+
measuredTp,
|
|
2796
|
+
targetLufs: resolvedLufs,
|
|
2797
|
+
platform: platform || null,
|
|
2798
|
+
passed: measuredLufs != null ? Math.abs(measuredLufs - resolvedLufs) <= 1.5 && measuredTp <= -0.5 : null,
|
|
2799
|
+
chain: {
|
|
2800
|
+
noiseGate: noiseGate ? 'agate -60dBFS' : 'off',
|
|
2801
|
+
hpf: '80Hz',
|
|
2802
|
+
presenceBoost: '+2dB @ 5kHz',
|
|
2803
|
+
airShelf: '+1.5dB @ 12kHz',
|
|
2804
|
+
deEsser: `${deEsserFreq}Hz dynamic`,
|
|
2805
|
+
compressor: '3:1 @ -20dBFS',
|
|
2806
|
+
roomAcoustics: resolvedRoom ? (roomProfile || 'cinematic') : 'off',
|
|
2807
|
+
sidechainDuck: musicPath ? `${duckLevel}dB` : 'no music',
|
|
2808
|
+
stereoWiden: stereoWiden && musicPath ? 'MS extrastereo 1.4x' : 'off',
|
|
2809
|
+
sfxCues: sfxCues.length,
|
|
2810
|
+
masterBus: `EBU R128 ${resolvedLufs} LUFS${platform ? ` (${platform})` : ''}`,
|
|
2811
|
+
},
|
|
2812
|
+
message: `Professional audio mix complete → ${outputPath} (${(sizeBytes / 1024).toFixed(0)}KB${measuredLufs != null ? `, ${measuredLufs.toFixed(1)} LUFS` : ''}${platform ? `, ${platform} target` : ''})`,
|
|
1854
2813
|
};
|
|
1855
2814
|
},
|
|
1856
2815
|
},
|
|
@@ -1861,7 +2820,7 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1861
2820
|
|
|
1862
2821
|
const SERVER_INFO = {
|
|
1863
2822
|
name: 'video-pipeline',
|
|
1864
|
-
version: '2.
|
|
2823
|
+
version: '2.7.0',
|
|
1865
2824
|
};
|
|
1866
2825
|
|
|
1867
2826
|
const CAPABILITIES = {
|