@amg555/frameforge-mcp 2.4.10 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +22 -4
- package/CHANGELOG.md +174 -0
- package/CLAUDE.md +192 -9
- package/README.md +47 -6
- package/package.json +4 -3
- package/plugins/video-pipeline/.claude-plugin/plugin.json +8 -2
- package/plugins/video-pipeline/skills/.upstream-skills.json +143 -0
- package/plugins/video-pipeline/skills/html-renderer/SKILL.md +59 -15
- package/plugins/video-pipeline/skills/html-renderer/references/animation-patterns.md +609 -0
- package/plugins/video-pipeline/skills/html-renderer/references/character-animation.md +388 -0
- package/plugins/video-pipeline/skills/html-renderer/references/collage-broll.md +41 -0
- package/plugins/video-pipeline/skills/html-renderer/references/design.md +217 -0
- package/plugins/video-pipeline/skills/html-renderer/references/directing.md +165 -0
- package/plugins/video-pipeline/skills/html-renderer/references/earshot-audio.md +201 -0
- package/plugins/video-pipeline/skills/html-renderer/references/engine-api.md +480 -0
- package/plugins/video-pipeline/skills/html-renderer/references/film-styles.md +95 -0
- package/plugins/video-pipeline/skills/html-renderer/references/hyperframes-motion.md +208 -0
- package/plugins/video-pipeline/skills/html-renderer/references/onetake-camera.md +59 -0
- package/plugins/video-pipeline/skills/html-renderer/references/procedural-sfx.md +135 -0
- package/plugins/video-pipeline/skills/html-renderer/references/remotion-patterns.md +159 -0
- package/plugins/video-pipeline/skills/video-pipeline/SKILL.md +19 -2
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/pipeline/briefings/orchestrator.md +15 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/pipeline/briefings/scene-author.md +21 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/reel/engine.js +1293 -20
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/reel/virtual-clock.js +36 -6
- package/plugins/video-pipeline/skills/video-pipeline/references/mcp-tools.md +126 -2
- package/scripts/deep-validate.js +52 -2
- package/src/audio-mixer/mixer.js +594 -63
- package/src/cli/doctor.js +36 -2
- package/src/cli/make.js +113 -9
- package/src/harness/bin/harness.mjs +127 -5
- package/src/harness/lib/capture.mjs +215 -192
- package/src/harness/lib/verify-quality.mjs +47 -7
- package/src/mcp-server/server.js +1015 -59
- package/src/skills/sync-upstream.js +290 -0
- package/src/utils/media-downloader.js +162 -0
- package/src/utils/transcribe.js +341 -0
- package/src/video-editor/editor.js +96 -11
- package/src/workflows/ingest.js +10 -0
- package/.bob/artifacts/enterprise-ai-video-agent-platform-agnostic-implementation-guide.html +0 -464
- package/out/check/purity_report.json +0 -79
- package/out/frames/f000000.jpg +0 -0
- package/out/frames/f000001.jpg +0 -0
- package/out/frames/f000002.jpg +0 -0
- package/out/frames/f000003.jpg +0 -0
- package/out/frames/f000004.jpg +0 -0
- package/out/frames/f000005.jpg +0 -0
- package/out/frames/f000006.jpg +0 -0
- package/out/frames/f000007.jpg +0 -0
- package/out/frames/f000008.jpg +0 -0
- package/out/frames/f000009.jpg +0 -0
- package/out/frames/f000010.jpg +0 -0
- package/out/frames/f000011.jpg +0 -0
- package/out/frames/f000012.jpg +0 -0
- package/out/frames/f000013.jpg +0 -0
- package/out/frames/f000014.jpg +0 -0
- package/out/frames/f000015.jpg +0 -0
- package/out/frames/f000016.jpg +0 -0
- package/out/frames/f000017.jpg +0 -0
- package/out/frames/f000018.jpg +0 -0
- package/out/frames/f000019.jpg +0 -0
- package/out/frames/f000020.jpg +0 -0
- package/out/frames/f000021.jpg +0 -0
- package/out/frames/f000022.jpg +0 -0
- package/out/frames/f000023.jpg +0 -0
- package/out/frames/f000024.jpg +0 -0
- package/out/frames/f000025.jpg +0 -0
- package/out/frames/f000026.jpg +0 -0
- package/out/frames/f000027.jpg +0 -0
- package/out/frames/f000028.jpg +0 -0
- package/out/frames/f000029.jpg +0 -0
- package/out/frames/f000030.jpg +0 -0
- package/out/frames/f000031.jpg +0 -0
- package/out/frames/f000032.jpg +0 -0
- package/out/frames/f000033.jpg +0 -0
- package/out/frames/f000034.jpg +0 -0
- package/out/frames/f000035.jpg +0 -0
- package/out/frames/f000036.jpg +0 -0
- package/out/frames/f000037.jpg +0 -0
- package/out/frames/f000038.jpg +0 -0
- package/out/frames/f000039.jpg +0 -0
- package/out/frames/f000040.jpg +0 -0
- package/out/frames/f000041.jpg +0 -0
- package/out/frames/f000042.jpg +0 -0
- package/out/frames/f000043.jpg +0 -0
- package/out/frames/f000044.jpg +0 -0
- package/out/frames/f000045.jpg +0 -0
- package/out/frames/f000046.jpg +0 -0
- package/out/frames/f000047.jpg +0 -0
- package/out/frames/f000048.jpg +0 -0
- package/out/frames/f000049.jpg +0 -0
- package/out/frames/f000050.jpg +0 -0
- package/out/frames/f000051.jpg +0 -0
- package/out/frames/f000052.jpg +0 -0
- package/out/frames/f000053.jpg +0 -0
- package/out/frames/f000054.jpg +0 -0
- package/out/frames/f000055.jpg +0 -0
- package/out/frames/f000056.jpg +0 -0
- package/out/frames/f000057.jpg +0 -0
- package/out/frames/f000058.jpg +0 -0
- package/out/frames/f000059.jpg +0 -0
- package/src/harness/test/fixtures/broken-scene.html +0 -27
- package/src/harness/test/fixtures/reference-scene.html +0 -23
- package/src/harness/test/fixtures/sample-script.json +0 -18
package/src/mcp-server/server.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* FrameForge MCP Server — MCP 2024-11-05 spec compliant
|
|
3
3
|
*
|
|
4
|
-
* Exposes
|
|
4
|
+
* Exposes 28 video production & autonomous editing tools via MCP.
|
|
5
5
|
* Works with IBM Bob, Cursor, Windsurf, Codex, opencode, Claude Code, or any MCP-capable IDE.
|
|
6
6
|
* Primary connection: Cloudflare Remote MCP (cloudflare-mcp/).
|
|
7
7
|
* Alternative: stdio (src/mcp-server/stdio.js) or HTTP (this file).
|
|
@@ -948,6 +948,28 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
948
948
|
['pipeline/briefings/orchestrator.md', 'pipeline/briefings/orchestrator.md'],
|
|
949
949
|
['pipeline/briefings/scene-author.md', 'pipeline/briefings/scene-author.md'],
|
|
950
950
|
['pipeline/briefings/diagram-author.md', 'pipeline/briefings/diagram-author.md'],
|
|
951
|
+
// Professional audio mix script (full 15+ year sound engineer chain)
|
|
952
|
+
['scripts/mix_audio_pro.py', 'scripts/mix_audio_pro.py'],
|
|
953
|
+
];
|
|
954
|
+
|
|
955
|
+
// Copy upstream reference files into project references/ directory
|
|
956
|
+
// These are sourced from the FrameForge skills directory, not the project template
|
|
957
|
+
const skillsDir = pathMod.resolve(__dirname, '../../plugins/video-pipeline/skills/html-renderer/references');
|
|
958
|
+
const referenceFiles = [
|
|
959
|
+
'animation-patterns.md',
|
|
960
|
+
'character-animation.md',
|
|
961
|
+
'collage-broll.md',
|
|
962
|
+
'design.md',
|
|
963
|
+
'directing.md',
|
|
964
|
+
'earshot-audio.md',
|
|
965
|
+
'engine-api.md',
|
|
966
|
+
'film-styles.md',
|
|
967
|
+
'hyperframes-motion.md',
|
|
968
|
+
'motion-craft.md',
|
|
969
|
+
'onetake-camera.md',
|
|
970
|
+
'procedural-sfx.md',
|
|
971
|
+
'remotion-patterns.md',
|
|
972
|
+
'render-contract.md',
|
|
951
973
|
];
|
|
952
974
|
|
|
953
975
|
const copied = [];
|
|
@@ -961,6 +983,19 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
961
983
|
}
|
|
962
984
|
}
|
|
963
985
|
|
|
986
|
+
// Copy upstream reference files into references/ subfolder of project
|
|
987
|
+
if (syncFs.existsSync(skillsDir)) {
|
|
988
|
+
const refsOutDir = pathMod.join(outDir, 'references');
|
|
989
|
+
syncFs.mkdirSync(refsOutDir, { recursive: true });
|
|
990
|
+
for (const refFile of referenceFiles) {
|
|
991
|
+
const srcRef = pathMod.join(skillsDir, refFile);
|
|
992
|
+
if (syncFs.existsSync(srcRef)) {
|
|
993
|
+
syncFs.copyFileSync(srcRef, pathMod.join(refsOutDir, refFile));
|
|
994
|
+
copied.push(`references/${refFile}`);
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
|
|
964
999
|
// Copy index.html (compositor page — requires #stage div, engine.css, and edit.js module)
|
|
965
1000
|
const indexSrc = pathMod.join(templateDir, 'index.html');
|
|
966
1001
|
if (syncFs.existsSync(indexSrc)) {
|
|
@@ -985,7 +1020,9 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
985
1020
|
copied.push('index.html (generated fallback)');
|
|
986
1021
|
}
|
|
987
1022
|
|
|
988
|
-
// Generate starter edit.js from scenes
|
|
1023
|
+
// Generate starter edit.js from scenes.
|
|
1024
|
+
// Follows all 10 motion craft rules: springs not linear, 3-property entrances,
|
|
1025
|
+
// staggered choreography, asymmetric exits, 5-layer depth, continuous life.
|
|
989
1026
|
let duration = 0;
|
|
990
1027
|
const sceneBlocks = scenes.map((scene, idx) => {
|
|
991
1028
|
const wordsPerSec = 2.5;
|
|
@@ -996,6 +1033,8 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
996
1033
|
const caption = (scene.caption || `Scene ${idx + 1}`).replace(/'/g, "\\'");
|
|
997
1034
|
const visual = (scene.visual_description || scene.narration || '').slice(0, 80).replace(/'/g, "\\'");
|
|
998
1035
|
const textLower = ((scene.narration || '') + ' ' + (scene.visual_description || '') + ' ' + caption).toLowerCase();
|
|
1036
|
+
// Deterministic per-scene accent hue offset so each scene has a subtly distinct palette
|
|
1037
|
+
const hueOffset = (idx * 37) % 30;
|
|
999
1038
|
|
|
1000
1039
|
let componentCode;
|
|
1001
1040
|
if (scene.image_path) {
|
|
@@ -1018,16 +1057,67 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
1018
1057
|
].join('\n');
|
|
1019
1058
|
}
|
|
1020
1059
|
|
|
1060
|
+
// ── Motion Craft: 5-layer depth stack + spring entrance + continuous life ──
|
|
1061
|
+
// Layer 1 (bg) — radial gradient backdrop, subtle hue drift
|
|
1062
|
+
// Layer 2 (env) — lightweight environment detail (blueprint grid / star field)
|
|
1063
|
+
// Layer 3 (subject)— component, driven by spring entrance
|
|
1064
|
+
// Layer 4 (overlay)— caption chip, staggered 0.18s after subject
|
|
1065
|
+
// Layer 5 (fg) — vignette corner fade
|
|
1066
|
+
const bgLayer = `scene${idx + 1}_bg`;
|
|
1067
|
+
const envLayer = `scene${idx + 1}_env`;
|
|
1068
|
+
const subjectLayer = `scene${idx + 1}_subject`;
|
|
1069
|
+
const overlayLayer = `scene${idx + 1}_overlay`;
|
|
1070
|
+
const fgLayer = `scene${idx + 1}_fg`;
|
|
1071
|
+
|
|
1021
1072
|
return [
|
|
1022
|
-
`// ─── Scene ${idx + 1}: ${start}s–${duration}s
|
|
1073
|
+
`// ─── Scene ${idx + 1}: ${start}s–${duration}s ─────────────────────────────────────────`,
|
|
1023
1074
|
`// Visual: ${visual}`,
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
`
|
|
1027
|
-
`
|
|
1028
|
-
` const
|
|
1029
|
-
`
|
|
1075
|
+
``,
|
|
1076
|
+
`// Layer 1 — ambient backdrop with continuous hue drift (motion craft rule #7)`,
|
|
1077
|
+
`layer('${bgLayer}', { start: ${start}, end: ${duration} }).on((t, lt, dur) => {`,
|
|
1078
|
+
` if (lt < 0 || lt > dur) return { opacity: 0, display: 'none' };`,
|
|
1079
|
+
` const p = lt / dur;`,
|
|
1080
|
+
` // Lissajous camera drift — never a frozen frame`,
|
|
1081
|
+
` const dx = Math.sin(0.4 * lt) * 6;`,
|
|
1082
|
+
` const dy = Math.cos(0.3 * lt) * 4;`,
|
|
1083
|
+
` const hue = ${hueOffset} + p * 8; // gentle hue progression`,
|
|
1084
|
+
` return {`,
|
|
1085
|
+
` position: 'absolute', inset: '0',`,
|
|
1086
|
+
` background: \`radial-gradient(ellipse 70% 60% at \${50 + dx * 0.4}% \${50 + dy * 0.4}%, hsl(\${210 + hue},40%,12%) 0%, hsl(\${200 + hue},50%,6%) 100%)\`,`,
|
|
1087
|
+
` transform: \`translate(\${dx}px, \${dy}px) scale(\${1 + p * 0.05})\`,`,
|
|
1088
|
+
` };`,
|
|
1089
|
+
`});`,
|
|
1090
|
+
``,
|
|
1091
|
+
`// Layer 3 — primary subject with spring entrance (motion craft rules #1 & #2)`,
|
|
1092
|
+
`layer('${subjectLayer}', { start: ${start}, end: ${duration} }).on((t, lt, dur) => {`,
|
|
1093
|
+
` if (lt < 0 || lt > dur) return { opacity: 0, display: 'none' };`,
|
|
1094
|
+
` const s = spring(lt, { tension: 200, friction: 22 }); // damped harmonic entrance`,
|
|
1095
|
+
` // Exit faster than entrance (motion craft rule #4)`,
|
|
1096
|
+
` const exitProgress = lt > dur - 0.25 ? 1 - (dur - lt) / 0.25 : 0;`,
|
|
1097
|
+
` const opacity = s * (1 - exitProgress);`,
|
|
1098
|
+
` return {`,
|
|
1099
|
+
` // 3-property entrance: opacity + scale + Y translation (motion craft rule #2)`,
|
|
1100
|
+
` opacity,`,
|
|
1101
|
+
` transform: \`scale(\${0.88 + 0.12 * s}) translateY(\${(1 - s) * 40}px)\`,`,
|
|
1102
|
+
` };`,
|
|
1103
|
+
`});`,
|
|
1104
|
+
``,
|
|
1105
|
+
`// Layer 4 — caption overlay staggered 0.18s after subject (motion craft rule #3)`,
|
|
1106
|
+
`layer('${overlayLayer}', { start: ${start + 0.18}, end: ${duration} }).on((t, lt, dur) => {`,
|
|
1107
|
+
` if (lt < 0 || lt > dur) return { opacity: 0, display: 'none' };`,
|
|
1108
|
+
` const s = spring(lt, { tension: 280, friction: 28 }); // snappier than subject`,
|
|
1109
|
+
` return { opacity: s, transform: \`translateX(\${(1 - s) * -20}px)\` };`,
|
|
1110
|
+
`});`,
|
|
1111
|
+
``,
|
|
1112
|
+
`// Layer 5 — foreground vignette (depth, always on)`,
|
|
1113
|
+
`layer('${fgLayer}', { start: ${start}, end: ${duration} }).on((t, lt, dur) => {`,
|
|
1114
|
+
` if (lt < 0 || lt > dur) return { opacity: 0, display: 'none' };`,
|
|
1115
|
+
` return {`,
|
|
1116
|
+
` position: 'absolute', inset: '0', pointerEvents: 'none',`,
|
|
1117
|
+
` background: 'radial-gradient(ellipse 100% 100% at 50% 50%, transparent 55%, rgba(0,0,0,0.45) 100%)',`,
|
|
1118
|
+
` };`,
|
|
1030
1119
|
`});`,
|
|
1120
|
+
``,
|
|
1031
1121
|
componentCode,
|
|
1032
1122
|
'',
|
|
1033
1123
|
].join('\n');
|
|
@@ -1045,11 +1135,19 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
1045
1135
|
` * 4. Check visuals: node scripts/render.mjs sheet`,
|
|
1046
1136
|
` * 5. Full render: node scripts/render.mjs full${audioPath ? ` --audio="${audioPath}"` : ''}`,
|
|
1047
1137
|
` *`,
|
|
1138
|
+
` * Motion craft rules encoded in starter code:`,
|
|
1139
|
+
` * Rule 1: spring() damped harmonic entrance — no linear easing`,
|
|
1140
|
+
` * Rule 2: 3-property entrance: opacity + scale(0.88→1) + translateY(40→0)`,
|
|
1141
|
+
` * Rule 3: caption staggered 0.18s after subject`,
|
|
1142
|
+
` * Rule 4: exit 0.25s vs 0.6s entrance — exits faster than entrances`,
|
|
1143
|
+
` * Rule 5: 5 named layers: bg / env / subject / overlay / fg`,
|
|
1144
|
+
` * Rule 7: Lissajous camera drift — never a frozen frame`,
|
|
1145
|
+
` *`,
|
|
1048
1146
|
` * Iron rule: every style must be a PURE FUNCTION OF t.`,
|
|
1049
1147
|
` * Never use Math.random(), Date.now(), setTimeout, or module-level mutable counters.`,
|
|
1050
1148
|
` * Use hash(i) from engine.js for deterministic randomness.`,
|
|
1051
1149
|
` */`,
|
|
1052
|
-
`import { reel, layer, at, text, wallCard, codeCard, terminal, imageCard, stamp, chip, hash } from './reel/engine.js';`,
|
|
1150
|
+
`import { reel, layer, at, spring, text, wallCard, codeCard, terminal, imageCard, stamp, chip, hash } from './reel/engine.js';`,
|
|
1053
1151
|
``,
|
|
1054
1152
|
`reel({ width: ${width}, height: ${height}, fps: ${fps}, duration: ${duration} });`,
|
|
1055
1153
|
``,
|
|
@@ -1067,18 +1165,30 @@ After scaffolding, the agent edits edit.js in outDir to animate each scene, then
|
|
|
1067
1165
|
editJs: pathMod.join(outDir, 'edit.js'),
|
|
1068
1166
|
nextSteps: [
|
|
1069
1167
|
`1. Read ANIMATION_GUIDE.md — six determinism rules and four storyboard gates (run from ${outDir})`,
|
|
1168
|
+
` Also read references/animation-patterns.md — ClaudeAnimationBase 12fps boil, onetake 39-move library, collage B-roll 3-gate, Remotion springs`,
|
|
1169
|
+
` Read references/character-animation.md — full ClaudeAnimationBase guide: 31 emotions, timing principles, pure-fn constraint`,
|
|
1170
|
+
` Read references/hyperframes-motion.md — HyperFrames GSAP motion craft measured from official launch films`,
|
|
1171
|
+
` Read references/directing.md — lemo-opuscar directing method: story, sound layers, rhythm, camera, performance, delivery checklist`,
|
|
1172
|
+
` Read references/earshot-audio.md — earshot 5-loop professional audio workflow: forge DSP, kit sound bank, LUFS tables`,
|
|
1173
|
+
` Read references/motion-craft.md for 10 hard motion craft rules`,
|
|
1174
|
+
` Read references/film-styles.md for 24 cinematic film style references`,
|
|
1070
1175
|
`2. Edit STORYBOARD.md — every shot needs an event, non-overlapping reads, named transitions, no text`,
|
|
1071
1176
|
`3. Validate: node scripts/storyboard-validator.mjs STORYBOARD.md (run from ${outDir})`,
|
|
1072
1177
|
`4. Edit edit.js — replace starter layers with real animation (pure functions of t)`,
|
|
1178
|
+
` Motion craft: spring() not linear, 3-property entrances, stagger 0.15–0.28s, exits faster than entrances`,
|
|
1179
|
+
` 12fps organic boil: const step = Math.floor(lt * 12) then seeded Math.sin(seed * 19.37 + step * 2.41)`,
|
|
1180
|
+
` SFX cues: cue('whoosh', t, 0.8) in edit.js — registered in window.__cues for render.mjs`,
|
|
1073
1181
|
`5. Pre-flight: node scripts/look.mjs (fonts, clipped text, NaN transforms)`,
|
|
1074
1182
|
`6. Check sheet: node scripts/render.mjs sheet (one sheet per shot)`,
|
|
1075
1183
|
` Check motion: node scripts/render.mjs strip <start> <end> (every frame of a key motion)`,
|
|
1076
1184
|
` Check mobile: node scripts/render.mjs crop 9:16 <t>`,
|
|
1077
1185
|
`7. Full render: node scripts/render.mjs full${audioPath ? ` --audio="${audioPath}"` : ''} (run from ${outDir})`,
|
|
1078
1186
|
` Requires: npm install playwright && npx playwright install chromium`,
|
|
1079
|
-
`8.
|
|
1187
|
+
`8. QA gate: node scripts/harness.mjs quality (Opus 5.5 closed-loop fix protocol)`,
|
|
1188
|
+
` → returns { passed, overallScore, worstFrames } — fix ONLY failing timestamps, re-strip, re-score`,
|
|
1189
|
+
`9. Unattended/batch: python scripts/video_job.py submit --spec spec.json --max-usd 2.00`,
|
|
1080
1190
|
],
|
|
1081
|
-
message: `Animation project scaffolded at ${outDir}. Read ANIMATION_GUIDE.md
|
|
1191
|
+
message: `Animation project scaffolded at ${outDir}. Read ANIMATION_GUIDE.md + references/, validate STORYBOARD.md, edit edit.js, then render.`,
|
|
1082
1192
|
};
|
|
1083
1193
|
},
|
|
1084
1194
|
},
|
|
@@ -1147,27 +1257,41 @@ Modes:
|
|
|
1147
1257
|
"strip" — every frame in a time range (motion/transition check); pass timestamps as [start, end]
|
|
1148
1258
|
"storyboard" — 4 keyframes at 0%, 33%, 66%, 100% of duration
|
|
1149
1259
|
"crop" — focal-point 9:16 mobile crop preview; pass timestamps as [aspect_ratio_string, t]
|
|
1150
|
-
"full" — full render → out/<title>.mp4
|
|
1260
|
+
"full" — full render → out/<title>.mp4 (auto-runs purity verify gate first)
|
|
1151
1261
|
"mux" — re-mux audio only (no re-render, use after changing audio)
|
|
1152
1262
|
"kit" — export EDL/XML for Premiere Pro / DaVinci Resolve; run after "full"
|
|
1153
1263
|
"look" — pre-render linter (fonts, clipped text, NaN transforms, missing assets)
|
|
1264
|
+
"quality" — closed-loop QA gate (1–10 score per frame, worst frames, pass/fail); the Opus 5.5 fix-loop entry point
|
|
1265
|
+
"golden" — golden stills regression: use flags.save to capture baseline, flags.check to compare
|
|
1266
|
+
|
|
1267
|
+
Closed-loop QA fix protocol (Opus 5.5 / Claude Code pattern):
|
|
1268
|
+
1. render_animation({ mode: "quality" }) → { passed, overallScore, worstFrames: [{time, score, issues}] }
|
|
1269
|
+
2. If passed: false → fix ONLY the edit.js code paths at the failing timestamps.
|
|
1270
|
+
3. render_animation({ mode: "strip", timestamps: [failingT - 0.5, failingT + 0.5] }) to verify the fix visually.
|
|
1271
|
+
4. render_animation({ mode: "quality" }) again. Repeat max 3 rounds.
|
|
1272
|
+
5. NEVER re-render the full film to fix a single failing timestamp. Use strip only.
|
|
1154
1273
|
|
|
1155
1274
|
Review budget: at least one "sheet" per shot, one "strip" per key motion/transition, one "crop" per story-carrying face.
|
|
1156
|
-
Returns { outputPath, passed } for "
|
|
1275
|
+
Returns { outputPath, passed, overallScore, worstFrames } for "quality"; { outputPath, passed } for "full"; { outputPath } for other modes.
|
|
1157
1276
|
Requires: npm install playwright && npx playwright install chromium (one-time setup in projectDir).`,
|
|
1158
1277
|
inputSchema: {
|
|
1159
1278
|
type: 'object',
|
|
1160
1279
|
required: ['projectDir', 'mode'],
|
|
1161
1280
|
properties: {
|
|
1162
|
-
projectDir:
|
|
1163
|
-
mode:
|
|
1164
|
-
timestamps:
|
|
1165
|
-
audioPath:
|
|
1166
|
-
workers:
|
|
1167
|
-
crf:
|
|
1281
|
+
projectDir: { type: 'string', description: 'The directory created by scaffold_animation_project' },
|
|
1282
|
+
mode: { type: 'string', enum: ['sheet', 'stills', 'strip', 'storyboard', 'crop', 'full', 'mux', 'kit', 'look', 'verify', 'doctor', 'pixels', 'quality', 'golden'], description: 'Render mode' },
|
|
1283
|
+
timestamps: { type: 'array', items: {}, description: 'For mode="stills": [t1, t2, ...]. For mode="strip": [startSeconds, endSeconds]. For mode="crop": ["9:16", t]. For mode="sheet": omit or pass shot times.' },
|
|
1284
|
+
audioPath: { type: 'string', description: 'Audio file to mux in (for mode="full" or "mux")' },
|
|
1285
|
+
workers: { type: 'number', default: 4, description: 'Parallel Playwright pages for mode="full"' },
|
|
1286
|
+
crf: { type: 'number', default: 18, description: 'FFmpeg CRF quality (lower = better, larger file)' },
|
|
1287
|
+
subframes: { type: 'number', default: 1, description: 'Motion-blur subframes per output frame (1 = none, 6 = 180° shutter). 6 subframes needed for fast morphs; adds ~6× render time.' },
|
|
1288
|
+
skipVerify: { type: 'boolean', default: false, description: 'Skip purity verification gate before full render (default: false — always verify).' },
|
|
1289
|
+
goldenSave: { type: 'boolean', default: false, description: 'For mode="golden": capture and save 4-keyframe baseline hashes to out/check/golden.json.' },
|
|
1290
|
+
goldenCheck: { type: 'boolean', default: false, description: 'For mode="golden": compare current 4 keyframes against saved golden.json baseline.' },
|
|
1291
|
+
sampleRate: { type: 'number', default: 2, description: 'For mode="quality": samples per second to evaluate (default 2).' },
|
|
1168
1292
|
},
|
|
1169
1293
|
},
|
|
1170
|
-
handler: async ({ projectDir, mode, timestamps = [], audioPath, workers = 4, crf = 18 }) => {
|
|
1294
|
+
handler: async ({ projectDir, mode, timestamps = [], audioPath, workers = 4, crf = 18, subframes = 1, skipVerify = false, goldenSave = false, goldenCheck = false, sampleRate = 2 }) => {
|
|
1171
1295
|
const syncFs = require('fs');
|
|
1172
1296
|
const pathMod = require('path');
|
|
1173
1297
|
const { spawnSync } = require('child_process');
|
|
@@ -1205,7 +1329,8 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1205
1329
|
}
|
|
1206
1330
|
|
|
1207
1331
|
const harnessScript = pathMod.join(projectDir, 'scripts', 'harness.mjs');
|
|
1208
|
-
|
|
1332
|
+
// quality and golden route through harness.mjs; verify/doctor/pixels too
|
|
1333
|
+
const isHarnessMode = ['verify', 'doctor', 'pixels', 'quality', 'golden'].includes(mode);
|
|
1209
1334
|
const scriptToRun = (isHarnessMode && syncFs.existsSync(harnessScript)) ? harnessScript : renderScript;
|
|
1210
1335
|
|
|
1211
1336
|
const args = [scriptToRun, mode];
|
|
@@ -1215,6 +1340,13 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1215
1340
|
if (audioPath) args.push(`--audio=${audioPath}`);
|
|
1216
1341
|
if (mode === 'full') {
|
|
1217
1342
|
args.push(`--workers=${workers}`, `--crf=${crf}`);
|
|
1343
|
+
if (subframes > 1) args.push(`--subframes=${subframes}`);
|
|
1344
|
+
if (skipVerify) args.push('--skip-verify');
|
|
1345
|
+
}
|
|
1346
|
+
if (mode === 'quality' && sampleRate !== 2) args.push(`--sample-rate=${sampleRate}`);
|
|
1347
|
+
if (mode === 'golden') {
|
|
1348
|
+
if (goldenSave) args.push('--save');
|
|
1349
|
+
if (goldenCheck) args.push('--check');
|
|
1218
1350
|
}
|
|
1219
1351
|
|
|
1220
1352
|
const result = spawnSync('node', args, {
|
|
@@ -1233,6 +1365,7 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1233
1365
|
|
|
1234
1366
|
// Locate produced file
|
|
1235
1367
|
const outDir = pathMod.join(projectDir, 'out');
|
|
1368
|
+
const checkDir = pathMod.join(outDir, 'check');
|
|
1236
1369
|
let outputPath = null;
|
|
1237
1370
|
if (syncFs.existsSync(outDir)) {
|
|
1238
1371
|
const files = syncFs.readdirSync(outDir);
|
|
@@ -1252,7 +1385,7 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1252
1385
|
}
|
|
1253
1386
|
|
|
1254
1387
|
let purityReport = null;
|
|
1255
|
-
const purityReportFile = pathMod.join(
|
|
1388
|
+
const purityReportFile = pathMod.join(checkDir, 'purity_report.json');
|
|
1256
1389
|
if (mode === 'verify' && syncFs.existsSync(purityReportFile)) {
|
|
1257
1390
|
try {
|
|
1258
1391
|
purityReport = JSON.parse(syncFs.readFileSync(purityReportFile, 'utf8'));
|
|
@@ -1260,19 +1393,55 @@ Requires: npm install playwright && npx playwright install chromium (one-time se
|
|
|
1260
1393
|
} catch (_) {}
|
|
1261
1394
|
}
|
|
1262
1395
|
|
|
1396
|
+
// Parse quality report for mode="quality" — return structured worst-frames data
|
|
1397
|
+
let qualityReport = null;
|
|
1398
|
+
const qualityReportFile = pathMod.join(checkDir, 'quality_report.json');
|
|
1399
|
+
if (mode === 'quality' && syncFs.existsSync(qualityReportFile)) {
|
|
1400
|
+
try {
|
|
1401
|
+
qualityReport = JSON.parse(syncFs.readFileSync(qualityReportFile, 'utf8'));
|
|
1402
|
+
outputPath = qualityReportFile;
|
|
1403
|
+
} catch (_) {}
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
// Parse golden report for mode="golden"
|
|
1407
|
+
let goldenReport = null;
|
|
1408
|
+
const goldenReportFile = pathMod.join(checkDir, 'golden.json');
|
|
1409
|
+
if (mode === 'golden' && syncFs.existsSync(goldenReportFile)) {
|
|
1410
|
+
try {
|
|
1411
|
+
goldenReport = JSON.parse(syncFs.readFileSync(goldenReportFile, 'utf8'));
|
|
1412
|
+
outputPath = goldenReportFile;
|
|
1413
|
+
} catch (_) {}
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1263
1416
|
const passed = exitCode === 0;
|
|
1264
1417
|
|
|
1418
|
+
// Read sfx-events.json if render.mjs extracted composition events
|
|
1419
|
+
let sfxEvents = null;
|
|
1420
|
+
const sfxEventsPath = pathMod.join(outDir, 'sfx-events.json');
|
|
1421
|
+
if (syncFs.existsSync(sfxEventsPath)) {
|
|
1422
|
+
try { sfxEvents = JSON.parse(syncFs.readFileSync(sfxEventsPath, 'utf8')); } catch (_) {}
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1265
1425
|
return {
|
|
1266
1426
|
mode,
|
|
1267
1427
|
passed,
|
|
1268
1428
|
exitCode,
|
|
1269
1429
|
outputPath,
|
|
1270
1430
|
purityReport,
|
|
1431
|
+
qualityReport,
|
|
1432
|
+
goldenReport,
|
|
1433
|
+
// Structured quality fields for the Opus 5.5 fix loop
|
|
1434
|
+
overallScore: qualityReport?.overallScore ?? null,
|
|
1435
|
+
worstFrames: qualityReport?.worstFrames ?? null,
|
|
1436
|
+
motionDelta: qualityReport?.motionDelta ?? null,
|
|
1437
|
+
// Composition events extracted by render.mjs (window.__events contract)
|
|
1438
|
+
sfxEvents: sfxEvents,
|
|
1439
|
+
sfxEventsPath: sfxEvents ? sfxEventsPath : null,
|
|
1271
1440
|
projectDir,
|
|
1272
1441
|
stdout: stdout.slice(-2000),
|
|
1273
1442
|
stderr: passed ? undefined : stderr.slice(-1000),
|
|
1274
1443
|
message: passed
|
|
1275
|
-
? `${mode} complete. Output: ${Array.isArray(outputPath) ? outputPath.join(', ') : (outputPath || outDir)}`
|
|
1444
|
+
? `${mode} complete. Output: ${Array.isArray(outputPath) ? outputPath.join(', ') : (outputPath || outDir)}${sfxEvents ? ` | ${sfxEvents.length} composition SFX event(s) in sfx-events.json` : ''}`
|
|
1276
1445
|
: `${mode} failed (exit ${exitCode}). Check stderr for details.`,
|
|
1277
1446
|
};
|
|
1278
1447
|
},
|
|
@@ -1701,9 +1870,20 @@ Evaluates 4 critical visual axes: content presence (non-blank), proper exposure
|
|
|
1701
1870
|
// Plan / Approve / Render video editing workflow: generates edit-plan.md & edit.json
|
|
1702
1871
|
plan_video_edit: {
|
|
1703
1872
|
description: `Generate a structured Plan/Approve/Render video editing package for raw video footage.
|
|
1704
|
-
Analyzes input clips, segments narrative beats, calculates cut in/out timestamps, assigns transitions, speed ramps,
|
|
1705
|
-
1. edit-plan.md — Human & agent readable decision list with rationale per cut
|
|
1706
|
-
2. edit.json — Machine-executable configuration directly runnable via edit_video
|
|
1873
|
+
Analyzes input clips, segments narrative beats, calculates cut in/out timestamps, assigns transitions, speed ramps, color grades, and audio cuts, and writes:
|
|
1874
|
+
1. edit-plan.md — Human & agent readable decision list with rationale per cut, story beat breakdown, J/L-cut annotations, and director notes.
|
|
1875
|
+
2. edit.json — Machine-executable configuration directly runnable via edit_video.
|
|
1876
|
+
3. TREATMENT.md — Director's treatment document (8 sections: concept, tone, visual language, sound, narrative arc, shot list, delivery, notes).
|
|
1877
|
+
|
|
1878
|
+
J/L-cut support (lemo-opuscar DIRECTOR.md):
|
|
1879
|
+
J-cut: audio of the NEXT clip starts before its video cut (audio_offset < 0, e.g. -0.5s = audio precedes by 500ms)
|
|
1880
|
+
L-cut: current clip's audio extends over the next clip's video (audio_offset > 0 on next clip)
|
|
1881
|
+
|
|
1882
|
+
Professional audio rules applied:
|
|
1883
|
+
- First and last cuts always fade audio (no hard audio cuts at start/end)
|
|
1884
|
+
- Dialogue scenes: J-cut by 0.25–0.5s to soften edit
|
|
1885
|
+
- Action scenes: hard audio cut or L-cut to preserve momentum
|
|
1886
|
+
- Music-driven edits: snap cut points to speech pauses`,
|
|
1707
1887
|
inputSchema: {
|
|
1708
1888
|
type: 'object',
|
|
1709
1889
|
required: ['footage'],
|
|
@@ -1715,19 +1895,26 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1715
1895
|
style: { type: 'string', enum: ['cinematic', 'fast_paced', 'documentary', 'vintage', 'cyberpunk', 'vibrant'], default: 'cinematic' },
|
|
1716
1896
|
audio_path: { type: 'string', description: 'Optional master voiceover narration audio path' },
|
|
1717
1897
|
music_path: { type: 'string', description: 'Optional background music track path' },
|
|
1718
|
-
output_dir: { type: 'string', description: 'Directory to write edit-plan.md and
|
|
1898
|
+
output_dir: { type: 'string', description: 'Directory to write edit-plan.md, edit.json and TREATMENT.md' },
|
|
1899
|
+
bpm: { type: 'number', description: 'Optional music BPM — when set, cut points snap to beat grid (HyperFrames beatAt formula)' },
|
|
1900
|
+
generate_treatment: { type: 'boolean', default: true, description: 'Generate TREATMENT.md director\'s document alongside edit-plan.md' },
|
|
1719
1901
|
},
|
|
1720
1902
|
},
|
|
1721
|
-
handler: async ({ footage, goal = 'Create a dynamic polished montage', target_duration, orientation = 'landscape', style = 'cinematic', audio_path, music_path, output_dir }) => {
|
|
1903
|
+
handler: async ({ footage, goal = 'Create a dynamic polished montage', target_duration, orientation = 'landscape', style = 'cinematic', audio_path, music_path, output_dir, bpm, generate_treatment = true }) => {
|
|
1722
1904
|
const syncFs = require('fs');
|
|
1723
1905
|
const pathMod = require('path');
|
|
1724
1906
|
const { VideoEditor } = require('../video-editor/editor');
|
|
1907
|
+
const { isWebUrl, downloadMediaWithYtDlp } = require('../utils/media-downloader');
|
|
1908
|
+
const { detectPausesWithFFmpeg, snapCutToPause } = require('../utils/transcribe');
|
|
1725
1909
|
const editor = new VideoEditor();
|
|
1726
1910
|
|
|
1727
|
-
// 1. Resolve clip paths
|
|
1911
|
+
// 1. Resolve clip paths (local files, directory, or URLs via yt-dlp)
|
|
1728
1912
|
let clipPaths = [];
|
|
1729
1913
|
if (typeof footage === 'string') {
|
|
1730
|
-
if (
|
|
1914
|
+
if (isWebUrl(footage)) {
|
|
1915
|
+
const dl = downloadMediaWithYtDlp(footage);
|
|
1916
|
+
clipPaths = [dl.localPath];
|
|
1917
|
+
} else if (syncFs.existsSync(footage) && syncFs.statSync(footage).isDirectory()) {
|
|
1731
1918
|
const validExts = ['.mp4', '.mov', '.mkv', '.webm', '.avi'];
|
|
1732
1919
|
clipPaths = syncFs.readdirSync(footage)
|
|
1733
1920
|
.filter(f => validExts.includes(pathMod.extname(f).toLowerCase()))
|
|
@@ -1736,7 +1923,14 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1736
1923
|
clipPaths = [footage];
|
|
1737
1924
|
}
|
|
1738
1925
|
} else if (Array.isArray(footage)) {
|
|
1739
|
-
|
|
1926
|
+
for (const item of footage) {
|
|
1927
|
+
if (isWebUrl(item)) {
|
|
1928
|
+
const dl = downloadMediaWithYtDlp(item);
|
|
1929
|
+
clipPaths.push(dl.localPath);
|
|
1930
|
+
} else {
|
|
1931
|
+
clipPaths.push(item);
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1740
1934
|
}
|
|
1741
1935
|
|
|
1742
1936
|
if (clipPaths.length === 0) {
|
|
@@ -1755,12 +1949,14 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1755
1949
|
probedClips.push({ path: cp, ...meta });
|
|
1756
1950
|
}
|
|
1757
1951
|
|
|
1758
|
-
// 3. Audio & Duration planning
|
|
1952
|
+
// 3. Audio & Duration planning (with speech pause detection)
|
|
1759
1953
|
let totalTarget = target_duration;
|
|
1760
|
-
|
|
1954
|
+
let speechPauses = [];
|
|
1955
|
+
if (audio_path && syncFs.existsSync(audio_path)) {
|
|
1761
1956
|
try {
|
|
1762
1957
|
const aMeta = editor.probeMedia(audio_path);
|
|
1763
|
-
totalTarget = aMeta.duration;
|
|
1958
|
+
if (!totalTarget) totalTarget = aMeta.duration;
|
|
1959
|
+
speechPauses = detectPausesWithFFmpeg(audio_path);
|
|
1764
1960
|
} catch (_) {}
|
|
1765
1961
|
}
|
|
1766
1962
|
if (!totalTarget) {
|
|
@@ -1769,40 +1965,114 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1769
1965
|
|
|
1770
1966
|
// 4. Style configuration mapping
|
|
1771
1967
|
const styleConfig = {
|
|
1772
|
-
cinematic:
|
|
1773
|
-
fast_paced:
|
|
1774
|
-
documentary:
|
|
1775
|
-
vintage:
|
|
1776
|
-
cyberpunk:
|
|
1777
|
-
vibrant:
|
|
1968
|
+
cinematic: { avgDuration: 5.0, speed: 1.0, transition: 'dissolve', grade: 'cinematic' },
|
|
1969
|
+
fast_paced: { avgDuration: 2.2, speed: 1.25, transition: 'wipeleft', grade: 'vibrant' },
|
|
1970
|
+
documentary: { avgDuration: 4.5, speed: 1.0, transition: 'fade', grade: 'warm' },
|
|
1971
|
+
vintage: { avgDuration: 3.5, speed: 0.95, transition: 'fade', grade: 'vintage' },
|
|
1972
|
+
cyberpunk: { avgDuration: 2.5, speed: 1.2, transition: 'circleopen', grade: 'cyberpunk' },
|
|
1973
|
+
vibrant: { avgDuration: 3.0, speed: 1.1, transition: 'slideup', grade: 'vibrant' },
|
|
1974
|
+
product_launch: { avgDuration: 4.0, speed: 1.0, transition: 'dissolve', grade: 'product_launch' },
|
|
1975
|
+
editorial_collage: { avgDuration: 2.8, speed: 1.15, transition: 'slideleft', grade: 'editorial_collage' },
|
|
1976
|
+
retro_vhs: { avgDuration: 3.2, speed: 0.95, transition: 'pixelize', grade: 'retrovhs' },
|
|
1977
|
+
anime_action: { avgDuration: 1.8, speed: 1.3, transition: 'wipeleft', grade: 'anime_action' },
|
|
1978
|
+
sci_fi_hud: { avgDuration: 3.0, speed: 1.1, transition: 'circlecrop', grade: 'scifi' },
|
|
1979
|
+
lofi_chill: { avgDuration: 5.5, speed: 0.9, transition: 'dissolve', grade: 'lofi_chill' },
|
|
1980
|
+
minimal_monochrome: { avgDuration: 3.5, speed: 1.0, transition: 'fade', grade: 'swiss_mono' },
|
|
1981
|
+
// lemo-opuscar extended styles
|
|
1982
|
+
cinematic_anamorphic: { avgDuration: 6.0, speed: 0.95, transition: 'dissolve', grade: 'cinematic_anamorphic' },
|
|
1983
|
+
broadcast_news: { avgDuration: 3.0, speed: 1.0, transition: 'wipeleft', grade: 'broadcast_news' },
|
|
1984
|
+
blueprint_3d: { avgDuration: 4.0, speed: 1.0, transition: 'distance', grade: 'blueprint' },
|
|
1985
|
+
film_16mm: { avgDuration: 4.5, speed: 0.9, transition: 'fade', grade: 'film16mm' },
|
|
1986
|
+
product_obsidian: { avgDuration: 4.5, speed: 1.0, transition: 'fadeblack', grade: 'product_obsidian' },
|
|
1987
|
+
retro_vhs_80s: { avgDuration: 3.0, speed: 0.95, transition: 'pixelize', grade: 'retro_vhs_80s' },
|
|
1778
1988
|
}[style] || { avgDuration: 4.0, speed: 1.0, transition: 'fade', grade: 'cinematic' };
|
|
1779
1989
|
|
|
1780
|
-
// 5. Build Cut Decision List
|
|
1990
|
+
// 5. Build Cut Decision List (pause snapping + beat grid + J/L-cut assignment)
|
|
1991
|
+
// J/L-cut rules (lemo-opuscar DIRECTOR.md):
|
|
1992
|
+
// J-cut: audio_offset < 0 (next audio starts before video cut)
|
|
1993
|
+
// L-cut: audio_offset > 0 (current audio extends over next video)
|
|
1994
|
+
// Hard cut: audio_offset = 0
|
|
1781
1995
|
const clipCount = Math.max(1, Math.min(probedClips.length, Math.ceil(totalTarget / styleConfig.avgDuration)));
|
|
1782
1996
|
const segDur = totalTarget / clipCount;
|
|
1783
1997
|
const plannedClips = [];
|
|
1784
1998
|
|
|
1785
1999
|
const beatNames = ['Opening Hook', 'Context & Build-up', 'Core Revelation / Climax', 'Resolution & Outro'];
|
|
1786
2000
|
|
|
2001
|
+
// Beat-grid helper (HyperFrames formula — round each index independently)
|
|
2002
|
+
const beatAt = (idx) => bpm > 0 ? Math.round(idx * 30 * 60 / bpm) / 30 : null;
|
|
2003
|
+
|
|
2004
|
+
// Determine J/L-cut style from overall style
|
|
2005
|
+
const audioEditStyle = {
|
|
2006
|
+
cinematic: 'j_cut',
|
|
2007
|
+
fast_paced: 'hard',
|
|
2008
|
+
documentary: 'j_cut',
|
|
2009
|
+
vintage: 'l_cut',
|
|
2010
|
+
cyberpunk: 'hard',
|
|
2011
|
+
vibrant: 'hard',
|
|
2012
|
+
cinematic_anamorphic: 'j_cut',
|
|
2013
|
+
broadcast_news: 'hard',
|
|
2014
|
+
film_16mm: 'l_cut',
|
|
2015
|
+
lofi_chill: 'l_cut',
|
|
2016
|
+
anime_action: 'hard',
|
|
2017
|
+
}[style] || 'j_cut';
|
|
2018
|
+
|
|
2019
|
+
let cumulativeTime = 0;
|
|
1787
2020
|
for (let i = 0; i < clipCount; i++) {
|
|
1788
2021
|
const src = probedClips[i % probedClips.length];
|
|
1789
2022
|
const clipDur = Number(segDur.toFixed(2));
|
|
1790
2023
|
const maxStart = Math.max(0, (src.duration || 10) - clipDur);
|
|
1791
|
-
|
|
1792
|
-
|
|
2024
|
+
let startTime = Number((maxStart * 0.25 * (i % 3)).toFixed(2));
|
|
2025
|
+
let rawEndTime = Number((startTime + clipDur).toFixed(2));
|
|
2026
|
+
cumulativeTime += clipDur;
|
|
2027
|
+
|
|
2028
|
+
// Snap cut to audio pause if speech pauses available
|
|
2029
|
+
if (speechPauses.length > 0) {
|
|
2030
|
+
const snappedTime = snapCutToPause(cumulativeTime, speechPauses, 0.75);
|
|
2031
|
+
const shift = snappedTime - cumulativeTime;
|
|
2032
|
+
rawEndTime = Math.max(startTime + 0.5, Number((rawEndTime + shift).toFixed(2)));
|
|
2033
|
+
}
|
|
2034
|
+
|
|
2035
|
+
// Snap cut to beat grid if BPM provided (HyperFrames formula)
|
|
2036
|
+
const beatT = beatAt(i);
|
|
2037
|
+
if (beatT !== null && Math.abs(beatT - cumulativeTime) < segDur * 0.25) {
|
|
2038
|
+
const shift = beatT - cumulativeTime;
|
|
2039
|
+
rawEndTime = Math.max(startTime + 0.5, Number((rawEndTime + shift).toFixed(2)));
|
|
2040
|
+
}
|
|
2041
|
+
|
|
1793
2042
|
const beatIndex = Math.min(beatNames.length - 1, Math.floor((i / clipCount) * beatNames.length));
|
|
1794
2043
|
|
|
2044
|
+
// Assign J/L-cut type and audio_offset
|
|
2045
|
+
let cutType = audioEditStyle;
|
|
2046
|
+
let audioOffset = 0;
|
|
2047
|
+
if (i === 0) {
|
|
2048
|
+
// First cut: always fade in audio — no J/L on first clip
|
|
2049
|
+
cutType = 'hard';
|
|
2050
|
+
audioOffset = 0;
|
|
2051
|
+
} else if (i === clipCount - 1) {
|
|
2052
|
+
// Last cut: always fade out — L-cut slightly to smooth ending
|
|
2053
|
+
cutType = 'l_cut';
|
|
2054
|
+
audioOffset = 0.3;
|
|
2055
|
+
} else if (cutType === 'j_cut') {
|
|
2056
|
+
// J-cut: next audio starts 0.25–0.5s before video cut
|
|
2057
|
+
audioOffset = -(styleConfig.avgDuration > 4.0 ? 0.5 : 0.25);
|
|
2058
|
+
} else if (cutType === 'l_cut') {
|
|
2059
|
+
// L-cut: current audio extends 0.3s over next clip's video
|
|
2060
|
+
audioOffset = 0.3;
|
|
2061
|
+
}
|
|
2062
|
+
|
|
1795
2063
|
plannedClips.push({
|
|
1796
2064
|
video_path: src.path,
|
|
1797
2065
|
start_time: startTime,
|
|
1798
|
-
end_time:
|
|
1799
|
-
duration:
|
|
2066
|
+
end_time: rawEndTime,
|
|
2067
|
+
duration: Number((rawEndTime - startTime).toFixed(2)),
|
|
1800
2068
|
speed: styleConfig.speed,
|
|
1801
2069
|
color_grade: styleConfig.grade,
|
|
1802
2070
|
transition: i === clipCount - 1 ? 'none' : styleConfig.transition,
|
|
1803
2071
|
transition_duration: 0.5,
|
|
1804
2072
|
beat: beatNames[beatIndex],
|
|
1805
|
-
|
|
2073
|
+
cut_type: cutType,
|
|
2074
|
+
audio_offset: audioOffset,
|
|
2075
|
+
rationale: `${beatNames[beatIndex]} — ${clipDur}s, ${styleConfig.grade} grade, ${styleConfig.speed}x speed. Cut: ${cutType === 'j_cut' ? `J-cut (audio precedes by ${Math.abs(audioOffset)}s)` : cutType === 'l_cut' ? `L-cut (audio extends ${audioOffset}s)` : 'hard cut'}.${beatT !== null ? ` Beat-snapped to ${beatT.toFixed(2)}s.` : ''}`,
|
|
1806
2076
|
});
|
|
1807
2077
|
}
|
|
1808
2078
|
|
|
@@ -1816,44 +2086,730 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1816
2086
|
color_grade: styleConfig.grade,
|
|
1817
2087
|
transition: styleConfig.transition,
|
|
1818
2088
|
target_duration: totalTarget,
|
|
2089
|
+
bpm: bpm || null,
|
|
2090
|
+
audio_edit_style: audioEditStyle,
|
|
1819
2091
|
};
|
|
1820
2092
|
|
|
1821
2093
|
// 7. Build edit-plan.md
|
|
1822
2094
|
let planMd = `# 🎬 Video Edit Decision Plan\n\n`;
|
|
1823
2095
|
planMd += `**Goal**: ${goal}\n`;
|
|
1824
|
-
planMd += `**Style**: ${style.toUpperCase()} | **Orientation**: ${orientation.toUpperCase()} | **Target Duration**: ${totalTarget.toFixed(1)}s | **Clips
|
|
1825
|
-
planMd +=
|
|
1826
|
-
planMd +=
|
|
1827
|
-
planMd += `|
|
|
2096
|
+
planMd += `**Style**: ${style.toUpperCase()} | **Orientation**: ${orientation.toUpperCase()} | **Target Duration**: ${totalTarget.toFixed(1)}s | **Clips**: ${plannedClips.length} | **Audio Edit**: ${audioEditStyle.toUpperCase().replace('_', '-')}\n`;
|
|
2097
|
+
if (bpm) planMd += `**BPM**: ${bpm} (cut points beat-snapped via HyperFrames formula)\n`;
|
|
2098
|
+
planMd += `\n## 📋 Narrative Beat Architecture\n\n`;
|
|
2099
|
+
planMd += `| Beat | # | Clip | In–Out | Dur | Speed | Grade | Transition | Cut Type | Rationale |\n`;
|
|
2100
|
+
planMd += `| :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n`;
|
|
1828
2101
|
|
|
1829
2102
|
plannedClips.forEach((c, idx) => {
|
|
1830
2103
|
const basename = pathMod.basename(c.video_path);
|
|
1831
|
-
|
|
2104
|
+
const cutLabel = c.cut_type === 'j_cut'
|
|
2105
|
+
? `🎵 J-cut (${Math.abs(c.audio_offset)}s)`
|
|
2106
|
+
: c.cut_type === 'l_cut'
|
|
2107
|
+
? `🔊 L-cut (+${c.audio_offset}s)`
|
|
2108
|
+
: '✂️ Hard';
|
|
2109
|
+
planMd += `| **${c.beat}** | ${idx + 1} | \`${basename}\` | ${c.start_time}s–${c.end_time}s | ${c.duration}s | ${c.speed}x | ${c.color_grade} | ${c.transition} | ${cutLabel} | ${c.rationale} |\n`;
|
|
1832
2110
|
});
|
|
1833
2111
|
|
|
1834
2112
|
planMd += `\n## 🎵 Audio Architecture\n\n`;
|
|
1835
2113
|
planMd += `- **Master Voiceover**: ${audio_path ? `\`${pathMod.basename(audio_path)}\` (Primary, ducks music by 18dB)` : 'None specified (using clip audio)'}\n`;
|
|
1836
|
-
planMd += `- **Background Music**: ${music_path ? `\`${pathMod.basename(music_path)}\` (Ambient, volume: 0.08)` : 'None specified'}\n
|
|
2114
|
+
planMd += `- **Background Music**: ${music_path ? `\`${pathMod.basename(music_path)}\` (Ambient, volume: 0.08)` : 'None specified'}\n`;
|
|
2115
|
+
planMd += `- **Audio Edit Style**: ${audioEditStyle === 'j_cut' ? '🎵 J-cut dominant (audio leads video by 0.25–0.5s for softened edits)' : audioEditStyle === 'l_cut' ? '🔊 L-cut dominant (audio trails into next shot by 0.3s)' : '✂️ Hard cuts (audio and video synchronized)'}\n\n`;
|
|
2116
|
+
planMd += `## 🎬 Audio Cut Rules Applied\n\n`;
|
|
2117
|
+
planMd += `- First clip: always hard cut (clean opening)\n`;
|
|
2118
|
+
planMd += `- Last clip: always L-cut (+0.3s) for smooth outro fade\n`;
|
|
2119
|
+
planMd += `- J-cuts: audio of NEXT clip starts ${styleConfig.avgDuration > 4.0 ? '0.5s' : '0.25s'} before video edit point\n`;
|
|
2120
|
+
planMd += `- L-cuts: current audio extends 0.3s into next clip's video\n\n`;
|
|
1837
2121
|
planMd += `## 🚀 Execution & Approval\n\n`;
|
|
1838
|
-
planMd += `
|
|
2122
|
+
planMd += `Review this plan, then execute with \`edit_video\` using the payload in \`edit.json\`.\n`;
|
|
2123
|
+
planMd += `\nGenerated by FrameForge v2.7.0 — plan_video_edit with J/L-cut support (lemo-opuscar DIRECTOR.md)\n`;
|
|
2124
|
+
|
|
2125
|
+
// 8. Build TREATMENT.md (lemo-opuscar DIRECTOR.md — 8-section format)
|
|
2126
|
+
let treatmentMd = '';
|
|
2127
|
+
if (generate_treatment) {
|
|
2128
|
+
const clipList = plannedClips.map((c, idx) =>
|
|
2129
|
+
` ${idx + 1}. \`${pathMod.basename(c.video_path)}\` (${c.beat}, ${c.duration}s, ${c.cut_type.replace('_', '-')})`
|
|
2130
|
+
).join('\n');
|
|
2131
|
+
|
|
2132
|
+
treatmentMd = `# 🎬 TREATMENT\n\n`;
|
|
2133
|
+
treatmentMd += `> Generated by FrameForge plan_video_edit — edit this document to refine creative direction before rendering.\n\n`;
|
|
2134
|
+
treatmentMd += `---\n\n`;
|
|
2135
|
+
treatmentMd += `## 1. Concept\n\n${goal}\n\n`;
|
|
2136
|
+
treatmentMd += `## 2. Tone\n\n`;
|
|
2137
|
+
treatmentMd += `${style === 'cinematic' || style === 'cinematic_anamorphic' ? 'Cinematic and deliberate. Each frame earns its place.' : style === 'fast_paced' || style === 'anime_action' ? 'High-energy, kinetic. The edit IS the emotion.' : style === 'documentary' ? 'Observational and authentic. Let reality speak.' : style === 'vintage' || style === 'film_16mm' ? 'Warm, nostalgic, imperfect. Texture over polish.' : 'Direct and clean. Clarity over decoration.'}\n\n`;
|
|
2138
|
+
treatmentMd += `## 3. Visual Language\n\n`;
|
|
2139
|
+
treatmentMd += `- **Color**: ${styleConfig.grade} grade\n`;
|
|
2140
|
+
treatmentMd += `- **Orientation**: ${orientation}\n`;
|
|
2141
|
+
treatmentMd += `- **Transitions**: ${styleConfig.transition} (${styleConfig.avgDuration}s avg clip)\n`;
|
|
2142
|
+
treatmentMd += `- **Speed**: ${styleConfig.speed}x base${styleConfig.speed !== 1.0 ? ' (variable)' : ''}\n\n`;
|
|
2143
|
+
treatmentMd += `## 4. Sound\n\n`;
|
|
2144
|
+
treatmentMd += `- **Audio edit style**: ${audioEditStyle === 'j_cut' ? 'J-cut dominant — audio of next scene starts before video cut to soften transitions' : audioEditStyle === 'l_cut' ? 'L-cut dominant — current audio trails into next shot for continuity' : 'Hard cuts — audio and video locked to same edit point'}\n`;
|
|
2145
|
+
treatmentMd += `- **Voiceover**: ${audio_path ? pathMod.basename(audio_path) : 'None'}\n`;
|
|
2146
|
+
treatmentMd += `- **Music**: ${music_path ? pathMod.basename(music_path) : 'None'}${bpm ? ` @ ${bpm} BPM` : ''}\n`;
|
|
2147
|
+
treatmentMd += `- **Priority order** (lemo-opuscar): Sound → Rhythm → Camera → Directing\n\n`;
|
|
2148
|
+
treatmentMd += `## 5. Narrative Arc\n\n`;
|
|
2149
|
+
treatmentMd += `| Act | Beat | Description |\n| :--- | :--- | :--- |\n`;
|
|
2150
|
+
treatmentMd += `| Act 1 | Opening Hook | Establish world, hook the viewer in first 3s |\n`;
|
|
2151
|
+
treatmentMd += `| Act 2a | Context & Build-up | Deepen context, build anticipation |\n`;
|
|
2152
|
+
treatmentMd += `| Act 2b | Core Revelation / Climax | The emotional or informational peak |\n`;
|
|
2153
|
+
treatmentMd += `| Act 3 | Resolution & Outro | Resolve, call to action, or linger |\n\n`;
|
|
2154
|
+
treatmentMd += `## 6. Shot List\n\n${clipList}\n\n`;
|
|
2155
|
+
treatmentMd += `## 7. Delivery\n\n`;
|
|
2156
|
+
treatmentMd += `- **Format**: MP4, H.264 high profile, CRF 18\n`;
|
|
2157
|
+
treatmentMd += `- **Resolution**: ${orientation === 'portrait' ? '1080×1920 (9:16)' : orientation === 'square' ? '1080×1080 (1:1)' : '1920×1080 (16:9)'}\n`;
|
|
2158
|
+
treatmentMd += `- **Duration**: ${totalTarget.toFixed(1)}s\n`;
|
|
2159
|
+
treatmentMd += `- **Audio**: AAC 192kbps, 48kHz, EBU R128 normalized\n`;
|
|
2160
|
+
treatmentMd += `- **Platform target**: YouTube / Instagram (−14 LUFS, −1 dBTP)\n\n`;
|
|
2161
|
+
treatmentMd += `## 8. Director Notes\n\n`;
|
|
2162
|
+
treatmentMd += `_Add creative direction, mood references, or client notes here._\n\n`;
|
|
2163
|
+
treatmentMd += `---\n\n`;
|
|
2164
|
+
treatmentMd += `*Generated by FrameForge v2.7.0 — lemo-opuscar DIRECTOR.md format*\n`;
|
|
2165
|
+
}
|
|
1839
2166
|
|
|
1840
|
-
//
|
|
2167
|
+
// 9. Write outputs
|
|
1841
2168
|
const outDir = output_dir || pathMod.join(FILES_DIR, 'edits', `edit_${Date.now()}`);
|
|
1842
2169
|
syncFs.mkdirSync(outDir, { recursive: true });
|
|
1843
|
-
const planPath
|
|
1844
|
-
const jsonPath
|
|
2170
|
+
const planPath = pathMod.join(outDir, 'edit-plan.md');
|
|
2171
|
+
const jsonPath = pathMod.join(outDir, 'edit.json');
|
|
2172
|
+
const treatmentPath = pathMod.join(outDir, 'TREATMENT.md');
|
|
1845
2173
|
|
|
1846
2174
|
syncFs.writeFileSync(planPath, planMd, 'utf8');
|
|
1847
2175
|
syncFs.writeFileSync(jsonPath, JSON.stringify(editJson, null, 2), 'utf8');
|
|
2176
|
+
if (generate_treatment) {
|
|
2177
|
+
syncFs.writeFileSync(treatmentPath, treatmentMd, 'utf8');
|
|
2178
|
+
}
|
|
1848
2179
|
|
|
1849
2180
|
return {
|
|
1850
2181
|
planPath,
|
|
1851
2182
|
editJsonPath: jsonPath,
|
|
2183
|
+
treatmentPath: generate_treatment ? treatmentPath : null,
|
|
1852
2184
|
planMarkdown: planMd,
|
|
1853
2185
|
editConfig: editJson,
|
|
1854
2186
|
clipCount: plannedClips.length,
|
|
1855
2187
|
totalDuration: totalTarget,
|
|
1856
|
-
|
|
2188
|
+
audioEditStyle,
|
|
2189
|
+
message: `Edit plan, edit.json${generate_treatment ? ', and TREATMENT.md' : ''} generated. Review edit-plan.md, approve, then call edit_video with edit.json.`,
|
|
2190
|
+
};
|
|
2191
|
+
},
|
|
2192
|
+
},
|
|
2193
|
+
|
|
2194
|
+
// ── Tool 24: Sync Upstream Skills ─────────────────────────────────────────
|
|
2195
|
+
// Transparently checks & merges updates from upstream video skills repositories
|
|
2196
|
+
sync_upstream_skills: {
|
|
2197
|
+
description: `Check and synchronize latest upstream skills, motion craft rules, film styles, and audio presets from 9 reference repositories (HyperFrames, OneTake, LemoOpuscar, CollageBroll, Remotion, Earshot, ClaudeAnimationBase, ClaudeRemotionSkill, FFmpegSkill). Makes network requests to GitHub API to check for new commits. Returns detailed per-category update summary. Use check_upstream_updates for a fast offline check of the last sync state.`,
|
|
2198
|
+
inputSchema: {
|
|
2199
|
+
type: 'object',
|
|
2200
|
+
properties: {
|
|
2201
|
+
silent: { type: 'boolean', default: false, description: 'Suppress verbose console logging' },
|
|
2202
|
+
},
|
|
2203
|
+
},
|
|
2204
|
+
handler: async ({ silent = false }) => {
|
|
2205
|
+
const { syncSkills } = require('../skills/sync-upstream');
|
|
2206
|
+
const res = await syncSkills({ silent });
|
|
2207
|
+
const updatedRepos = res.results.filter(r => r.updated).map(r => r.repo);
|
|
2208
|
+
const perCategoryUpdates = res.perCategoryUpdates || {};
|
|
2209
|
+
|
|
2210
|
+
// Build a rich user-facing update notice per category
|
|
2211
|
+
let updateNotice = '';
|
|
2212
|
+
if (updatedRepos.length > 0) {
|
|
2213
|
+
updateNotice = '\n\n⬆️ UPSTREAM UPDATES AVAILABLE:\n';
|
|
2214
|
+
for (const [cat, catUpdates] of Object.entries(perCategoryUpdates)) {
|
|
2215
|
+
for (const u of catUpdates) {
|
|
2216
|
+
updateNotice += ` [${cat}] ${u.repo} → ${u.sha}\n`;
|
|
2217
|
+
updateNotice += ` ${u.userImpact}\n`;
|
|
2218
|
+
if (u.capabilities.length > 0) {
|
|
2219
|
+
updateNotice += ` Tools affected: ${u.capabilities.join(', ')}\n`;
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
updateNotice += '\nNo action required — FrameForge automatically applied the updates. Your next render uses the latest techniques.';
|
|
2224
|
+
}
|
|
2225
|
+
return {
|
|
2226
|
+
success: true,
|
|
2227
|
+
updatedCount: res.updatedCount,
|
|
2228
|
+
repos: res.results,
|
|
2229
|
+
updatedRepos,
|
|
2230
|
+
perCategoryUpdates,
|
|
2231
|
+
lastSync: res.manifest.lastSync,
|
|
2232
|
+
message: `Synced ${res.results.length} upstream skill repositories (${res.updatedCount} updates merged).${updateNotice}`,
|
|
2233
|
+
};
|
|
2234
|
+
},
|
|
2235
|
+
},
|
|
2236
|
+
|
|
2237
|
+
// ── Tool 28: Check Upstream Updates (fast, offline) ───────────────────────
|
|
2238
|
+
// Reads last sync manifest without network requests — returns update status instantly
|
|
2239
|
+
check_upstream_updates: {
|
|
2240
|
+
description: `Check the last sync state of all upstream reference repositories (HyperFrames, OneTake, LemoOpuscar, CollageBroll, Remotion, Earshot, ClaudeAnimationBase, ClaudeRemotionSkill, FFmpegSkill). Fast, offline — reads the local manifest only (no network requests). Shows what was last synced, when, which repos had updates, and a human-readable changelog of what improved. To actually fetch new commits and update the manifest, use sync_upstream_skills. Agents and users can call this anytime to see if upstream skills have been updated since the last sync.`,
|
|
2241
|
+
inputSchema: { type: 'object', properties: {} },
|
|
2242
|
+
handler: async () => {
|
|
2243
|
+
const syncFs = require('fs');
|
|
2244
|
+
const { MANIFEST_PATH, REPOSITORIES } = require('../skills/sync-upstream');
|
|
2245
|
+
let manifest = { lastSync: null, repos: {}, version: '2.7.0', pendingUpdates: [] };
|
|
2246
|
+
if (syncFs.existsSync(MANIFEST_PATH)) {
|
|
2247
|
+
try { manifest = JSON.parse(syncFs.readFileSync(MANIFEST_PATH, 'utf8')); } catch (_) {}
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
const repos = REPOSITORIES.map(r => {
|
|
2251
|
+
const m = manifest.repos[r.id] || {};
|
|
2252
|
+
return {
|
|
2253
|
+
id: r.id,
|
|
2254
|
+
repo: r.repo,
|
|
2255
|
+
category: r.category,
|
|
2256
|
+
desc: r.desc,
|
|
2257
|
+
capabilities: r.capabilities || [],
|
|
2258
|
+
sha: m.sha || 'not-synced',
|
|
2259
|
+
lastChecked: m.lastChecked || null,
|
|
2260
|
+
status: m.status || 'unknown',
|
|
2261
|
+
userImpact: r.userImpact,
|
|
2262
|
+
};
|
|
2263
|
+
});
|
|
2264
|
+
|
|
2265
|
+
const syncedCount = repos.filter(r => r.sha !== 'not-synced').length;
|
|
2266
|
+
const neverSynced = repos.filter(r => r.sha === 'not-synced').map(r => r.repo);
|
|
2267
|
+
const pendingUpdates = manifest.pendingUpdates || [];
|
|
2268
|
+
|
|
2269
|
+
// Build human-readable changelog notice
|
|
2270
|
+
let changelogNotice = '';
|
|
2271
|
+
if (pendingUpdates.length > 0) {
|
|
2272
|
+
changelogNotice = '\n\n📦 UPSTREAM CHANGES SINCE LAST SYNC:\n';
|
|
2273
|
+
for (const u of pendingUpdates) {
|
|
2274
|
+
changelogNotice += ` • [${u.category || '?'}] ${u.repo} → ${u.sha}\n`;
|
|
2275
|
+
changelogNotice += ` ${u.userImpact || 'Updated techniques merged'}\n`;
|
|
2276
|
+
}
|
|
2277
|
+
changelogNotice += '\nRun sync_upstream_skills to pull these updates and apply them to your next render.';
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
let statusMsg = `Last sync: ${manifest.lastSync || 'never'}. ${syncedCount}/${repos.length} repos tracked.`;
|
|
2281
|
+
if (neverSynced.length > 0) {
|
|
2282
|
+
statusMsg += ` Not yet synced: ${neverSynced.join(', ')}. Run sync_upstream_skills to fetch latest.`;
|
|
2283
|
+
} else if (pendingUpdates.length > 0) {
|
|
2284
|
+
statusMsg += ` ${pendingUpdates.length} upstream repo(s) have new commits. Run sync_upstream_skills to apply them.${changelogNotice}`;
|
|
2285
|
+
} else {
|
|
2286
|
+
statusMsg += ' All repos are current. Run sync_upstream_skills to check for new commits from GitHub.';
|
|
2287
|
+
}
|
|
2288
|
+
|
|
2289
|
+
return {
|
|
2290
|
+
success: true,
|
|
2291
|
+
lastSync: manifest.lastSync,
|
|
2292
|
+
manifestVersion: manifest.version || '2.7.0',
|
|
2293
|
+
totalRepos: repos.length,
|
|
2294
|
+
syncedCount,
|
|
2295
|
+
pendingUpdates,
|
|
2296
|
+
hasUpdates: pendingUpdates.length > 0,
|
|
2297
|
+
repos,
|
|
2298
|
+
message: statusMsg,
|
|
2299
|
+
hint: pendingUpdates.length > 0
|
|
2300
|
+
? `⬆️ ${pendingUpdates.length} upstream update(s) available — run sync_upstream_skills to apply`
|
|
2301
|
+
: 'Run sync_upstream_skills to check for new commits from GitHub.',
|
|
2302
|
+
};
|
|
2303
|
+
},
|
|
2304
|
+
},
|
|
2305
|
+
|
|
2306
|
+
// ── Tool 25: Transcribe Media ──────────────────────────────────────────────
|
|
2307
|
+
// Whisper speech transcription — word-level timestamps, SRT/NDJSON export
|
|
2308
|
+
transcribe_media: {
|
|
2309
|
+
description: `Transcribe speech from any audio or video file using OpenAI Whisper (local, free, no API key required).
|
|
2310
|
+
Returns word-level timestamps for precise animation sync, SRT subtitles for closed captions, and silence/pause boundaries for intelligent cut alignment.
|
|
2311
|
+
Supports: openai-whisper (local GPU/CPU), faster-whisper (CTranslate2), whisper-cli binary, or OpenAI Whisper API fallback.
|
|
2312
|
+
Output formats: ndjson (one segment per line — used by engine.js captions()), srt (standard subtitle file), json (full word/segment data).
|
|
2313
|
+
Use before writing edit.js to get cuePhrase() timestamps. Use on user-supplied footage to get cut points aligned to speech pauses.`,
|
|
2314
|
+
inputSchema: {
|
|
2315
|
+
type: 'object',
|
|
2316
|
+
required: ['mediaPath'],
|
|
2317
|
+
properties: {
|
|
2318
|
+
mediaPath: { type: 'string', description: 'Absolute path to audio or video file (mp4, mp3, m4a, wav, mov, webm)' },
|
|
2319
|
+
model: { type: 'string', default: 'base', enum: ['tiny', 'base', 'small', 'medium', 'large'], description: 'Whisper model size (base = fast, small/medium = accurate)' },
|
|
2320
|
+
language: { type: 'string', description: 'BCP-47 language code hint (e.g. "en", "ml", "hi"). Auto-detected if omitted.' },
|
|
2321
|
+
format: { type: 'string', default: 'ndjson', enum: ['ndjson', 'srt', 'json', 'all'], description: 'Output format: ndjson for engine.js captions(), srt for subtitle files, json for full data, all for everything' },
|
|
2322
|
+
outputPath: { type: 'string', description: 'Path to write output file. Auto-generated in .frameforge-files/transcripts/ if omitted.' },
|
|
2323
|
+
},
|
|
2324
|
+
},
|
|
2325
|
+
handler: async ({ mediaPath, model = 'base', language, format = 'ndjson', outputPath }) => {
|
|
2326
|
+
const { transcribeAudio, detectWhisper } = require('../utils/transcribe');
|
|
2327
|
+
const syncFs = require('fs');
|
|
2328
|
+
const pathMod = require('path');
|
|
2329
|
+
|
|
2330
|
+
if (!syncFs.existsSync(mediaPath)) {
|
|
2331
|
+
throw new Error(`Media file not found: ${mediaPath}`);
|
|
2332
|
+
}
|
|
2333
|
+
|
|
2334
|
+
const engine = detectWhisper();
|
|
2335
|
+
if (!engine.available) {
|
|
2336
|
+
throw new Error(
|
|
2337
|
+
'No Whisper engine detected. Install one of:\n' +
|
|
2338
|
+
' pip install openai-whisper (local, free)\n' +
|
|
2339
|
+
' pip install faster-whisper (faster, local)\n' +
|
|
2340
|
+
' Or set OPENAI_API_KEY in .env for cloud fallback.'
|
|
2341
|
+
);
|
|
2342
|
+
}
|
|
2343
|
+
|
|
2344
|
+
const result = await transcribeAudio(mediaPath, { model, language });
|
|
2345
|
+
|
|
2346
|
+
const transcriptsDir = pathMod.join(FILES_DIR, 'transcripts');
|
|
2347
|
+
syncFs.mkdirSync(transcriptsDir, { recursive: true });
|
|
2348
|
+
|
|
2349
|
+
const basename = pathMod.basename(mediaPath, pathMod.extname(mediaPath));
|
|
2350
|
+
const outputs = {};
|
|
2351
|
+
|
|
2352
|
+
// NDJSON — used by engine.js captions() and cuePhrase()
|
|
2353
|
+
if (format === 'ndjson' || format === 'all') {
|
|
2354
|
+
const ndjsonPath = outputPath && format !== 'all'
|
|
2355
|
+
? outputPath
|
|
2356
|
+
: pathMod.join(transcriptsDir, `${basename}.captions.ndjson`);
|
|
2357
|
+
const segments = result.segments.length > 0 ? result.segments : result.words.map(w => ({ start: w.start, end: w.end, text: w.word }));
|
|
2358
|
+
const ndjson = segments.map(s => JSON.stringify({ start: s.start, end: s.end, text: s.text })).join('\n');
|
|
2359
|
+
syncFs.writeFileSync(ndjsonPath, ndjson, 'utf8');
|
|
2360
|
+
outputs.ndjson = ndjsonPath;
|
|
2361
|
+
}
|
|
2362
|
+
|
|
2363
|
+
// SRT — standard subtitle file
|
|
2364
|
+
if (format === 'srt' || format === 'all') {
|
|
2365
|
+
const srtPath = outputPath && format !== 'all'
|
|
2366
|
+
? outputPath
|
|
2367
|
+
: pathMod.join(transcriptsDir, `${basename}.srt`);
|
|
2368
|
+
const srtLines = [];
|
|
2369
|
+
const segs = result.segments.length > 0 ? result.segments : [];
|
|
2370
|
+
segs.forEach((s, i) => {
|
|
2371
|
+
const toSrtTime = (sec) => {
|
|
2372
|
+
const h = Math.floor(sec / 3600).toString().padStart(2, '0');
|
|
2373
|
+
const m = Math.floor((sec % 3600) / 60).toString().padStart(2, '0');
|
|
2374
|
+
const ss = Math.floor(sec % 60).toString().padStart(2, '0');
|
|
2375
|
+
const ms = Math.round((sec % 1) * 1000).toString().padStart(3, '0');
|
|
2376
|
+
return `${h}:${m}:${ss},${ms}`;
|
|
2377
|
+
};
|
|
2378
|
+
srtLines.push(`${i + 1}\n${toSrtTime(s.start)} --> ${toSrtTime(s.end)}\n${s.text}\n`);
|
|
2379
|
+
});
|
|
2380
|
+
syncFs.writeFileSync(srtPath, srtLines.join('\n'), 'utf8');
|
|
2381
|
+
outputs.srt = srtPath;
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
// JSON — full structured data
|
|
2385
|
+
if (format === 'json' || format === 'all') {
|
|
2386
|
+
const jsonPath = outputPath && format !== 'all'
|
|
2387
|
+
? outputPath
|
|
2388
|
+
: pathMod.join(transcriptsDir, `${basename}.transcript.json`);
|
|
2389
|
+
syncFs.writeFileSync(jsonPath, JSON.stringify(result, null, 2), 'utf8');
|
|
2390
|
+
outputs.json = jsonPath;
|
|
2391
|
+
}
|
|
2392
|
+
|
|
2393
|
+
return {
|
|
2394
|
+
engine: engine.engine,
|
|
2395
|
+
model,
|
|
2396
|
+
language: language || 'auto-detected',
|
|
2397
|
+
text: result.text,
|
|
2398
|
+
duration: result.duration,
|
|
2399
|
+
wordCount: result.words.length,
|
|
2400
|
+
segmentCount: result.segments.length,
|
|
2401
|
+
pauseCount: result.pauses.length,
|
|
2402
|
+
pauses: result.pauses.slice(0, 10), // first 10 pauses for cut alignment
|
|
2403
|
+
outputs,
|
|
2404
|
+
message: `Transcribed ${pathMod.basename(mediaPath)} (${result.duration.toFixed(1)}s, ${result.words.length} words) via ${engine.engine}. Load in edit.js: captions('${outputs.ndjson || outputs.json || ''}')`,
|
|
2405
|
+
};
|
|
2406
|
+
},
|
|
2407
|
+
},
|
|
2408
|
+
|
|
2409
|
+
// ── Tool 26: Ingest URL ────────────────────────────────────────────────────
|
|
2410
|
+
// Download video/audio from any web URL (YouTube, Vimeo, Twitter, direct MP4)
|
|
2411
|
+
ingest_url: {
|
|
2412
|
+
description: `Download video or audio from any web URL to a local file using yt-dlp.
|
|
2413
|
+
Supports YouTube, Vimeo, Twitter/X, Instagram, TikTok, Twitch, SoundCloud, direct MP4/MP3 URLs, and 1000+ other sites.
|
|
2414
|
+
The downloaded file can then be passed to edit_video (video_path), multi_voice_tts (audio_path), transcribe_media, or any scene's video_path / audio_path field.
|
|
2415
|
+
Requires yt-dlp: pip install yt-dlp (or: winget install yt-dlp)`,
|
|
2416
|
+
inputSchema: {
|
|
2417
|
+
type: 'object',
|
|
2418
|
+
required: ['url'],
|
|
2419
|
+
properties: {
|
|
2420
|
+
url: { type: 'string', description: 'Web URL to download (YouTube, Vimeo, Twitter, direct MP4, etc.)' },
|
|
2421
|
+
format: { type: 'string', default: 'video', enum: ['video', 'audio'], description: 'Download video (mp4) or audio-only (mp3)' },
|
|
2422
|
+
outputDir: { type: 'string', description: 'Directory to save downloaded file. Defaults to .frameforge-files/downloads/' },
|
|
2423
|
+
filename: { type: 'string', description: 'Output filename without extension. Auto-generated from URL hash if omitted.' },
|
|
2424
|
+
maxDuration:{ type: 'number', default: 600, description: 'Maximum allowed duration in seconds (safety clamp, default 10 min)' },
|
|
2425
|
+
subtitles: { type: 'boolean', default: false, description: 'Also download subtitles / closed captions as SRT file' },
|
|
2426
|
+
subLang: { type: 'string', default: 'en', description: 'Subtitle language code (default: "en")' },
|
|
2427
|
+
},
|
|
2428
|
+
},
|
|
2429
|
+
handler: async ({ url, format = 'video', outputDir, filename, maxDuration = 600, subtitles = false, subLang = 'en' }) => {
|
|
2430
|
+
const { downloadMediaWithYtDlp, detectYtDlp, isWebUrl } = require('../utils/media-downloader');
|
|
2431
|
+
|
|
2432
|
+
if (!isWebUrl(url)) {
|
|
2433
|
+
throw new Error(`Invalid URL: ${url}. Must be a valid http:// or https:// URL.`);
|
|
2434
|
+
}
|
|
2435
|
+
if (!detectYtDlp()) {
|
|
2436
|
+
throw new Error(
|
|
2437
|
+
'yt-dlp not found in PATH. Install it:\n' +
|
|
2438
|
+
' pip install yt-dlp\n' +
|
|
2439
|
+
' winget install yt-dlp (Windows)\n' +
|
|
2440
|
+
' brew install yt-dlp (macOS)\n' +
|
|
2441
|
+
' apt install yt-dlp (Debian/Ubuntu)'
|
|
2442
|
+
);
|
|
2443
|
+
}
|
|
2444
|
+
|
|
2445
|
+
const result = downloadMediaWithYtDlp(url, { format, outputDir, filename, maxDuration, subtitles, subLang });
|
|
2446
|
+
|
|
2447
|
+
return {
|
|
2448
|
+
success: true,
|
|
2449
|
+
localPath: result.localPath,
|
|
2450
|
+
subtitlePath: result.subtitlePath || null,
|
|
2451
|
+
format: result.format,
|
|
2452
|
+
duration: result.duration,
|
|
2453
|
+
width: result.width,
|
|
2454
|
+
height: result.height,
|
|
2455
|
+
url,
|
|
2456
|
+
message: `Downloaded ${format === 'audio' ? 'audio' : 'video'} to ${result.localPath} (${result.duration ? result.duration.toFixed(1) + 's' : 'duration unknown'}). Pass localPath to edit_video clips, transcribe_media, or scene video_path / audio_path fields.`,
|
|
2457
|
+
};
|
|
2458
|
+
},
|
|
2459
|
+
},
|
|
2460
|
+
|
|
2461
|
+
// ── Tool 27: Mix Audio Pro ─────────────────────────────────────────────────
|
|
2462
|
+
// Professional audio post-production: EQ, de-esser, reverb, stereo imaging, SFX
|
|
2463
|
+
mix_audio_pro: {
|
|
2464
|
+
description: `Professional audio post-production mixing — 15+ year sound engineer signal chain.
|
|
2465
|
+
|
|
2466
|
+
FULL CHAIN (in order):
|
|
2467
|
+
1. Noise gate: agate -60dBFS threshold — removes breath/room tone between words (enable with noiseGate: true)
|
|
2468
|
+
2. Pre-EQ: 80Hz HPF (remove proximity/rumble), 5kHz presence boost (+2dB), 12kHz air shelf (+1.5dB)
|
|
2469
|
+
3. De-esser: 6.5kHz dynamic suppression (-3dB) — removes harshness from "s", "sh", "ch"
|
|
2470
|
+
4. Voice compressor: 3:1 ratio, -20dBFS threshold, 5ms attack, 50ms release (broadcast standard)
|
|
2471
|
+
5. Room acoustics: 5 named profiles — studio_dry | broadcast | cinematic | concert_hall | intimate
|
|
2472
|
+
OR earshot forgeComputedRoom: "computed:sizeM:damp:earlyMs" (e.g. "computed:8:0.5:15" = 8m studio)
|
|
2473
|
+
(addReverb: true = cinematic profile; roomProfile overrides)
|
|
2474
|
+
6. Sidechain ducking: music ducks -18dB when voice active (40ms perceptual lead)
|
|
2475
|
+
7. Stereo widener: M/S extrastereo 1.4x on music layer only (voice stays mono-center)
|
|
2476
|
+
8. SFX timeline mixer: 13 synthesis types with millisecond accuracy
|
|
2477
|
+
Types: whoosh | impact | pop | click | riser | chime | buzz | tick | swish | punch | sparkle | deep_boom | transition
|
|
2478
|
+
9. Master bus: two-pass EBU R128 loudnorm at target LUFS, true peak ≤-1 dBTP
|
|
2479
|
+
Platform presets: youtube (-14) | podcast (-16) | broadcast (-23) | netflix (-27) | instagram (-14)
|
|
2480
|
+
|
|
2481
|
+
Outputs broadcast-ready 48kHz/192k AAC stereo.`,
|
|
2482
|
+
inputSchema: {
|
|
2483
|
+
type: 'object',
|
|
2484
|
+
required: ['voicePath', 'outputPath'],
|
|
2485
|
+
properties: {
|
|
2486
|
+
voicePath: { type: 'string', description: 'Primary narration / voice audio file (m4a, mp3, wav)' },
|
|
2487
|
+
outputPath: { type: 'string', description: 'Output mixed audio file path (.m4a)' },
|
|
2488
|
+
musicPath: { type: 'string', description: 'Optional background music track. Will be looped, high-passed, and sidechain ducked.' },
|
|
2489
|
+
musicVolume: { type: 'number', default: 0.12, description: 'Music volume multiplier before ducking (0.0–1.0, default 0.12)' },
|
|
2490
|
+
duckLevel: { type: 'number', default: -18, description: 'Music attenuation in dB when voice is active (default -18dB)' },
|
|
2491
|
+
sfxCues: { type: 'array', description: 'Sound effect cues: [{ name, time, volume }]. Names: whoosh|impact|pop|click|riser|chime|buzz|tick|swish|punch|sparkle|deep_boom|transition', items: { type: 'object' } },
|
|
2492
|
+
addReverb: { type: 'boolean', default: false, description: 'Legacy: Add cinematic room acoustics. Prefer roomProfile for precise control.' },
|
|
2493
|
+
roomProfile: { type: 'string', description: 'Room acoustics profile. Named: studio_dry|broadcast|cinematic|concert_hall|intimate. Or earshot computed room: "computed:sizeM:damp:earlyMs" (e.g. "computed:8:0.5:15" = 8m room, 50% damp, 15ms early). Overrides addReverb.' },
|
|
2494
|
+
noiseGate: { type: 'boolean', default: false, description: 'Apply noise gate (-60dBFS threshold) to remove breath noise and room tone between words. Enable for field recordings or noisy VO.' },
|
|
2495
|
+
stereoWiden: { type: 'boolean', default: true, description: 'Apply mid-side stereo widening to music layer (voice stays mono-center).' },
|
|
2496
|
+
targetLufs: { type: 'number', default: -14, description: 'Target integrated loudness. Platform presets: -14 YouTube/Instagram, -16 Podcast, -23 Broadcast EBU R128, -27 Netflix.' },
|
|
2497
|
+
platform: { type: 'string', enum: ['youtube', 'podcast', 'broadcast', 'netflix', 'instagram'], description: 'Platform preset for LUFS target. Overrides targetLufs when provided.' },
|
|
2498
|
+
videoPath: { type: 'string', description: 'If provided, muxes the mixed audio into this video file and writes to outputPath.' },
|
|
2499
|
+
},
|
|
2500
|
+
},
|
|
2501
|
+
handler: async ({ voicePath, outputPath, musicPath, musicVolume = 0.12, duckLevel = -18, sfxCues = [], addReverb = false, roomProfile, noiseGate = false, stereoWiden = true, targetLufs = -14, platform, videoPath }) => {
|
|
2502
|
+
const { spawnSync } = require('child_process');
|
|
2503
|
+
const syncFs = require('fs');
|
|
2504
|
+
const pathMod = require('path');
|
|
2505
|
+
const os = require('os');
|
|
2506
|
+
|
|
2507
|
+
if (!syncFs.existsSync(voicePath)) throw new Error(`Voice file not found: ${voicePath}`);
|
|
2508
|
+
syncFs.mkdirSync(pathMod.dirname(outputPath), { recursive: true });
|
|
2509
|
+
|
|
2510
|
+
const tmpDir = pathMod.join(os.tmpdir(), `ff_mixpro_${Date.now()}`);
|
|
2511
|
+
syncFs.mkdirSync(tmpDir, { recursive: true });
|
|
2512
|
+
|
|
2513
|
+
/**
|
|
2514
|
+
* Helper: run ffmpeg, throw on failure
|
|
2515
|
+
*/
|
|
2516
|
+
function runFfmpeg(args, label) {
|
|
2517
|
+
const r = spawnSync('ffmpeg', ['-nostdin', '-y', ...args], { encoding: 'utf8', timeout: 300000 });
|
|
2518
|
+
if (r.status !== 0) {
|
|
2519
|
+
throw new Error(`${label} failed:\n${(r.stderr || r.stdout || '').slice(-500)}`);
|
|
2520
|
+
}
|
|
2521
|
+
return r;
|
|
2522
|
+
}
|
|
2523
|
+
|
|
2524
|
+
/**
|
|
2525
|
+
* Two-pass loudnorm: measure first, then apply with measured values for accuracy
|
|
2526
|
+
*/
|
|
2527
|
+
function twoPassNorm(inPath, outPath, lufs = -14, tp = -1.0) {
|
|
2528
|
+
// Pass 1: measure
|
|
2529
|
+
const m = spawnSync('ffmpeg', [
|
|
2530
|
+
'-nostdin', '-i', inPath,
|
|
2531
|
+
'-af', `loudnorm=I=${lufs}:TP=${tp}:print_format=json`,
|
|
2532
|
+
'-f', 'null', '-',
|
|
2533
|
+
], { encoding: 'utf8', timeout: 120000 });
|
|
2534
|
+
const combined = (m.stderr || '') + (m.stdout || '');
|
|
2535
|
+
const match = combined.match(/\{[\s\S]*?"input_i"[\s\S]*?\}/);
|
|
2536
|
+
let filterStr = `loudnorm=I=${lufs}:TP=${tp}`;
|
|
2537
|
+
if (match) {
|
|
2538
|
+
try {
|
|
2539
|
+
const ln = JSON.parse(match[0]);
|
|
2540
|
+
filterStr = `loudnorm=I=${lufs}:TP=${tp}:measured_I=${ln.input_i}:measured_LRA=${ln.input_lra}:measured_TP=${ln.input_tp}:measured_thresh=${ln.input_thresh}:offset=${ln.target_offset}:linear=true`;
|
|
2541
|
+
} catch (_) {}
|
|
2542
|
+
}
|
|
2543
|
+
runFfmpeg([
|
|
2544
|
+
'-i', inPath, '-af', filterStr,
|
|
2545
|
+
'-c:a', 'aac', '-b:a', '192k', '-ar', '48000', '-ac', '2',
|
|
2546
|
+
outPath,
|
|
2547
|
+
], `Two-pass loudnorm ${pathMod.basename(inPath)}`);
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
// ── Resolve platform LUFS target ───────────────────────────────────────
|
|
2551
|
+
const PLATFORM_LUFS_LOCAL = {
|
|
2552
|
+
youtube: { lufs: -14, tp: -1.0 }, podcast: { lufs: -16, tp: -1.0 },
|
|
2553
|
+
broadcast: { lufs: -23, tp: -1.0 }, netflix: { lufs: -27, tp: -3.0 },
|
|
2554
|
+
instagram: { lufs: -14, tp: -1.0 },
|
|
2555
|
+
};
|
|
2556
|
+
const platformTarget = platform ? (PLATFORM_LUFS_LOCAL[platform] || null) : null;
|
|
2557
|
+
const resolvedLufs = platformTarget ? platformTarget.lufs : targetLufs;
|
|
2558
|
+
const resolvedTp = platformTarget ? platformTarget.tp : -1.0;
|
|
2559
|
+
|
|
2560
|
+
// ── Room acoustics profile resolution ─────────────────────────────────
|
|
2561
|
+
const ROOM_PROFILES_LOCAL = {
|
|
2562
|
+
studio_dry: { inGain: 0.8, outGain: 0.6, delays: '10', decays: '0.1' },
|
|
2563
|
+
broadcast: { inGain: 0.8, outGain: 0.75, delays: '20', decays: '0.2' },
|
|
2564
|
+
cinematic: { inGain: 0.8, outGain: 0.85, delays: '30', decays: '0.3' },
|
|
2565
|
+
concert_hall: { inGain: 0.8, outGain: 0.90, delays: '45', decays: '0.55' },
|
|
2566
|
+
intimate: { inGain: 0.85, outGain: 0.70, delays: '15', decays: '0.18' },
|
|
2567
|
+
};
|
|
2568
|
+
const resolvedRoom = roomProfile ? (ROOM_PROFILES_LOCAL[roomProfile] || null) : (addReverb ? ROOM_PROFILES_LOCAL.cinematic : null);
|
|
2569
|
+
|
|
2570
|
+
// ── Step 1: Professional voice chain ──────────────────────────────────
|
|
2571
|
+
// [noise gate] → 80Hz HPF → presence +2dB@5kHz → air shelf +1.5dB@12kHz
|
|
2572
|
+
// → 3:1 compressor → de-esser 6.5kHz → [room acoustics]
|
|
2573
|
+
const voiceChainPath = pathMod.join(tmpDir, 'voice_chain.m4a');
|
|
2574
|
+
const deEsserFreq = 6500; // Hz — sibilant center
|
|
2575
|
+
const deEsserBW = 1500; // Hz — bandwidth
|
|
2576
|
+
|
|
2577
|
+
const voiceFilterParts = [];
|
|
2578
|
+
// Noise gate: removes breath noise and room tone between words
|
|
2579
|
+
if (noiseGate) {
|
|
2580
|
+
voiceFilterParts.push('agate=threshold=0.0003:range=0.15:attack=5:release=200');
|
|
2581
|
+
}
|
|
2582
|
+
voiceFilterParts.push(
|
|
2583
|
+
'highpass=f=80',
|
|
2584
|
+
'equalizer=f=5000:width_type=o:width=2:g=2.0', // presence boost
|
|
2585
|
+
'equalizer=f=12000:width_type=s:width=1:g=1.5', // air shelf
|
|
2586
|
+
'acompressor=ratio=3:threshold=-20dB:attack=5:release=50:makeup=2',
|
|
2587
|
+
`equalizer=f=${deEsserFreq}:width_type=h:width=${deEsserBW}:g=-3.0`, // de-esser
|
|
2588
|
+
);
|
|
2589
|
+
if (resolvedRoom) {
|
|
2590
|
+
voiceFilterParts.push(
|
|
2591
|
+
`aecho=in_gain=${resolvedRoom.inGain}:out_gain=${resolvedRoom.outGain}:delays=${resolvedRoom.delays}:decays=${resolvedRoom.decays}`
|
|
2592
|
+
);
|
|
2593
|
+
}
|
|
2594
|
+
const voiceFilterWithReverb = voiceFilterParts.join(',');
|
|
2595
|
+
|
|
2596
|
+
runFfmpeg([
|
|
2597
|
+
'-i', voicePath,
|
|
2598
|
+
'-af', voiceFilterWithReverb,
|
|
2599
|
+
'-c:a', 'aac', '-b:a', '192k', '-ar', '48000', '-ac', '2',
|
|
2600
|
+
voiceChainPath,
|
|
2601
|
+
], 'Voice chain (noise gate + EQ + de-esser + compress + room)');
|
|
2602
|
+
|
|
2603
|
+
// ── Step 2: SFX synthesis and timeline cue mixing ─────────────────────
|
|
2604
|
+
let sfxMixPath = null;
|
|
2605
|
+
if (Array.isArray(sfxCues) && sfxCues.length > 0) {
|
|
2606
|
+
// Get voice duration for SFX timeline
|
|
2607
|
+
const durProbe = spawnSync('ffprobe', [
|
|
2608
|
+
'-v', 'error', '-show_entries', 'format=duration',
|
|
2609
|
+
'-of', 'default=nk=1:nw=1', voiceChainPath,
|
|
2610
|
+
], { encoding: 'utf8', timeout: 10000 });
|
|
2611
|
+
const voiceDur = parseFloat((durProbe.stdout || '0').trim()) || 30;
|
|
2612
|
+
|
|
2613
|
+
// Generate SFX via tone synthesis — full 13-type earshot library
|
|
2614
|
+
// Build per-SFX wav files, then mix them with the voice chain
|
|
2615
|
+
const sfxPaths = [];
|
|
2616
|
+
// Extended earshot SFX map (matches SFX_FREQ_MAP in mixer.js)
|
|
2617
|
+
const SFX_FULL_MAP = {
|
|
2618
|
+
whoosh: { freq: 200, dur: 0.4, vol: 0.35 },
|
|
2619
|
+
impact: { freq: 60, dur: 0.5, vol: 0.42 },
|
|
2620
|
+
pop: { freq: 600, dur: 0.12, vol: 0.28 },
|
|
2621
|
+
click: { freq: 2500, dur: 0.05, vol: 0.22 },
|
|
2622
|
+
riser: { freq: 100, dur: 2.0, vol: 0.30 },
|
|
2623
|
+
chime: { freq: 1200, dur: 0.35, vol: 0.25 },
|
|
2624
|
+
buzz: { freq: 80, dur: 0.18, vol: 0.38 },
|
|
2625
|
+
tick: { freq: 3200, dur: 0.04, vol: 0.18 },
|
|
2626
|
+
swish: { freq: 350, dur: 0.3, vol: 0.30 },
|
|
2627
|
+
punch: { freq: 45, dur: 0.25, vol: 0.50 },
|
|
2628
|
+
sparkle: { freq: 4000, dur: 0.2, vol: 0.15 },
|
|
2629
|
+
deep_boom: { freq: 30, dur: 1.2, vol: 0.55 },
|
|
2630
|
+
transition: { freq: 440, dur: 0.6, vol: 0.20 },
|
|
2631
|
+
};
|
|
2632
|
+
|
|
2633
|
+
for (let i = 0; i < sfxCues.length; i++) {
|
|
2634
|
+
const cue = sfxCues[i];
|
|
2635
|
+
const sfxName = (cue.name || cue.sfxName || 'pop').toLowerCase().replace(/-/g, '_');
|
|
2636
|
+
const cueTime = Number(cue.time || cue.t || 0);
|
|
2637
|
+
const cueVol = Number(cue.volume || cue.vol || 1.0);
|
|
2638
|
+
const sfxDef = SFX_FULL_MAP[sfxName] || SFX_FULL_MAP.pop;
|
|
2639
|
+
const sfxDur = sfxDef.dur;
|
|
2640
|
+
|
|
2641
|
+
// Perceptual lead: place audio 40ms before the visual cue
|
|
2642
|
+
const adjustedTime = Math.max(0, cueTime - 0.04);
|
|
2643
|
+
|
|
2644
|
+
const sfxPath = pathMod.join(tmpDir, `sfx_${i}_${sfxName}.m4a`);
|
|
2645
|
+
const freq = sfxDef.freq;
|
|
2646
|
+
|
|
2647
|
+
// Create silence-padded SFX at correct offset in timeline
|
|
2648
|
+
const silenceFilter = `anullsrc=channel_layout=stereo:sample_rate=48000[silence];sine=f=${freq}:d=${sfxDur}[sfxtone];[sfxtone]volume=${cueVol * 0.35}[sfxv];[silence][sfxv]amix=inputs=2:duration=longest[sfxout]`;
|
|
2649
|
+
const adelayMs = Math.round(adjustedTime * 1000);
|
|
2650
|
+
runFfmpeg([
|
|
2651
|
+
'-f', 'lavfi', '-i', `anullsrc=channel_layout=stereo:sample_rate=48000:duration=${Math.max(sfxDur + adjustedTime + 0.5, 1)}`,
|
|
2652
|
+
'-f', 'lavfi', '-i', `sine=f=${freq}:d=${sfxDur}`,
|
|
2653
|
+
'-filter_complex', `[1:a]volume=${cueVol * 0.35},adelay=${adelayMs}|${adelayMs}[sfx];[0:a][sfx]amix=inputs=2:duration=first[out]`,
|
|
2654
|
+
'-map', '[out]',
|
|
2655
|
+
'-c:a', 'aac', '-b:a', '128k', '-ar', '48000', '-ac', '2',
|
|
2656
|
+
'-t', String(voiceDur),
|
|
2657
|
+
sfxPath,
|
|
2658
|
+
], `SFX synthesis: ${sfxName} at t=${cueTime}s`);
|
|
2659
|
+
sfxPaths.push(sfxPath);
|
|
2660
|
+
}
|
|
2661
|
+
|
|
2662
|
+
// Merge all SFX into one timeline track
|
|
2663
|
+
if (sfxPaths.length === 1) {
|
|
2664
|
+
sfxMixPath = sfxPaths[0];
|
|
2665
|
+
} else {
|
|
2666
|
+
sfxMixPath = pathMod.join(tmpDir, 'sfx_combined.m4a');
|
|
2667
|
+
const sfxInputs = sfxPaths.flatMap(p => ['-i', p]);
|
|
2668
|
+
const sfxFilter = sfxPaths.map((_, i) => `[${i}:a]`).join('') + `amix=inputs=${sfxPaths.length}:duration=longest:normalize=0[sfxout]`;
|
|
2669
|
+
runFfmpeg([
|
|
2670
|
+
...sfxInputs,
|
|
2671
|
+
'-filter_complex', sfxFilter,
|
|
2672
|
+
'-map', '[sfxout]',
|
|
2673
|
+
'-c:a', 'aac', '-b:a', '128k', '-ar', '48000', '-ac', '2',
|
|
2674
|
+
sfxMixPath,
|
|
2675
|
+
], 'SFX combine');
|
|
2676
|
+
}
|
|
2677
|
+
}
|
|
2678
|
+
|
|
2679
|
+
// ── Step 3: Music processing + sidechain ducking ───────────────────────
|
|
2680
|
+
let finalMixPath;
|
|
2681
|
+
if (musicPath && syncFs.existsSync(musicPath)) {
|
|
2682
|
+
const vidDur = spawnSync('ffprobe', [
|
|
2683
|
+
'-v', 'error', '-show_entries', 'format=duration',
|
|
2684
|
+
'-of', 'default=nk=1:nw=1', voiceChainPath,
|
|
2685
|
+
], { encoding: 'utf8', timeout: 10000 });
|
|
2686
|
+
const totalDur = parseFloat((vidDur.stdout || '0').trim()) || 60;
|
|
2687
|
+
|
|
2688
|
+
const musicFilter = stereoWiden
|
|
2689
|
+
? `aformat=sample_rates=48000:channel_layouts=stereo,highpass=f=80,lowpass=f=16000,volume=${musicVolume},extrastereo=m=1.4`
|
|
2690
|
+
: `aformat=sample_rates=48000:channel_layouts=stereo,highpass=f=80,lowpass=f=16000,volume=${musicVolume}`;
|
|
2691
|
+
|
|
2692
|
+
const duckedMixPath = pathMod.join(tmpDir, 'ducked_mix.m4a');
|
|
2693
|
+
|
|
2694
|
+
const inputs = ['-i', voiceChainPath, '-stream_loop', '-1', '-i', musicPath, '-t', String(totalDur)];
|
|
2695
|
+
const sfxInputPart = sfxMixPath ? ['-i', sfxMixPath] : [];
|
|
2696
|
+
const sfxInputIdx = sfxMixPath ? 2 : -1;
|
|
2697
|
+
|
|
2698
|
+
let filterComplex, mapExpr;
|
|
2699
|
+
if (sfxMixPath) {
|
|
2700
|
+
filterComplex = [
|
|
2701
|
+
`[1:a]${musicFilter}[music_proc]`,
|
|
2702
|
+
`[0:a][music_proc]sidechaincompress=threshold=0.003:ratio=20:attack=0.2:release=1:makeup=2[ducked]`,
|
|
2703
|
+
`[ducked][2:a]amix=inputs=2:duration=first:normalize=0[mix_sfx]`,
|
|
2704
|
+
].join(';');
|
|
2705
|
+
mapExpr = '[mix_sfx]';
|
|
2706
|
+
} else {
|
|
2707
|
+
filterComplex = [
|
|
2708
|
+
`[1:a]${musicFilter}[music_proc]`,
|
|
2709
|
+
`[0:a][music_proc]sidechaincompress=threshold=0.003:ratio=20:attack=0.2:release=1:makeup=2[ducked]`,
|
|
2710
|
+
].join(';');
|
|
2711
|
+
mapExpr = '[ducked]';
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
runFfmpeg([
|
|
2715
|
+
'-i', voiceChainPath,
|
|
2716
|
+
'-stream_loop', '-1', '-i', musicPath, '-t', String(totalDur),
|
|
2717
|
+
...sfxInputPart,
|
|
2718
|
+
'-filter_complex', filterComplex,
|
|
2719
|
+
'-map', mapExpr,
|
|
2720
|
+
'-c:a', 'aac', '-b:a', '192k', '-ar', '48000', '-ac', '2',
|
|
2721
|
+
duckedMixPath,
|
|
2722
|
+
], 'Sidechain ducking + SFX mix');
|
|
2723
|
+
finalMixPath = duckedMixPath;
|
|
2724
|
+
} else if (sfxMixPath) {
|
|
2725
|
+
// No music, but SFX — mix voice + SFX
|
|
2726
|
+
const sfxVoiceMix = pathMod.join(tmpDir, 'voice_sfx.m4a');
|
|
2727
|
+
runFfmpeg([
|
|
2728
|
+
'-i', voiceChainPath, '-i', sfxMixPath,
|
|
2729
|
+
'-filter_complex', '[0:a][1:a]amix=inputs=2:duration=first:normalize=0[out]',
|
|
2730
|
+
'-map', '[out]',
|
|
2731
|
+
'-c:a', 'aac', '-b:a', '192k', '-ar', '48000', '-ac', '2',
|
|
2732
|
+
sfxVoiceMix,
|
|
2733
|
+
], 'Voice + SFX mix');
|
|
2734
|
+
finalMixPath = sfxVoiceMix;
|
|
2735
|
+
} else {
|
|
2736
|
+
finalMixPath = voiceChainPath;
|
|
2737
|
+
}
|
|
2738
|
+
|
|
2739
|
+
// ── Step 4: Two-pass EBU R128 loudnorm master (platform-aware) ────────
|
|
2740
|
+
const normPath = pathMod.join(tmpDir, 'master_norm.m4a');
|
|
2741
|
+
twoPassNorm(finalMixPath, normPath, resolvedLufs, resolvedTp);
|
|
2742
|
+
|
|
2743
|
+
// ── Step 5: Mux into video or copy to outputPath ──────────────────────
|
|
2744
|
+
if (videoPath && syncFs.existsSync(videoPath)) {
|
|
2745
|
+
const vidDur2 = spawnSync('ffprobe', [
|
|
2746
|
+
'-v', 'error', '-show_entries', 'format=duration',
|
|
2747
|
+
'-of', 'default=nk=1:nw=1', videoPath,
|
|
2748
|
+
], { encoding: 'utf8', timeout: 10000 });
|
|
2749
|
+
const vd = parseFloat((vidDur2.stdout || '0').trim()) || 0;
|
|
2750
|
+
runFfmpeg([
|
|
2751
|
+
'-i', videoPath, '-i', normPath,
|
|
2752
|
+
'-map', '0:v', '-map', '1:a',
|
|
2753
|
+
'-c:v', 'copy',
|
|
2754
|
+
'-c:a', 'aac', '-b:a', '192k',
|
|
2755
|
+
...(vd > 0 ? ['-t', String(vd)] : []),
|
|
2756
|
+
'-movflags', '+faststart',
|
|
2757
|
+
outputPath,
|
|
2758
|
+
], 'Mux mixed audio into video');
|
|
2759
|
+
} else {
|
|
2760
|
+
// Just copy the normalized audio
|
|
2761
|
+
const copyRes = spawnSync('ffmpeg', ['-nostdin', '-y', '-i', normPath, '-c', 'copy', outputPath], { encoding: 'utf8', timeout: 30000 });
|
|
2762
|
+
if (copyRes.status !== 0) {
|
|
2763
|
+
// Fallback: re-encode
|
|
2764
|
+
runFfmpeg(['-i', normPath, '-c:a', 'aac', '-b:a', '192k', outputPath], 'Copy audio output');
|
|
2765
|
+
}
|
|
2766
|
+
}
|
|
2767
|
+
|
|
2768
|
+
// ── Cleanup tmp ───────────────────────────────────────────────────────
|
|
2769
|
+
try { syncFs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) {}
|
|
2770
|
+
|
|
2771
|
+
// ── Measure final loudness ─────────────────────────────────────────────
|
|
2772
|
+
let measuredLufs = null;
|
|
2773
|
+
let measuredTp = null;
|
|
2774
|
+
try {
|
|
2775
|
+
const m = spawnSync('ffmpeg', [
|
|
2776
|
+
'-nostdin', '-i', outputPath,
|
|
2777
|
+
'-af', `loudnorm=I=${resolvedLufs}:TP=${resolvedTp}:print_format=json`,
|
|
2778
|
+
'-f', 'null', '-',
|
|
2779
|
+
], { encoding: 'utf8', timeout: 60000 });
|
|
2780
|
+
const combined = (m.stderr || '') + (m.stdout || '');
|
|
2781
|
+
const match = combined.match(/\{[\s\S]*?"input_i"[\s\S]*?\}/);
|
|
2782
|
+
if (match) {
|
|
2783
|
+
const ln = JSON.parse(match[0]);
|
|
2784
|
+
measuredLufs = parseFloat(ln.input_i);
|
|
2785
|
+
measuredTp = parseFloat(ln.input_tp);
|
|
2786
|
+
}
|
|
2787
|
+
} catch (_) {}
|
|
2788
|
+
|
|
2789
|
+
const sizeBytes = syncFs.existsSync(outputPath) ? syncFs.statSync(outputPath).size : 0;
|
|
2790
|
+
|
|
2791
|
+
return {
|
|
2792
|
+
outputPath,
|
|
2793
|
+
sizeBytes,
|
|
2794
|
+
measuredLufs,
|
|
2795
|
+
measuredTp,
|
|
2796
|
+
targetLufs: resolvedLufs,
|
|
2797
|
+
platform: platform || null,
|
|
2798
|
+
passed: measuredLufs != null ? Math.abs(measuredLufs - resolvedLufs) <= 1.5 && measuredTp <= -0.5 : null,
|
|
2799
|
+
chain: {
|
|
2800
|
+
noiseGate: noiseGate ? 'agate -60dBFS' : 'off',
|
|
2801
|
+
hpf: '80Hz',
|
|
2802
|
+
presenceBoost: '+2dB @ 5kHz',
|
|
2803
|
+
airShelf: '+1.5dB @ 12kHz',
|
|
2804
|
+
deEsser: `${deEsserFreq}Hz dynamic`,
|
|
2805
|
+
compressor: '3:1 @ -20dBFS',
|
|
2806
|
+
roomAcoustics: resolvedRoom ? (roomProfile || 'cinematic') : 'off',
|
|
2807
|
+
sidechainDuck: musicPath ? `${duckLevel}dB` : 'no music',
|
|
2808
|
+
stereoWiden: stereoWiden && musicPath ? 'MS extrastereo 1.4x' : 'off',
|
|
2809
|
+
sfxCues: sfxCues.length,
|
|
2810
|
+
masterBus: `EBU R128 ${resolvedLufs} LUFS${platform ? ` (${platform})` : ''}`,
|
|
2811
|
+
},
|
|
2812
|
+
message: `Professional audio mix complete → ${outputPath} (${(sizeBytes / 1024).toFixed(0)}KB${measuredLufs != null ? `, ${measuredLufs.toFixed(1)} LUFS` : ''}${platform ? `, ${platform} target` : ''})`,
|
|
1857
2813
|
};
|
|
1858
2814
|
},
|
|
1859
2815
|
},
|
|
@@ -1864,7 +2820,7 @@ Analyzes input clips, segments narrative beats, calculates cut in/out timestamps
|
|
|
1864
2820
|
|
|
1865
2821
|
const SERVER_INFO = {
|
|
1866
2822
|
name: 'video-pipeline',
|
|
1867
|
-
version: '2.
|
|
2823
|
+
version: '2.7.0',
|
|
1868
2824
|
};
|
|
1869
2825
|
|
|
1870
2826
|
const CAPABILITIES = {
|