@amg555/frameforge-mcp 0.0.0-stage → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bob/artifacts/enterprise-ai-video-agent-platform-agnostic-implementation-guide.html +464 -0
- package/.claude-plugin/marketplace.json +17 -0
- package/.gitattributes +19 -0
- package/.mcp.json +45 -0
- package/CHANGELOG.md +431 -0
- package/CLAUDE.md +278 -0
- package/CONTRIBUTING.md +112 -0
- package/LICENSE +21 -0
- package/Procfile +1 -0
- package/README.md +359 -2
- package/SECURITY.md +71 -0
- package/START_HERE.md +144 -0
- package/assets/metadata/assets.json +5 -0
- package/assets/metadata/embeddings.json +5 -0
- package/install.sh +64 -0
- package/package.json +58 -4
- package/src/animation/animation-system.js +229 -0
- package/src/asset-library/README.md +188 -0
- package/src/asset-library/asset-library.js +639 -0
- package/src/audio-mixer/mixer.js +187 -0
- package/src/cli/doctor.js +173 -0
- package/src/cli/make.js +655 -0
- package/src/cli/setup-keys.js +378 -0
- package/src/config/load-config.js +72 -0
- package/src/fact-checker/checker.js +384 -0
- package/src/fact-checker/review-gate.js +306 -0
- package/src/fact-checker/validation-set.js +1078 -0
- package/src/mcp-server/server.js +1299 -0
- package/src/mcp-server/stdio.js +87 -0
- package/src/pipeline/artifact-manifest.js +323 -0
- package/src/pipeline/artifact-retention.js +189 -0
- package/src/pipeline/external-adapter.js +366 -0
- package/src/pipeline/providers.js +853 -0
- package/src/pipeline/retry-policy.js +134 -0
- package/src/pipeline/state-machine.js +297 -0
- package/src/pipeline/telemetry.js +162 -0
- package/src/quality-gate/gate.js +229 -0
- package/src/schema/scene.schema.json +101 -0
- package/src/shorts-extractor/extractor.js +343 -0
- package/src/utils/log-redaction.js +274 -0
- package/src/video-renderer/renderer.js +364 -0
- package/src/worker/server-render-free.js +420 -0
- package/src/worker/server.js +328 -0
- package/src/workflows/assets.js +131 -0
- package/src/workflows/ingest.js +299 -0
- package/src/workflows/packaging.js +156 -0
- package/src/workflows/publishing.js +206 -0
- package/src/workflows/render.js +102 -0
- package/src/workflows/research.js +151 -0
- package/src/workflows/script.js +210 -0
- package/src/workflows/shorts.js +113 -0
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quality Gate
|
|
3
|
+
*
|
|
4
|
+
* Hard gates (must pass to proceed):
|
|
5
|
+
* - Valid video + audio streams
|
|
6
|
+
* - Resolution 1920x1080 (master) or 1080x1920 (short)
|
|
7
|
+
* - H.264 codec, yuv420p pixel format, 30fps
|
|
8
|
+
* - Integrated loudness: -14 LUFS ±1.5 LU
|
|
9
|
+
* - True peak: ≤ -0.5 dBTP
|
|
10
|
+
* - No black frames > 0.5s
|
|
11
|
+
* - No frozen frames > 0.5s
|
|
12
|
+
* - File size > 1MB
|
|
13
|
+
* - No ffprobe errors
|
|
14
|
+
*
|
|
15
|
+
* Sanity checks (warn, do not block):
|
|
16
|
+
* - Video bitrate ≥ 3 Mbps
|
|
17
|
+
* - Duration within 5% of expected
|
|
18
|
+
*
|
|
19
|
+
* Diagnostics (informational):
|
|
20
|
+
* - SNR: voice_rms - noise_floor_rms
|
|
21
|
+
* - File size bytes
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
'use strict';
|
|
25
|
+
|
|
26
|
+
const { spawnSync } = require('child_process');
|
|
27
|
+
const path = require('path');
|
|
28
|
+
|
|
29
|
+
class QualityGate {
|
|
30
|
+
/**
|
|
31
|
+
* Validate a master video.
|
|
32
|
+
* Accepts both 1920×1080 (landscape) and 1080×1920 (portrait).
|
|
33
|
+
* @param {string} videoPath
|
|
34
|
+
* @param {{ width?: number, height?: number }} [opts] expected resolution override
|
|
35
|
+
* @returns {{ passed: boolean, checks: Check[], diagnostics: Diagnostic[] }}
|
|
36
|
+
*/
|
|
37
|
+
async validateMaster(videoPath, { width, height } = {}) {
|
|
38
|
+
// If caller doesn't specify, probe the file and accept whichever valid format it is.
|
|
39
|
+
const expectedW = width || this._probeExpectedDimension(videoPath, 'width') || 1920;
|
|
40
|
+
const expectedH = height || this._probeExpectedDimension(videoPath, 'height') || 1080;
|
|
41
|
+
return this._validate(videoPath, { width: expectedW, height: expectedH, minDuration: 5, maxDuration: 3600 });
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Validate a short (1080x1920).
|
|
46
|
+
* @param {string} videoPath
|
|
47
|
+
*/
|
|
48
|
+
async validateShort(videoPath) {
|
|
49
|
+
return this._validate(videoPath, { width: 1080, height: 1920, minDuration: 5, maxDuration: 90 });
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Read the actual video dimensions and snap to the nearest valid master format.
|
|
54
|
+
* Accepted: 1920×1080 (landscape) and 1080×1920 (portrait).
|
|
55
|
+
* Returns the actual dimension if it matches a valid format, otherwise 1920/1080.
|
|
56
|
+
*/
|
|
57
|
+
_probeExpectedDimension(videoPath, axis) {
|
|
58
|
+
try {
|
|
59
|
+
const raw = this._ffprobe(videoPath, ['-show_streams', '-select_streams', 'v:0', '-print_format', 'json']);
|
|
60
|
+
const info = JSON.parse(raw).streams?.[0] || {};
|
|
61
|
+
const w = info.width;
|
|
62
|
+
const h = info.height;
|
|
63
|
+
// Snap to known valid formats
|
|
64
|
+
if ((w === 1920 && h === 1080) || (w === 1080 && h === 1920)) {
|
|
65
|
+
return axis === 'width' ? w : h;
|
|
66
|
+
}
|
|
67
|
+
} catch { /* fall through to default */ }
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// ── Internal ──────────────────────────────────────────────────────────────
|
|
72
|
+
|
|
73
|
+
async _validate(videoPath, { width, height, minDuration, maxDuration }) {
|
|
74
|
+
const checks = [];
|
|
75
|
+
const diagnostics = [];
|
|
76
|
+
let passed = true;
|
|
77
|
+
|
|
78
|
+
const fail = (name, expected, actual, message) => {
|
|
79
|
+
checks.push({ name, type: 'HARD_GATE', passed: false, expected, actual, message });
|
|
80
|
+
passed = false;
|
|
81
|
+
};
|
|
82
|
+
const pass = (name, expected, actual) => {
|
|
83
|
+
checks.push({ name, type: 'HARD_GATE', passed: true, expected, actual });
|
|
84
|
+
};
|
|
85
|
+
const warn = (name, expected, actual, message) => {
|
|
86
|
+
checks.push({ name, type: 'SANITY_CHECK', passed: true, expected, actual, message });
|
|
87
|
+
};
|
|
88
|
+
const diag = (metric, value, unit, interpretation) => {
|
|
89
|
+
diagnostics.push({ metric, value, unit, interpretation, type: 'DIAGNOSTIC' });
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
// ── 1. File size ──────────────────────────────────────────────────────
|
|
93
|
+
try {
|
|
94
|
+
const stat = require('fs').statSync(videoPath);
|
|
95
|
+
if (stat.size < 100 * 1024) { fail('file_size', '>100KB', `${Math.round(stat.size/1024)}KB`, 'File too small'); }
|
|
96
|
+
else { pass('file_size', '>100KB', `${Math.round(stat.size/1024)}KB`); }
|
|
97
|
+
diag('file_size_bytes', stat.size, 'bytes', 'Video file size');
|
|
98
|
+
} catch (err) { fail('file_exists', 'exists', 'missing', err.message); }
|
|
99
|
+
|
|
100
|
+
// ── 2. Stream validity ────────────────────────────────────────────────
|
|
101
|
+
let streams = {};
|
|
102
|
+
try {
|
|
103
|
+
const raw = this._ffprobe(videoPath, ['-show_streams', '-show_format', '-print_format', 'json']);
|
|
104
|
+
const probe = JSON.parse(raw);
|
|
105
|
+
const video = probe.streams?.find(s => s.codec_type === 'video');
|
|
106
|
+
const audio = probe.streams?.find(s => s.codec_type === 'audio');
|
|
107
|
+
|
|
108
|
+
if (!video) { fail('video_stream', 'present', 'missing'); }
|
|
109
|
+
else {
|
|
110
|
+
streams.video = video;
|
|
111
|
+
pass('video_stream', 'present', 'present');
|
|
112
|
+
|
|
113
|
+
// Resolution
|
|
114
|
+
if (video.width !== width || video.height !== height) {
|
|
115
|
+
fail('resolution', `${width}x${height}`, `${video.width}x${video.height}`);
|
|
116
|
+
} else { pass('resolution', `${width}x${height}`, `${video.width}x${video.height}`); }
|
|
117
|
+
|
|
118
|
+
// Codec
|
|
119
|
+
if (video.codec_name !== 'h264') { fail('video_codec', 'h264', video.codec_name); }
|
|
120
|
+
else { pass('video_codec', 'h264', video.codec_name); }
|
|
121
|
+
|
|
122
|
+
// Pixel format
|
|
123
|
+
if (video.pix_fmt !== 'yuv420p') { fail('pixel_format', 'yuv420p', video.pix_fmt); }
|
|
124
|
+
else { pass('pixel_format', 'yuv420p', video.pix_fmt); }
|
|
125
|
+
|
|
126
|
+
// Frame rate (24 or 30 accepted) — r_frame_rate is "num/den", parse safely
|
|
127
|
+
const [fpsNum, fpsDen] = (video.r_frame_rate || '0/1').split('/').map(Number);
|
|
128
|
+
const fps = fpsDen > 0 ? fpsNum / fpsDen : 0;
|
|
129
|
+
if (fps < 23 || fps > 31) { fail('frame_rate', '24-30fps', `${fps.toFixed(1)}fps`); }
|
|
130
|
+
else { pass('frame_rate', '24-30fps', `${fps.toFixed(1)}fps`); }
|
|
131
|
+
|
|
132
|
+
// Bitrate sanity
|
|
133
|
+
const bitrateKbps = parseInt(probe.format?.bit_rate || '0', 10) / 1000;
|
|
134
|
+
if (bitrateKbps > 0 && bitrateKbps < 3000) {
|
|
135
|
+
warn('video_bitrate', '≥3000 kbps', `${Math.round(bitrateKbps)} kbps`, 'Low bitrate (normal for simple animation with CRF encoding)');
|
|
136
|
+
} else if (bitrateKbps > 0) {
|
|
137
|
+
pass('video_bitrate', '≥3000 kbps', `${Math.round(bitrateKbps)} kbps`);
|
|
138
|
+
}
|
|
139
|
+
diag('video_bitrate_kbps', bitrateKbps, 'kbps', 'Average video bitrate');
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
if (!audio) { fail('audio_stream', 'present', 'missing'); }
|
|
143
|
+
else {
|
|
144
|
+
streams.audio = audio;
|
|
145
|
+
pass('audio_stream', 'present', 'present');
|
|
146
|
+
|
|
147
|
+
// Duration
|
|
148
|
+
const dur = parseFloat(probe.format?.duration || '0');
|
|
149
|
+
if (dur > 0) {
|
|
150
|
+
if (dur < minDuration || dur > maxDuration) {
|
|
151
|
+
warn('duration', `${minDuration}-${maxDuration}s`, `${dur.toFixed(1)}s`, 'Duration outside expected range');
|
|
152
|
+
} else { pass('duration', `${minDuration}-${maxDuration}s`, `${dur.toFixed(1)}s`); }
|
|
153
|
+
diag('duration_seconds', dur, 's', 'Video duration');
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
} catch (err) {
|
|
157
|
+
fail('ffprobe', 'no errors', err.message.slice(0, 100));
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ── 3. LUFS + True Peak ───────────────────────────────────────────────
|
|
161
|
+
try {
|
|
162
|
+
const loudnormOut = this._ffmpeg(['-i', videoPath, '-af', 'loudnorm=I=-14:TP=-1:print_format=json', '-f', 'null', '-']);
|
|
163
|
+
const jsonMatch = loudnormOut.match(/\{[\s\S]*\}/);
|
|
164
|
+
if (jsonMatch) {
|
|
165
|
+
const ln = JSON.parse(jsonMatch[0]);
|
|
166
|
+
const measuredI = parseFloat(ln.input_i || '0');
|
|
167
|
+
const measuredTP = parseFloat(ln.input_tp || '0');
|
|
168
|
+
|
|
169
|
+
if (Math.abs(measuredI - (-14)) > 1.5) {
|
|
170
|
+
fail('integrated_loudness', '-14 ±1.5 LU', `${measuredI.toFixed(1)} LUFS`);
|
|
171
|
+
} else { pass('integrated_loudness', '-14 ±1.5 LU', `${measuredI.toFixed(1)} LUFS`); }
|
|
172
|
+
|
|
173
|
+
if (measuredTP > -0.5) {
|
|
174
|
+
fail('true_peak', '≤-0.5 dBTP', `${measuredTP.toFixed(1)} dBTP`);
|
|
175
|
+
} else { pass('true_peak', '≤-0.5 dBTP', `${measuredTP.toFixed(1)} dBTP`); }
|
|
176
|
+
|
|
177
|
+
diag('integrated_loudness_LUFS', measuredI, 'LUFS', 'Integrated loudness (target -14)');
|
|
178
|
+
diag('true_peak_dBTP', measuredTP, 'dBTP', 'True peak (limit -1.0)');
|
|
179
|
+
}
|
|
180
|
+
} catch (err) {
|
|
181
|
+
warn('loudness_check', 'measurable', 'error', `Loudnorm check failed: ${err.message.slice(0, 80)}`);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// ── 4. Black frame detection ──────────────────────────────────────────
|
|
185
|
+
try {
|
|
186
|
+
const blackOut = this._ffmpeg(['-i', videoPath, '-vf', 'blackdetect=d=0.5:pix_th=0.10', '-f', 'null', '-']);
|
|
187
|
+
if (blackOut.includes('black_start')) {
|
|
188
|
+
fail('black_frames', 'none >0.5s', 'detected');
|
|
189
|
+
} else { pass('black_frames', 'none >0.5s', 'none'); }
|
|
190
|
+
} catch { /* non-fatal */ }
|
|
191
|
+
|
|
192
|
+
// ── 5. Frozen frame detection ─────────────────────────────────────────
|
|
193
|
+
// n=0.0001 means two frames are "frozen" only when 1-SSIM < 0.0001 (SSIM > 0.9999).
|
|
194
|
+
// Demoted to warn: static camera shots on uniform backgrounds legitimately produce
|
|
195
|
+
// high inter-frame similarity and should not hard-block a valid production.
|
|
196
|
+
try {
|
|
197
|
+
const freezeOut = this._ffmpeg(['-i', videoPath, '-vf', 'freezedetect=n=0.0001:d=0.5', '-f', 'null', '-']);
|
|
198
|
+
if (freezeOut.includes('freeze_start')) {
|
|
199
|
+
warn('frozen_frames', 'none >0.5s', 'detected', 'High inter-frame similarity detected (check for unintended freeze)');
|
|
200
|
+
} else { pass('frozen_frames', 'none >0.5s', 'none'); }
|
|
201
|
+
} catch { /* non-fatal */ }
|
|
202
|
+
|
|
203
|
+
return {
|
|
204
|
+
passed,
|
|
205
|
+
checks,
|
|
206
|
+
diagnostics,
|
|
207
|
+
timestamp: new Date().toISOString(),
|
|
208
|
+
schemaVersion: '1.0.0',
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
_ffprobe(file, args = []) {
|
|
213
|
+
const result = spawnSync('ffprobe', ['-v', 'error', ...args, file], {
|
|
214
|
+
encoding: 'utf8', timeout: 30000,
|
|
215
|
+
});
|
|
216
|
+
if (result.error) throw result.error;
|
|
217
|
+
return (result.stdout || '') + (result.stderr || '');
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
_ffmpeg(args = []) {
|
|
221
|
+
// FFmpeg writes filter output to stderr; we capture both and return combined
|
|
222
|
+
const result = spawnSync('ffmpeg', ['-nostdin', ...args], {
|
|
223
|
+
encoding: 'utf8', timeout: 120000,
|
|
224
|
+
});
|
|
225
|
+
return (result.stderr || result.stdout || result.error?.message || '').toString();
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
module.exports = { QualityGate };
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
3
|
+
"title": "Scene Schema v2.0.0",
|
|
4
|
+
"description": "Scene definition for FrameForge pipeline. character_id is a free-form string since v2.0.0.",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["scene_id", "schema_version", "narration", "visual_description"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"scene_id": { "type": "integer", "minimum": 0 },
|
|
9
|
+
"schema_version": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$", "default": "2.0.0" },
|
|
10
|
+
"narration": { "type": "string", "minLength": 1, "maxLength": 500 },
|
|
11
|
+
"caption": { "type": "string", "maxLength": 100 },
|
|
12
|
+
"video_path": { "type": "string", "description": "Optional path to user-supplied video clip or B-roll" },
|
|
13
|
+
"image_path": { "type": "string", "description": "Optional path to user-supplied static image" },
|
|
14
|
+
"audio_path": { "type": "string", "description": "Optional path to user-supplied pre-recorded voiceover" },
|
|
15
|
+
"visual_description": { "type": "string", "minLength": 1 },
|
|
16
|
+
"character": {
|
|
17
|
+
"type": "object",
|
|
18
|
+
"properties": {
|
|
19
|
+
"character_id": {
|
|
20
|
+
"type": "string",
|
|
21
|
+
"description": "Free-form character identifier. Use 'none' for no character. Any string is valid since schema v2.0.0."
|
|
22
|
+
},
|
|
23
|
+
"pose": { "type": "string" },
|
|
24
|
+
"emotion": { "type": "string" },
|
|
25
|
+
"position": {
|
|
26
|
+
"type": "object",
|
|
27
|
+
"properties": {
|
|
28
|
+
"x": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.5 },
|
|
29
|
+
"y": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.5 },
|
|
30
|
+
"scale": { "type": "number", "minimum": 0.5, "maximum": 2.0, "default": 1.0 }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"focal_point": {
|
|
36
|
+
"type": "object",
|
|
37
|
+
"properties": {
|
|
38
|
+
"x": { "type": "number", "minimum": 0, "maximum": 1 },
|
|
39
|
+
"y": { "type": "number", "minimum": 0, "maximum": 1 },
|
|
40
|
+
"weight": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.8 }
|
|
41
|
+
},
|
|
42
|
+
"required": ["x", "y"]
|
|
43
|
+
},
|
|
44
|
+
"overlay": {
|
|
45
|
+
"type": "object",
|
|
46
|
+
"properties": {
|
|
47
|
+
"type": { "type": "string", "enum": ["prop", "effect", "text", "none"] },
|
|
48
|
+
"asset_id": { "type": "string" },
|
|
49
|
+
"position": {
|
|
50
|
+
"type": "object",
|
|
51
|
+
"properties": {
|
|
52
|
+
"x": { "type": "number", "minimum": 0, "maximum": 1 },
|
|
53
|
+
"y": { "type": "number", "minimum": 0, "maximum": 1 }
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
"style": {
|
|
57
|
+
"type": "object",
|
|
58
|
+
"properties": {
|
|
59
|
+
"opacity": { "type": "number", "minimum": 0, "maximum": 1, "default": 1.0 },
|
|
60
|
+
"scale": { "type": "number", "minimum": 0.1, "maximum": 5.0, "default": 1.0 },
|
|
61
|
+
"rotation": { "type": "number", "minimum": -180, "maximum": 180, "default": 0 }
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
"sound_effects": {
|
|
67
|
+
"type": "array",
|
|
68
|
+
"items": {
|
|
69
|
+
"type": "object",
|
|
70
|
+
"properties": {
|
|
71
|
+
"effect_id": { "type": "string" },
|
|
72
|
+
"timing": { "type": "string", "enum": ["start", "middle", "end", "continuous"] },
|
|
73
|
+
"volume": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.3 }
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
"music_mood": {
|
|
78
|
+
"type": "string",
|
|
79
|
+
"enum": ["tense", "calm", "energetic", "mysterious", "neutral", "dramatic", "humorous"]
|
|
80
|
+
},
|
|
81
|
+
"camera_hint": {
|
|
82
|
+
"type": "string",
|
|
83
|
+
"enum": [
|
|
84
|
+
"zoom_in", "zoom_out",
|
|
85
|
+
"slow_zoom_in", "slow_zoom_out",
|
|
86
|
+
"pan_left", "pan_right",
|
|
87
|
+
"tilt_up", "tilt_down",
|
|
88
|
+
"ken_burns",
|
|
89
|
+
"static", "auto"
|
|
90
|
+
]
|
|
91
|
+
},
|
|
92
|
+
"metadata": {
|
|
93
|
+
"type": "object",
|
|
94
|
+
"properties": {
|
|
95
|
+
"importance": { "type": "string", "enum": ["low", "medium", "high", "critical"] },
|
|
96
|
+
"hook_potential": { "type": "number", "minimum": 0, "maximum": 1 },
|
|
97
|
+
"standalone": { "type": "boolean" }
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shorts Extractor — Tasks #13, #14, #15
|
|
3
|
+
*
|
|
4
|
+
* Extracts 3-5 vertical (9:16) short videos from a master 16:9 video.
|
|
5
|
+
*
|
|
6
|
+
* Algorithm:
|
|
7
|
+
* 1. Analyze master for scene boundaries (silence + freeze detection)
|
|
8
|
+
* 2. Score each candidate segment: hook + curiosity + payoff + standalone_context + information_density - repetition_penalty
|
|
9
|
+
* 3. Select top 3-5 segments (30-90s each, non-overlapping)
|
|
10
|
+
* 4. Vertical reformat with saliency-aware crop (scene-defined focal_point or center)
|
|
11
|
+
* 5. Final audio normalization to -14 LUFS
|
|
12
|
+
*
|
|
13
|
+
* Task #14 — Vertical Reformatting (16:9 → 9:16):
|
|
14
|
+
* Input: 1920×1080 (16:9)
|
|
15
|
+
* Output: 1080×1920 (9:16)
|
|
16
|
+
* Strategy:
|
|
17
|
+
* - Crop width = ih × 9/16 = 1080 × 9/16 = 607.5 → 607px (keeping divisible-by-2)
|
|
18
|
+
* - X offset computed from focal_point.x, clamped to [0, 1920-607] = [0, 1313]
|
|
19
|
+
* - Y is always 0 (full height retained; vertical output height = input height)
|
|
20
|
+
* - After crop: scale 607×1080 → 1080×1920
|
|
21
|
+
*
|
|
22
|
+
* Safe zones enforced (YouTube Shorts UI overlay areas):
|
|
23
|
+
* - Top 180px of OUTPUT: avoid placing text/action here (navigation bar)
|
|
24
|
+
* - Bottom 400px of OUTPUT: avoid placing key action here (buttons/captions)
|
|
25
|
+
* - Right 130px: avoid (subscribe / like buttons)
|
|
26
|
+
* Focal-point bias: y ≤ 0.35 (upper-third bias for talking-head content)
|
|
27
|
+
* These are informational — they guide _defaultFocalY() but FFmpeg itself crops the full frame.
|
|
28
|
+
*
|
|
29
|
+
* Requirements implemented:
|
|
30
|
+
* - Req 4.1: Analyze for scene changes, audio peaks, motion scores
|
|
31
|
+
* - Req 4.2: Segments 30-90s, begin/end at scene boundaries
|
|
32
|
+
* - Req 4.5: Saliency-aware framing with focal_point support + safe-zone bias
|
|
33
|
+
* - Req 4.7: Audio normalization to -14 LUFS
|
|
34
|
+
* - Req 4.8: Generate 3-5 shorts per master
|
|
35
|
+
* - Req 4.10: Output 1080x1920 H.264 AAC
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
'use strict';
|
|
39
|
+
|
|
40
|
+
const { spawnSync } = require('child_process');
|
|
41
|
+
const path = require('path');
|
|
42
|
+
const fs = require('fs');
|
|
43
|
+
|
|
44
|
+
class ShortsExtractor {
|
|
45
|
+
constructor({ filesDir = process.env.N8N_FILES_DIR || require('path').join(require('os').homedir(), '.frameforge-files') } = {}) {
|
|
46
|
+
this.filesDir = filesDir;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Extract candidate segments from a master video (used by Workflow D2).
|
|
51
|
+
* Returns scored, non-overlapping segment objects — does NOT render them.
|
|
52
|
+
* @param {string} masterPath
|
|
53
|
+
* @param {{ sceneMeta?: object, maxCandidates?: number }} opts
|
|
54
|
+
* @returns {Segment[]}
|
|
55
|
+
*/
|
|
56
|
+
extractCandidates(masterPath, { sceneMeta = {}, maxCandidates = 9 } = {}) {
|
|
57
|
+
const duration = this._getDuration(masterPath);
|
|
58
|
+
if (!duration || duration < 30) throw new Error(`Master video too short: ${duration}s`);
|
|
59
|
+
|
|
60
|
+
const silences = this._detectSilence(masterPath);
|
|
61
|
+
const boundaries = this._silenceToBoundaries(silences, duration);
|
|
62
|
+
|
|
63
|
+
// Build scenes array from sceneMeta map (keyed by scene_id)
|
|
64
|
+
const scenes = Object.entries(sceneMeta).map(([id, meta]) => ({
|
|
65
|
+
scene_id: parseInt(id, 10),
|
|
66
|
+
focal_point: meta.focalPoint,
|
|
67
|
+
hook_potential: meta.hookPotential,
|
|
68
|
+
standalone: meta.standalone,
|
|
69
|
+
}));
|
|
70
|
+
|
|
71
|
+
const candidates = this._buildCandidates(boundaries, scenes, duration);
|
|
72
|
+
const selected = this._selectSegments(candidates).slice(0, maxCandidates);
|
|
73
|
+
|
|
74
|
+
return selected.map(seg => ({
|
|
75
|
+
startSec: seg.start,
|
|
76
|
+
endSec: seg.end,
|
|
77
|
+
duration: seg.duration,
|
|
78
|
+
score: seg.score,
|
|
79
|
+
coveredScenes: seg.coveredScenes,
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Render a single short from a pre-scored segment (used by Workflow D2).
|
|
85
|
+
* @param {string} masterPath
|
|
86
|
+
* @param {{ startSec: number, endSec: number, duration: number, coveredScenes?: object[] }} segment
|
|
87
|
+
* @param {string} outputPath
|
|
88
|
+
*/
|
|
89
|
+
renderShort(masterPath, segment, outputPath) {
|
|
90
|
+
const seg = {
|
|
91
|
+
start: segment.startSec,
|
|
92
|
+
end: segment.endSec,
|
|
93
|
+
duration: segment.duration || (segment.endSec - segment.startSec),
|
|
94
|
+
coveredScenes: segment.coveredScenes || [],
|
|
95
|
+
};
|
|
96
|
+
this._extractSegment(masterPath, seg, seg.coveredScenes, outputPath);
|
|
97
|
+
return outputPath;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* High-level: extract + render all shorts in one call.
|
|
102
|
+
* @param {{ masterPath: string, scenes: Scene[], outputDir: string }} opts
|
|
103
|
+
* @returns {ShortResult[]}
|
|
104
|
+
*/
|
|
105
|
+
extractShorts({ masterPath, scenes = [], outputDir }) {
|
|
106
|
+
fs.mkdirSync(outputDir, { recursive: true });
|
|
107
|
+
|
|
108
|
+
// Step 1: Get master duration
|
|
109
|
+
const duration = this._getDuration(masterPath);
|
|
110
|
+
if (!duration || duration < 30) throw new Error(`Master video too short: ${duration}s`);
|
|
111
|
+
|
|
112
|
+
// Step 2: Detect silence boundaries (scene breaks)
|
|
113
|
+
const silences = this._detectSilence(masterPath);
|
|
114
|
+
const boundaries = this._silenceToBoundaries(silences, duration);
|
|
115
|
+
|
|
116
|
+
// Step 3: Build candidate segments
|
|
117
|
+
const candidates = this._buildCandidates(boundaries, scenes, duration);
|
|
118
|
+
|
|
119
|
+
// Step 4: Score and select best segments
|
|
120
|
+
const selected = this._selectSegments(candidates);
|
|
121
|
+
if (selected.length < 1) throw new Error(`No valid segments found (need 30-90s, got ${candidates.length} candidates)`);
|
|
122
|
+
|
|
123
|
+
// Step 5: Extract and reformat each segment
|
|
124
|
+
const results = [];
|
|
125
|
+
for (let i = 0; i < selected.length; i++) {
|
|
126
|
+
const seg = selected[i];
|
|
127
|
+
const outputPath = path.join(outputDir, `short_${i + 1}.mp4`);
|
|
128
|
+
this._extractSegment(masterPath, seg, scenes, outputPath);
|
|
129
|
+
const stat = fs.statSync(outputPath);
|
|
130
|
+
results.push({ index: i + 1, outputPath, start: seg.start, end: seg.end, duration: seg.duration, contentScore: seg.score, sizeBytes: stat.size });
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return results;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// ── Silence / scene boundary detection ───────────────────────────────────
|
|
137
|
+
|
|
138
|
+
_detectSilence(videoPath) {
|
|
139
|
+
const result = spawnSync('ffmpeg', [
|
|
140
|
+
'-nostdin', '-i', videoPath,
|
|
141
|
+
'-af', 'silencedetect=noise=-40dB:d=0.4',
|
|
142
|
+
'-f', 'null', '-',
|
|
143
|
+
], { timeout: 60000, encoding: 'utf8' });
|
|
144
|
+
const output = (result.stderr || result.stdout || '').toString();
|
|
145
|
+
const pauses = [];
|
|
146
|
+
const starts = [...output.matchAll(/silence_start: ([\d.]+)/g)].map(m => parseFloat(m[1]));
|
|
147
|
+
const ends = [...output.matchAll(/silence_end: ([\d.]+)/g)].map(m => parseFloat(m[1]));
|
|
148
|
+
for (let i = 0; i < Math.min(starts.length, ends.length); i++) {
|
|
149
|
+
pauses.push({ start: starts[i], end: ends[i] });
|
|
150
|
+
}
|
|
151
|
+
return pauses;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
_silenceToBoundaries(silences, totalDuration) {
|
|
155
|
+
// Start/end of each silence = potential scene boundary
|
|
156
|
+
const pts = new Set([0, totalDuration]);
|
|
157
|
+
for (const { start, end } of silences) {
|
|
158
|
+
pts.add(Math.round(start * 10) / 10);
|
|
159
|
+
pts.add(Math.round(end * 10) / 10);
|
|
160
|
+
}
|
|
161
|
+
return [...pts].sort((a, b) => a - b);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// ── Segment building ──────────────────────────────────────────────────────
|
|
165
|
+
|
|
166
|
+
_buildCandidates(boundaries, scenes, totalDuration) {
|
|
167
|
+
const candidates = [];
|
|
168
|
+
const MIN = 30, MAX = 90;
|
|
169
|
+
|
|
170
|
+
for (let i = 0; i < boundaries.length - 1; i++) {
|
|
171
|
+
for (let j = i + 1; j < boundaries.length; j++) {
|
|
172
|
+
const start = boundaries[i];
|
|
173
|
+
const end = boundaries[j];
|
|
174
|
+
const duration = end - start;
|
|
175
|
+
if (duration < MIN) continue;
|
|
176
|
+
if (duration > MAX) break;
|
|
177
|
+
// Estimate which scenes fall in this segment
|
|
178
|
+
const coveredScenes = this._scenesInRange(scenes, start, end, totalDuration);
|
|
179
|
+
candidates.push({ start, end, duration, coveredScenes, score: 0 });
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Score each candidate
|
|
184
|
+
for (const cand of candidates) {
|
|
185
|
+
cand.score = this._scoreSegment(cand);
|
|
186
|
+
}
|
|
187
|
+
return candidates;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
_scenesInRange(scenes, start, end, totalDuration) {
|
|
191
|
+
if (!scenes.length) return [];
|
|
192
|
+
const sceneDur = totalDuration / scenes.length;
|
|
193
|
+
return scenes.filter((_, i) => {
|
|
194
|
+
const sStart = i * sceneDur;
|
|
195
|
+
const sEnd = (i + 1) * sceneDur;
|
|
196
|
+
return sStart >= start && sEnd <= end;
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// ── Content scoring ───────────────────────────────────────────────────────
|
|
201
|
+
|
|
202
|
+
_scoreSegment(segment) {
|
|
203
|
+
const { duration, coveredScenes } = segment;
|
|
204
|
+
// Prefer segments near the start (hook potential), but not the very first 30s
|
|
205
|
+
const positionScore = segment.start > 30 && segment.start < 300 ? 0.8 : 0.4;
|
|
206
|
+
// Prefer denser content (more scenes)
|
|
207
|
+
const densityScore = Math.min(coveredScenes.length / 8, 1.0);
|
|
208
|
+
// Prefer 45-70s duration (sweet spot for Shorts)
|
|
209
|
+
const durationScore = duration >= 45 && duration <= 70 ? 1.0 : 0.6;
|
|
210
|
+
return (positionScore + densityScore + durationScore) / 3;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
_selectSegments(candidates) {
|
|
214
|
+
if (!candidates.length) return [];
|
|
215
|
+
// Sort by score descending
|
|
216
|
+
const sorted = [...candidates].sort((a, b) => b.score - a.score);
|
|
217
|
+
// Greedily select non-overlapping segments
|
|
218
|
+
const selected = [];
|
|
219
|
+
for (const cand of sorted) {
|
|
220
|
+
if (selected.length >= 5) break;
|
|
221
|
+
const overlaps = selected.some(s => cand.start < s.end && cand.end > s.start);
|
|
222
|
+
if (!overlaps) selected.push(cand);
|
|
223
|
+
}
|
|
224
|
+
return selected.slice(0, 5);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// ── Extraction + vertical reformat ───────────────────────────────────────
|
|
228
|
+
|
|
229
|
+
_extractSegment(masterPath, segment, scenes, outputPath) {
|
|
230
|
+
// Task #14: focal-point aware crop with safe-zone bias
|
|
231
|
+
const focal = this._focalPointForSegment(segment, scenes);
|
|
232
|
+
const xOff = this._focalToFFmpegX(focal.x);
|
|
233
|
+
const yOff = this._focalToFFmpegY();
|
|
234
|
+
// cropW=608, cropH=1080 → scale to 1080×1920 (9:16)
|
|
235
|
+
const CROP_W = 608;
|
|
236
|
+
const CROP_H = 1080;
|
|
237
|
+
|
|
238
|
+
// Extract → vertical crop → scale → normalise audio
|
|
239
|
+
// Note: crop filter uses pixel integers — no FFmpeg expression needed
|
|
240
|
+
const vf = [
|
|
241
|
+
`crop=${CROP_W}:${CROP_H}:${xOff}:${yOff}`,
|
|
242
|
+
'scale=1080:1920:flags=lanczos',
|
|
243
|
+
// Gentle quality enhancement — safe for any content
|
|
244
|
+
'unsharp=3:3:0.3:3:3:0.0',
|
|
245
|
+
'eq=contrast=1.02:brightness=0:saturation=1.04',
|
|
246
|
+
'format=yuv420p',
|
|
247
|
+
].join(',');
|
|
248
|
+
|
|
249
|
+
const result = spawnSync('ffmpeg', [
|
|
250
|
+
'-nostdin', '-y',
|
|
251
|
+
'-ss', String(segment.start),
|
|
252
|
+
'-t', String(segment.duration),
|
|
253
|
+
'-i', masterPath,
|
|
254
|
+
'-vf', vf,
|
|
255
|
+
'-af', 'loudnorm=I=-14:TP=-1:LRA=11',
|
|
256
|
+
'-c:v', 'libx264',
|
|
257
|
+
'-preset', 'fast',
|
|
258
|
+
'-crf', '22',
|
|
259
|
+
'-profile:v', 'high',
|
|
260
|
+
'-level', '4.1',
|
|
261
|
+
'-color_range', 'tv',
|
|
262
|
+
'-colorspace', 'bt709',
|
|
263
|
+
'-color_primaries', 'bt709',
|
|
264
|
+
'-color_trc', 'bt709',
|
|
265
|
+
'-c:a', 'aac',
|
|
266
|
+
'-b:a', '192k',
|
|
267
|
+
'-ar', '48000',
|
|
268
|
+
'-ac', '2',
|
|
269
|
+
'-movflags', '+faststart',
|
|
270
|
+
outputPath,
|
|
271
|
+
], { timeout: 300000, encoding: 'utf8' });
|
|
272
|
+
|
|
273
|
+
if (result.status !== 0) {
|
|
274
|
+
throw new Error(`Short extraction failed: ${(result.stderr || '').slice(-300)}`);
|
|
275
|
+
}
|
|
276
|
+
return outputPath;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Task #14 — Vertical Reformat helpers
|
|
281
|
+
*
|
|
282
|
+
* For a 1920×1080 input:
|
|
283
|
+
* cropW = floor(1080 * 9 / 16 / 2) * 2 = 608 (must be even for libx264)
|
|
284
|
+
* cropH = 1080
|
|
285
|
+
* x offset = clamp(round(focalX * 1920 - cropW/2), 0, 1920 - cropW)
|
|
286
|
+
* y offset = 0
|
|
287
|
+
*/
|
|
288
|
+
|
|
289
|
+
// Safe-zone bias: subjects are more compelling in the upper third of a Shorts frame.
|
|
290
|
+
// Focal y ≤ 0.35 maps naturally to upper-third; fallback for non-talking content is 0.5.
|
|
291
|
+
_defaultFocalY(scenes) {
|
|
292
|
+
for (const s of (scenes || [])) {
|
|
293
|
+
if (s.focal_point?.y !== undefined && s.focal_point.y <= 0.5) {
|
|
294
|
+
return s.focal_point.y;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return 0.35; // upper-third bias — safe against both top (180px) and bottom (400px) zones
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
_focalPointForSegment(segment, scenes) {
|
|
301
|
+
for (const scene of (segment.coveredScenes || [])) {
|
|
302
|
+
if (scene.focal_point?.x !== undefined) {
|
|
303
|
+
return { x: scene.focal_point.x, y: scene.focal_point.y ?? 0.35 };
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
// No focal point from scenes — apply safe-zone upper-third bias
|
|
307
|
+
const defaultY = this._defaultFocalY(scenes);
|
|
308
|
+
return { x: 0.5, y: defaultY };
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Compute pixel X offset for a 9:16 crop of a 1920×1080 frame.
|
|
313
|
+
* cropW = 608 (largest even number ≤ 1080 × 9/16 = 607.5)
|
|
314
|
+
* @param {number} focalX 0-1 relative horizontal position of subject
|
|
315
|
+
* @returns {number} integer pixel offset (clamped, even)
|
|
316
|
+
*/
|
|
317
|
+
_focalToFFmpegX(focalX) {
|
|
318
|
+
const INPUT_W = 1920;
|
|
319
|
+
const CROP_W = 608; // must be even; = floor(1080*9/16/2)*2
|
|
320
|
+
const raw = Math.round(focalX * INPUT_W - CROP_W / 2);
|
|
321
|
+
const clamped = Math.max(0, Math.min(INPUT_W - CROP_W, raw));
|
|
322
|
+
// Force even (libx264 requires even dimensions)
|
|
323
|
+
return clamped % 2 === 0 ? clamped : clamped - 1;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
_focalToFFmpegY() {
|
|
327
|
+
return 0; // for landscape→portrait, Y crop is always from top (retain full height)
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
_getDuration(videoPath) {
|
|
331
|
+
try {
|
|
332
|
+
const result = spawnSync('ffprobe', [
|
|
333
|
+
'-v', 'error',
|
|
334
|
+
'-show_entries', 'format=duration',
|
|
335
|
+
'-of', 'default=nk=1:nw=1',
|
|
336
|
+
videoPath,
|
|
337
|
+
], { encoding: 'utf8', timeout: 15000 });
|
|
338
|
+
return parseFloat((result.stdout || result.stderr || '0').trim());
|
|
339
|
+
} catch { return 0; }
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
module.exports = { ShortsExtractor };
|