@amg555/frameforge-mcp 2.3.1 → 2.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.mcp.json +2 -4
- package/CHANGELOG.md +20 -1
- package/CLAUDE.md +154 -18
- package/README.md +16 -12
- package/SECURITY.md +1 -3
- package/install.sh +1 -1
- package/package.json +7 -3
- package/plugins/video-pipeline/.claude-plugin/plugin.json +19 -0
- package/plugins/video-pipeline/skills/html-renderer/SKILL.md +292 -0
- package/plugins/video-pipeline/skills/video-pipeline/SKILL.md +308 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/ANIMATION_GUIDE.md +257 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/STORYBOARD.md +77 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/index.html +80 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/pipeline/briefings/diagram-author.md +43 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/pipeline/briefings/orchestrator.md +56 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/pipeline/briefings/scene-author.md +42 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/reel/engine.css +140 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/reel/engine.js +870 -0
- package/plugins/video-pipeline/skills/video-pipeline/assets/project/reel/virtual-clock.js +138 -0
- package/plugins/video-pipeline/skills/video-pipeline/references/mcp-tools.md +167 -0
- package/plugins/video-pipeline/skills/video-pipeline/references/providers.md +163 -0
- package/scripts/db-migrate.js +112 -0
- package/scripts/deep-validate.js +326 -0
- package/scripts/generate-initial-assets.js +342 -0
- package/scripts/html-renderer/new-project.mjs +408 -0
- package/scripts/html-renderer/transcribe.py +294 -0
- package/src/asset-library/README.md +2 -2
- package/src/asset-library/asset-library.js +1 -1
- package/src/cli/doctor.js +31 -1
- package/src/cli/make.js +52 -71
- package/src/cli/setup-keys.js +24 -19
- package/src/fact-checker/review-gate.js +5 -1
- package/src/harness/bin/harness.mjs +324 -0
- package/src/harness/lib/browser.mjs +137 -0
- package/src/harness/lib/capture.mjs +161 -0
- package/src/harness/lib/encode.mjs +89 -0
- package/src/harness/lib/jobs.mjs +135 -0
- package/src/harness/lib/pixels.mjs +137 -0
- package/src/harness/lib/sheets.mjs +164 -0
- package/src/harness/lib/verify-purity.mjs +119 -0
- package/src/harness/test/fixtures/broken-scene.html +27 -0
- package/src/harness/test/fixtures/reference-scene.html +23 -0
- package/src/harness/test/fixtures/sample-script.json +18 -0
- package/src/mcp-server/server.js +318 -66
- package/src/pipeline/external-adapter.js +55 -15
- package/src/pipeline/providers.js +46 -0
- package/src/pipeline/telemetry.js +1 -0
- package/src/utils/log-redaction.js +5 -3
- package/src/video-renderer/renderer.js +12 -4
- package/src/worker/server-render-free.js +2 -3
- package/src/workflows/assets.js +20 -43
- package/src/workflows/ingest.js +5 -5
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
transcribe.py — Generate a Whisper-format caption outline from FrameForge TTS audio.
|
|
4
|
+
|
|
5
|
+
Reads all voice_*.m4a (or voice_*.mp3) files from the project's audio/ directory,
|
|
6
|
+
transcribes each with OpenAI Whisper (if installed) or falls back to silence-based
|
|
7
|
+
segmentation via ffmpeg, and writes audio/outline.txt.
|
|
8
|
+
|
|
9
|
+
audio/outline.txt format (tab-separated, one word-group per line):
|
|
10
|
+
<start_seconds>\t<end_seconds>\t<text>
|
|
11
|
+
|
|
12
|
+
This file is consumed by captions() in reel/engine.js:
|
|
13
|
+
captions('audio/outline.txt');
|
|
14
|
+
|
|
15
|
+
Usage:
|
|
16
|
+
# From inside a scaffolded html-renderer project:
|
|
17
|
+
uv run scripts/transcribe.py
|
|
18
|
+
|
|
19
|
+
# Or with an explicit audio directory:
|
|
20
|
+
uv run scripts/transcribe.py --audio-dir /path/to/audio/
|
|
21
|
+
|
|
22
|
+
# Or without uv (python ≥ 3.9 required):
|
|
23
|
+
python scripts/transcribe.py
|
|
24
|
+
|
|
25
|
+
Requirements (Whisper path — highest quality):
|
|
26
|
+
pip install openai-whisper
|
|
27
|
+
# or: uv add openai-whisper
|
|
28
|
+
|
|
29
|
+
Requirements (ffmpeg fallback — no ML dependency):
|
|
30
|
+
ffmpeg must be on PATH
|
|
31
|
+
|
|
32
|
+
Environment:
|
|
33
|
+
WHISPER_MODEL Model size: tiny | base | small | medium | large (default: base)
|
|
34
|
+
AUDIO_DIR Override audio directory (default: audio/)
|
|
35
|
+
OUTLINE_PATH Override output path (default: audio/outline.txt)
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
import argparse
|
|
39
|
+
import json
|
|
40
|
+
import os
|
|
41
|
+
import re
|
|
42
|
+
import subprocess
|
|
43
|
+
import sys
|
|
44
|
+
import tempfile
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# ── Configuration ──────────────────────────────────────────────────────────────
|
|
49
|
+
|
|
50
|
+
DEFAULT_AUDIO_DIR = "audio"
|
|
51
|
+
DEFAULT_OUTLINE = "audio/outline.txt"
|
|
52
|
+
WHISPER_MODEL = os.environ.get("WHISPER_MODEL", "base")
|
|
53
|
+
WORDS_PER_GROUP = 6 # words per caption line (for ffmpeg fallback)
|
|
54
|
+
SILENCE_THRESHOLD = -35 # dB (for ffmpeg silence detection fallback)
|
|
55
|
+
SILENCE_DURATION = 0.3 # seconds
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# ── Helpers ────────────────────────────────────────────────────────────────────
|
|
59
|
+
|
|
60
|
+
def find_audio_files(audio_dir: Path) -> list[Path]:
|
|
61
|
+
"""Return all voice_*.m4a / voice_*.mp3 files sorted by scene index."""
|
|
62
|
+
files = sorted(
|
|
63
|
+
[p for p in audio_dir.iterdir() if re.match(r"voice_\d+\.(m4a|mp3)", p.name)],
|
|
64
|
+
key=lambda p: int(re.search(r"voice_(\d+)", p.name).group(1)),
|
|
65
|
+
)
|
|
66
|
+
if not files:
|
|
67
|
+
# Also accept any .m4a/.mp3 in the directory
|
|
68
|
+
files = sorted(
|
|
69
|
+
[p for p in audio_dir.iterdir() if p.suffix in (".m4a", ".mp3")],
|
|
70
|
+
key=lambda p: p.name,
|
|
71
|
+
)
|
|
72
|
+
return files
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def get_audio_duration(path: Path) -> float:
|
|
76
|
+
"""Return duration in seconds using ffprobe."""
|
|
77
|
+
result = subprocess.run(
|
|
78
|
+
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_streams", str(path)],
|
|
79
|
+
capture_output=True, text=True, timeout=30,
|
|
80
|
+
)
|
|
81
|
+
if result.returncode != 0:
|
|
82
|
+
return 5.0 # fallback estimate
|
|
83
|
+
data = json.loads(result.stdout)
|
|
84
|
+
for stream in data.get("streams", []):
|
|
85
|
+
if "duration" in stream:
|
|
86
|
+
return float(stream["duration"])
|
|
87
|
+
return 5.0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def convert_to_wav(src: Path, dst: Path) -> None:
|
|
91
|
+
"""Convert audio to 16kHz mono WAV for Whisper."""
|
|
92
|
+
subprocess.run(
|
|
93
|
+
["ffmpeg", "-y", "-i", str(src), "-ar", "16000", "-ac", "1", str(dst)],
|
|
94
|
+
capture_output=True, check=True, timeout=120,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# ── Whisper transcription ──────────────────────────────────────────────────────
|
|
99
|
+
|
|
100
|
+
def transcribe_whisper(audio_file: Path, time_offset: float) -> list[tuple[float, float, str]]:
|
|
101
|
+
"""
|
|
102
|
+
Transcribe one audio file with Whisper.
|
|
103
|
+
Returns list of (start, end, text) tuples with absolute timestamps.
|
|
104
|
+
"""
|
|
105
|
+
import whisper # type: ignore
|
|
106
|
+
|
|
107
|
+
model = whisper.load_model(WHISPER_MODEL)
|
|
108
|
+
|
|
109
|
+
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
|
110
|
+
wav_path = Path(tmp.name)
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
convert_to_wav(audio_file, wav_path)
|
|
114
|
+
result = model.transcribe(str(wav_path), word_timestamps=True, language=None)
|
|
115
|
+
finally:
|
|
116
|
+
wav_path.unlink(missing_ok=True)
|
|
117
|
+
|
|
118
|
+
segments = []
|
|
119
|
+
for seg in result.get("segments", []):
|
|
120
|
+
start = seg["start"] + time_offset
|
|
121
|
+
end = seg["end"] + time_offset
|
|
122
|
+
text = seg["text"].strip()
|
|
123
|
+
if text:
|
|
124
|
+
segments.append((start, end, text))
|
|
125
|
+
|
|
126
|
+
return segments
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ── FFmpeg silence-based fallback ──────────────────────────────────────────────
|
|
130
|
+
|
|
131
|
+
def transcribe_ffmpeg_fallback(audio_file: Path, time_offset: float, narration: str | None = None) -> list[tuple[float, float, str]]:
|
|
132
|
+
"""
|
|
133
|
+
Fallback when Whisper is not installed.
|
|
134
|
+
Uses ffmpeg silence detection to estimate word timings, then distributes
|
|
135
|
+
the narration text evenly across the detected speech segments.
|
|
136
|
+
Falls back to even word distribution if narration is unavailable.
|
|
137
|
+
"""
|
|
138
|
+
duration = get_audio_duration(audio_file)
|
|
139
|
+
|
|
140
|
+
# Detect silence boundaries
|
|
141
|
+
result = subprocess.run(
|
|
142
|
+
[
|
|
143
|
+
"ffmpeg", "-i", str(audio_file),
|
|
144
|
+
"-af", f"silencedetect=noise={SILENCE_THRESHOLD}dB:d={SILENCE_DURATION}",
|
|
145
|
+
"-f", "null", "-",
|
|
146
|
+
],
|
|
147
|
+
capture_output=True, text=True, timeout=60,
|
|
148
|
+
)
|
|
149
|
+
stderr = result.stderr
|
|
150
|
+
|
|
151
|
+
# Parse silence_start / silence_end pairs
|
|
152
|
+
starts = [float(m) for m in re.findall(r"silence_start: ([\d.]+)", stderr)]
|
|
153
|
+
ends = [float(m) for m in re.findall(r"silence_end: ([\d.]+)", stderr)]
|
|
154
|
+
|
|
155
|
+
# Build speech segments from silence gaps
|
|
156
|
+
speech_segments: list[tuple[float, float]] = []
|
|
157
|
+
cursor = 0.0
|
|
158
|
+
for s, e in zip(starts, ends):
|
|
159
|
+
if s > cursor + 0.1:
|
|
160
|
+
speech_segments.append((cursor, s))
|
|
161
|
+
cursor = e
|
|
162
|
+
if cursor < duration - 0.1:
|
|
163
|
+
speech_segments.append((cursor, duration))
|
|
164
|
+
|
|
165
|
+
if not speech_segments:
|
|
166
|
+
speech_segments = [(0.0, duration)]
|
|
167
|
+
|
|
168
|
+
# Distribute narration text across speech segments
|
|
169
|
+
words = (narration or "").split() if narration else []
|
|
170
|
+
if not words:
|
|
171
|
+
# No narration — emit one entry per speech segment with placeholder
|
|
172
|
+
return [
|
|
173
|
+
(seg[0] + time_offset, seg[1] + time_offset, f"[scene {i+1}]")
|
|
174
|
+
for i, seg in enumerate(speech_segments)
|
|
175
|
+
]
|
|
176
|
+
|
|
177
|
+
total_seg_dur = sum(e - s for s, e in speech_segments)
|
|
178
|
+
captions: list[tuple[float, float, str]] = []
|
|
179
|
+
word_idx = 0
|
|
180
|
+
|
|
181
|
+
for seg_start, seg_end in speech_segments:
|
|
182
|
+
seg_dur = seg_end - seg_start
|
|
183
|
+
seg_word_count = max(1, round(len(words) * (seg_dur / total_seg_dur)))
|
|
184
|
+
seg_words = words[word_idx : word_idx + seg_word_count]
|
|
185
|
+
word_idx += seg_word_count
|
|
186
|
+
|
|
187
|
+
# Split segment words into groups of WORDS_PER_GROUP
|
|
188
|
+
for i in range(0, len(seg_words), WORDS_PER_GROUP):
|
|
189
|
+
group = seg_words[i : i + WORDS_PER_GROUP]
|
|
190
|
+
frac_start = i / len(seg_words)
|
|
191
|
+
frac_end = min(1.0, (i + WORDS_PER_GROUP) / len(seg_words))
|
|
192
|
+
t_start = seg_start + frac_start * seg_dur + time_offset
|
|
193
|
+
t_end = seg_start + frac_end * seg_dur + time_offset
|
|
194
|
+
captions.append((t_start, t_end, " ".join(group)))
|
|
195
|
+
|
|
196
|
+
# Append any remaining words to the last segment
|
|
197
|
+
if word_idx < len(words):
|
|
198
|
+
last = captions[-1] if captions else (time_offset, time_offset + duration, "")
|
|
199
|
+
captions.append((last[1], last[1] + 1.0, " ".join(words[word_idx:])))
|
|
200
|
+
|
|
201
|
+
return captions
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
# ── Main ───────────────────────────────────────────────────────────────────────
|
|
205
|
+
|
|
206
|
+
def main() -> None:
|
|
207
|
+
parser = argparse.ArgumentParser(description="Generate caption outline from FrameForge TTS audio.")
|
|
208
|
+
parser.add_argument("--audio-dir", default=os.environ.get("AUDIO_DIR", DEFAULT_AUDIO_DIR),
|
|
209
|
+
help="Directory containing voice_*.m4a files (default: audio/)")
|
|
210
|
+
parser.add_argument("--outline", default=os.environ.get("OUTLINE_PATH", DEFAULT_OUTLINE),
|
|
211
|
+
help="Output path for outline.txt (default: audio/outline.txt)")
|
|
212
|
+
parser.add_argument("--script", default=None,
|
|
213
|
+
help="Optional: path to FrameForge script JSON for narration text (improves ffmpeg fallback)")
|
|
214
|
+
parser.add_argument("--no-whisper", action="store_true",
|
|
215
|
+
help="Force ffmpeg fallback even if Whisper is installed")
|
|
216
|
+
args = parser.parse_args()
|
|
217
|
+
|
|
218
|
+
audio_dir = Path(args.audio_dir)
|
|
219
|
+
outline_path = Path(args.outline)
|
|
220
|
+
|
|
221
|
+
if not audio_dir.exists():
|
|
222
|
+
print(f"[transcribe] ERROR: audio directory not found: {audio_dir}", file=sys.stderr)
|
|
223
|
+
print(f" Copy FrameForge TTS outputs (voice_0.m4a, voice_1.m4a …) into {audio_dir}/", file=sys.stderr)
|
|
224
|
+
sys.exit(1)
|
|
225
|
+
|
|
226
|
+
audio_files = find_audio_files(audio_dir)
|
|
227
|
+
if not audio_files:
|
|
228
|
+
print(f"[transcribe] ERROR: no voice_*.m4a or voice_*.mp3 files found in {audio_dir}", file=sys.stderr)
|
|
229
|
+
sys.exit(1)
|
|
230
|
+
|
|
231
|
+
print(f"[transcribe] Found {len(audio_files)} audio file(s) in {audio_dir}")
|
|
232
|
+
|
|
233
|
+
# Load narration text from script JSON if provided (for ffmpeg fallback)
|
|
234
|
+
narrations: list[str | None] = [None] * len(audio_files)
|
|
235
|
+
if args.script:
|
|
236
|
+
try:
|
|
237
|
+
raw = json.loads(Path(args.script).read_text())
|
|
238
|
+
script = raw.get("script", raw)
|
|
239
|
+
scenes = script.get("scenes", [])
|
|
240
|
+
for i, scene in enumerate(scenes):
|
|
241
|
+
if i < len(narrations):
|
|
242
|
+
narrations[i] = scene.get("narration")
|
|
243
|
+
print(f"[transcribe] Loaded {len(scenes)} narration(s) from {args.script}")
|
|
244
|
+
except Exception as exc:
|
|
245
|
+
print(f"[transcribe] WARNING: could not load script JSON: {exc}", file=sys.stderr)
|
|
246
|
+
|
|
247
|
+
# Choose transcription backend
|
|
248
|
+
use_whisper = False
|
|
249
|
+
if not args.no_whisper:
|
|
250
|
+
try:
|
|
251
|
+
import whisper # type: ignore # noqa: F401
|
|
252
|
+
use_whisper = True
|
|
253
|
+
print(f"[transcribe] Using Whisper model '{WHISPER_MODEL}'")
|
|
254
|
+
except ImportError:
|
|
255
|
+
print("[transcribe] Whisper not installed — using ffmpeg silence-detection fallback.")
|
|
256
|
+
print(" Install for better accuracy: pip install openai-whisper")
|
|
257
|
+
|
|
258
|
+
# Transcribe each file and accumulate captions with absolute timestamps
|
|
259
|
+
all_captions: list[tuple[float, float, str]] = []
|
|
260
|
+
time_offset = 0.0
|
|
261
|
+
|
|
262
|
+
for i, audio_file in enumerate(audio_files):
|
|
263
|
+
print(f"[transcribe] {audio_file.name} …", end=" ", flush=True)
|
|
264
|
+
try:
|
|
265
|
+
if use_whisper:
|
|
266
|
+
segments = transcribe_whisper(audio_file, time_offset)
|
|
267
|
+
else:
|
|
268
|
+
segments = transcribe_ffmpeg_fallback(audio_file, time_offset, narrations[i])
|
|
269
|
+
all_captions.extend(segments)
|
|
270
|
+
duration = get_audio_duration(audio_file)
|
|
271
|
+
time_offset += duration
|
|
272
|
+
print(f"{len(segments)} segment(s), duration {duration:.1f}s")
|
|
273
|
+
except Exception as exc:
|
|
274
|
+
duration = get_audio_duration(audio_file)
|
|
275
|
+
time_offset += duration
|
|
276
|
+
print(f"ERROR: {exc}", file=sys.stderr)
|
|
277
|
+
|
|
278
|
+
if not all_captions:
|
|
279
|
+
print("[transcribe] ERROR: no captions generated.", file=sys.stderr)
|
|
280
|
+
sys.exit(1)
|
|
281
|
+
|
|
282
|
+
# Write outline.txt
|
|
283
|
+
outline_path.parent.mkdir(parents=True, exist_ok=True)
|
|
284
|
+
with open(outline_path, "w", encoding="utf-8") as fh:
|
|
285
|
+
for start, end, text in all_captions:
|
|
286
|
+
fh.write(f"{start:.3f}\t{end:.3f}\t{text}\n")
|
|
287
|
+
|
|
288
|
+
print(f"[transcribe] Written {len(all_captions)} caption(s) → {outline_path}")
|
|
289
|
+
print(f"[transcribe] Total duration: {time_offset:.1f}s")
|
|
290
|
+
print(f"[transcribe] Use in edit.js: captions('{outline_path}')")
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
if __name__ == "__main__":
|
|
294
|
+
main()
|
|
@@ -148,7 +148,7 @@ Single character only.
|
|
|
148
148
|
1920x1080 landscape.
|
|
149
149
|
`;
|
|
150
150
|
|
|
151
|
-
const imageUrl = await
|
|
151
|
+
const imageUrl = await generateAssetVisual({
|
|
152
152
|
prompt,
|
|
153
153
|
width: 1920,
|
|
154
154
|
height: 1080,
|
|
@@ -179,7 +179,7 @@ All assets track `schema_version: "1.0.0"` for evolution:
|
|
|
179
179
|
- **Metadata lookup:** <10ms (PostgreSQL indexed)
|
|
180
180
|
- **Semantic search:** <200ms (cached embeddings)
|
|
181
181
|
- **Asset composition:** <500ms (image layering)
|
|
182
|
-
- **Asset generation:**
|
|
182
|
+
- **Asset generation:** <100ms (code-driven / local procedural)
|
|
183
183
|
|
|
184
184
|
## Storage
|
|
185
185
|
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* Architecture:
|
|
8
8
|
* - Metadata filtering (primary): Fast PostgreSQL lookup
|
|
9
9
|
* - Semantic search (fallback): Embedding similarity if no metadata match
|
|
10
|
-
* - Generation:
|
|
10
|
+
* - Generation: Procedural SVG / local visual composition
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
const fs = require('fs').promises;
|
package/src/cli/doctor.js
CHANGED
|
@@ -89,7 +89,37 @@ if (edgeTts.status === 0) {
|
|
|
89
89
|
);
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
-
// 5.
|
|
92
|
+
// 5. Playwright / Chromium (required for code-driven animation — Path 2)
|
|
93
|
+
const playwright = spawnSync('node', ['-e', "require('playwright')"], {
|
|
94
|
+
encoding: 'utf8', timeout: 5000,
|
|
95
|
+
// Look in the project's own node_modules first (how it ships in the npm package)
|
|
96
|
+
cwd: path.resolve(__dirname, '../..'),
|
|
97
|
+
});
|
|
98
|
+
const playwrightGlobal = spawnSync('node', ['-e', "require('playwright')"], {
|
|
99
|
+
encoding: 'utf8', timeout: 5000,
|
|
100
|
+
});
|
|
101
|
+
const hasPlaywright = playwright.status === 0 || playwrightGlobal.status === 0;
|
|
102
|
+
|
|
103
|
+
// Check for a locally installed Chromium browser
|
|
104
|
+
const chromiumPaths = [
|
|
105
|
+
path.join(require('os').homedir(), '.cache', 'ms-playwright'), // Linux / macOS default
|
|
106
|
+
path.join(process.env.LOCALAPPDATA || '', 'ms-playwright'), // Windows
|
|
107
|
+
path.join(require('os').homedir(), 'Library', 'Caches', 'ms-playwright'), // macOS alt
|
|
108
|
+
];
|
|
109
|
+
const hasChromium = chromiumPaths.some(p => { try { return require('fs').readdirSync(p).length > 0; } catch { return false; } });
|
|
110
|
+
|
|
111
|
+
if (hasPlaywright && hasChromium) {
|
|
112
|
+
pass('Playwright', 'playwright package + Chromium browser installed (code-driven animation ready)');
|
|
113
|
+
} else if (hasPlaywright && !hasChromium) {
|
|
114
|
+
warn('Playwright', 'playwright package found but Chromium browser not installed',
|
|
115
|
+
'Run in your animation project dir: npx playwright install chromium');
|
|
116
|
+
} else {
|
|
117
|
+
warn('Playwright', 'Not installed — required for code-driven animation (Path 2)',
|
|
118
|
+
'Run in your animation project dir: npm install playwright && npx playwright install chromium\n' +
|
|
119
|
+
' (Path 1 via create_video_from_script does not need Playwright)');
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// 6. System Font (for video subtitles/captions)
|
|
93
123
|
const FONT_CANDIDATES = [
|
|
94
124
|
'C:\\Windows\\Fonts\\arialbd.ttf',
|
|
95
125
|
'C:\\Windows\\Fonts\\calibrib.ttf',
|
package/src/cli/make.js
CHANGED
|
@@ -36,7 +36,8 @@
|
|
|
36
36
|
* --channels <list> file,webhook (default: file)
|
|
37
37
|
* --no-research Skip research stage (use --topic as raw facts)
|
|
38
38
|
* --no-script Skip script stage (requires --scenes-file)
|
|
39
|
-
* --no-images Skip AI image generation (use
|
|
39
|
+
* --no-images Skip AI image generation (use code-driven animation or user clips)
|
|
40
|
+
* --ai-images Opt-in: generate AI diffusion images (default: code animation)
|
|
40
41
|
* --no-tts Skip TTS synthesis (use audio_path in scenes)
|
|
41
42
|
* --no-shorts Skip Shorts extraction
|
|
42
43
|
* --no-publish Skip publishing (render only)
|
|
@@ -84,6 +85,14 @@ const LANGUAGE_VOICES = {
|
|
|
84
85
|
|
|
85
86
|
// ── Argument parsing ──────────────────────────────────────────────────────────
|
|
86
87
|
|
|
88
|
+
// Dispatch to code-driven animation harness if invoked as `frameforge harness ...`
|
|
89
|
+
if (process.argv[2] === 'harness') {
|
|
90
|
+
const harnessBin = path.resolve(__dirname, '../harness/bin/harness.mjs');
|
|
91
|
+
const { spawnSync } = require('child_process');
|
|
92
|
+
const res = spawnSync(process.execPath, [harnessBin, ...process.argv.slice(3)], { stdio: 'inherit' });
|
|
93
|
+
process.exit(res.status ?? 0);
|
|
94
|
+
}
|
|
95
|
+
|
|
87
96
|
function parseArgs(argv) {
|
|
88
97
|
const args = {
|
|
89
98
|
mode: 'full-auto',
|
|
@@ -108,6 +117,7 @@ function parseArgs(argv) {
|
|
|
108
117
|
case '--no-research': args.noResearch = true; break;
|
|
109
118
|
case '--no-script': args.noScript = true; break;
|
|
110
119
|
case '--no-images': args.noImages = true; break;
|
|
120
|
+
case '--ai-images': args.aiImages = true; break;
|
|
111
121
|
case '--no-tts': args.noTts = true; break;
|
|
112
122
|
case '--no-shorts': args.shorts = false; break;
|
|
113
123
|
case '--no-publish': args.noPublish = true; break;
|
|
@@ -225,84 +235,48 @@ function logSection(title) {
|
|
|
225
235
|
}
|
|
226
236
|
|
|
227
237
|
// ── Thumbnail generator ───────────────────────────────────────────────────────
|
|
228
|
-
//
|
|
229
|
-
//
|
|
230
|
-
// Non-fatal: failures are logged as warnings, not errors.
|
|
238
|
+
// Generates a crisp code-driven procedural thumbnail with FFmpeg.
|
|
239
|
+
// Saves to <outDir>/thumbnail.jpg. Non-fatal: failures are logged as warnings.
|
|
231
240
|
|
|
232
241
|
async function generateThumbnail({ thumbnailPrompt, title, outDir }) {
|
|
233
|
-
if (!thumbnailPrompt) return null;
|
|
242
|
+
if (!thumbnailPrompt && !title) return null;
|
|
234
243
|
const outputPath = path.join(outDir, 'thumbnail.jpg');
|
|
235
244
|
try {
|
|
236
|
-
const model = process.env.POLLINATIONS_MODEL || 'flux';
|
|
237
|
-
const prompt = encodeURIComponent(thumbnailPrompt.slice(0, 500));
|
|
238
|
-
const imgUrl = `https://image.pollinations.ai/prompt/${prompt}?width=1280&height=720&model=${model}&nologo=true`;
|
|
239
|
-
// Download image
|
|
240
|
-
const imgPath = path.join(outDir, 'thumbnail_raw.jpg');
|
|
241
|
-
await _downloadFile(imgUrl, imgPath);
|
|
242
|
-
if (!fs.existsSync(imgPath)) return null;
|
|
243
|
-
|
|
244
|
-
// Optionally burn title text with FFmpeg
|
|
245
245
|
const { spawnSync } = require('child_process');
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
} else {
|
|
273
|
-
fs.renameSync(imgPath, outputPath);
|
|
246
|
+
const safeTitle = (title || 'FrameForge Video').replace(/['":\\]/g, ' ').slice(0, 80);
|
|
247
|
+
const fontCandidates = process.platform === 'win32'
|
|
248
|
+
? ['C:/Windows/Fonts/arialbd.ttf', 'C:/Windows/Fonts/calibrib.ttf', 'C:/Windows/Fonts/segoeui.ttf']
|
|
249
|
+
: process.platform === 'darwin'
|
|
250
|
+
? ['/System/Library/Fonts/Helvetica.ttc', '/Library/Fonts/Arial Bold.ttf']
|
|
251
|
+
: ['/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', '/usr/share/fonts/truetype/freefont/FreeSansBold.ttf'];
|
|
252
|
+
const font = process.env.FFMPEG_FONT
|
|
253
|
+
|| fontCandidates.find(f => fs.existsSync(f))
|
|
254
|
+
|| '';
|
|
255
|
+
|
|
256
|
+
const drawFilter = font
|
|
257
|
+
? `drawtext=fontfile='${font}':text='${safeTitle}':fontcolor=white:fontsize=56:x=(w-text_w)/2:y=(h-text_h)/2:shadowx=3:shadowy=3:shadowcolor=black`
|
|
258
|
+
: `drawtext=text='${safeTitle}':fontcolor=white:fontsize=56:x=(w-text_w)/2:y=(h-text_h)/2:shadowx=3:shadowy=3:shadowcolor=black`;
|
|
259
|
+
|
|
260
|
+
const result = spawnSync('ffmpeg', [
|
|
261
|
+
'-y',
|
|
262
|
+
'-f', 'lavfi',
|
|
263
|
+
'-i', 'color=c=0x0f172a:s=1280x720:d=1',
|
|
264
|
+
'-vf', drawFilter,
|
|
265
|
+
'-frames:v', '1',
|
|
266
|
+
'-q:v', '3',
|
|
267
|
+
outputPath,
|
|
268
|
+
], { encoding: 'utf8' });
|
|
269
|
+
|
|
270
|
+
if (result.status === 0 && fs.existsSync(outputPath)) {
|
|
271
|
+
return outputPath;
|
|
274
272
|
}
|
|
275
|
-
return
|
|
273
|
+
return null;
|
|
276
274
|
} catch (err) {
|
|
277
275
|
log('Thumbnail', `⚠️ Thumbnail generation failed (non-fatal): ${err.message}`);
|
|
278
276
|
return null;
|
|
279
277
|
}
|
|
280
278
|
}
|
|
281
279
|
|
|
282
|
-
function _downloadFile(url, destPath) {
|
|
283
|
-
return new Promise((resolve, reject) => {
|
|
284
|
-
const client = url.startsWith('https') ? https : http;
|
|
285
|
-
const file = fs.createWriteStream(destPath);
|
|
286
|
-
function follow(u, hops) {
|
|
287
|
-
if (hops > 5) return reject(new Error('Too many redirects'));
|
|
288
|
-
client.get(u, (res) => {
|
|
289
|
-
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
290
|
-
res.resume();
|
|
291
|
-
return follow(res.headers.location, hops + 1);
|
|
292
|
-
}
|
|
293
|
-
if (res.statusCode !== 200) {
|
|
294
|
-
res.resume();
|
|
295
|
-
return reject(new Error(`HTTP ${res.statusCode} downloading ${u}`));
|
|
296
|
-
}
|
|
297
|
-
res.pipe(file);
|
|
298
|
-
file.on('finish', () => file.close(resolve));
|
|
299
|
-
file.on('error', reject);
|
|
300
|
-
}).on('error', reject);
|
|
301
|
-
}
|
|
302
|
-
follow(url, 0);
|
|
303
|
-
});
|
|
304
|
-
}
|
|
305
|
-
|
|
306
280
|
// ── Single video runner (used by both single-topic and batch modes) ───────────
|
|
307
281
|
|
|
308
282
|
async function runSingleVideo(args, topic, outDir, workflows) {
|
|
@@ -414,12 +388,18 @@ async function runSingleVideo(args, topic, outDir, workflows) {
|
|
|
414
388
|
// ── Stage C: Assets (images + TTS) ────────────────────────────────────────
|
|
415
389
|
logSection('Stage C: Assets (images + TTS)');
|
|
416
390
|
|
|
391
|
+
// Code-driven first: AI diffusion images are disabled by default to eliminate AI artifacts.
|
|
392
|
+
const enableAiImages = Boolean(args.aiImages || process.env.ENABLE_AI_IMAGES === 'true');
|
|
393
|
+
const skipImages = args.noImages !== undefined ? args.noImages : !enableAiImages;
|
|
394
|
+
|
|
395
|
+
if (!enableAiImages) {
|
|
396
|
+
log('Assets', 'ℹ️ Code-driven animation: kinetic typography & user media (diffusion disabled)');
|
|
397
|
+
}
|
|
398
|
+
|
|
417
399
|
// Pass stage-skip flags into scenes so runAssets can honour them
|
|
418
400
|
const preparedScenes = scenes.map(s => ({
|
|
419
401
|
...s,
|
|
420
|
-
|
|
421
|
-
_skipImageGen: args.noImages,
|
|
422
|
-
// --no-tts: mark scene as if it has pre-recorded audio → renderer skips TTS
|
|
402
|
+
_skipImageGen: skipImages,
|
|
423
403
|
_skipTts: args.noTts,
|
|
424
404
|
}));
|
|
425
405
|
|
|
@@ -427,8 +407,9 @@ async function runSingleVideo(args, topic, outDir, workflows) {
|
|
|
427
407
|
productionId,
|
|
428
408
|
scenes: preparedScenes,
|
|
429
409
|
parallelism: 4,
|
|
430
|
-
skipImages
|
|
410
|
+
skipImages,
|
|
431
411
|
skipTts: args.noTts,
|
|
412
|
+
enableAiImages,
|
|
432
413
|
});
|
|
433
414
|
log('Assets', `✅ ${assetsResult.sceneCount} scenes prepared`);
|
|
434
415
|
|
package/src/cli/setup-keys.js
CHANGED
|
@@ -31,12 +31,9 @@
|
|
|
31
31
|
* openai — OpenAI TTS. Uses OPENAI_API_KEY.
|
|
32
32
|
* webhook — Any TTS endpoint. Set WEBHOOK_TTS_URL.
|
|
33
33
|
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
* replicate — FLUX models. Get key: replicate.com
|
|
38
|
-
* dalle — DALL-E 3. Uses OPENAI_API_KEY.
|
|
39
|
-
* webhook — Any image endpoint. Set WEBHOOK_IMAGE_URL.
|
|
34
|
+
* Visuals & Media:
|
|
35
|
+
* code-driven — Default. Crisp kinetic typography and cards. 0 keys needed.
|
|
36
|
+
* user-media — Directly supply image_path or video_path per scene.
|
|
40
37
|
*/
|
|
41
38
|
|
|
42
39
|
'use strict';
|
|
@@ -128,7 +125,7 @@ function checkEnv(envPath) {
|
|
|
128
125
|
{ key: 'PROVIDER_TTS', label: 'TTS provider', optional: false },
|
|
129
126
|
{ key: 'ELEVENLABS_API_KEY', label: 'ElevenLabs key', optional: true, needs: env.PROVIDER_TTS === 'elevenlabs' },
|
|
130
127
|
{ key: 'AZURE_SPEECH_KEY', label: 'Azure key', optional: true, needs: env.PROVIDER_TTS === 'azure' },
|
|
131
|
-
{ key: 'PROVIDER_IMAGE', label: 'Image provider', optional:
|
|
128
|
+
{ key: 'PROVIDER_IMAGE', label: 'Image provider', optional: true },
|
|
132
129
|
{ key: 'FALAI_API_KEY', label: 'fal.ai key', optional: true, needs: env.PROVIDER_IMAGE === 'falai' },
|
|
133
130
|
{ key: 'REPLICATE_API_KEY', label: 'Replicate key', optional: true, needs: env.PROVIDER_IMAGE === 'replicate' },
|
|
134
131
|
{ key: 'PUBLISH_WEBHOOK_URL', label: 'Publish webhook URL', optional: true },
|
|
@@ -175,7 +172,7 @@ Press Enter to accept a default. Ctrl+C to cancel.
|
|
|
175
172
|
✅ No keys are required to make videos.
|
|
176
173
|
Your IDE writes the scenes — FrameForge uses free providers by default:
|
|
177
174
|
• edge-tts — free voice (no account)
|
|
178
|
-
•
|
|
175
|
+
• code-driven animation — free motion graphics (no account, zero keys)
|
|
179
176
|
• FFmpeg — free local render (already on your machine)
|
|
180
177
|
|
|
181
178
|
You only need keys for the auto-research pipeline (FrameForge writes the
|
|
@@ -241,12 +238,19 @@ Press Enter to accept a default. Ctrl+C to cancel.
|
|
|
241
238
|
cfg.PROVIDER_TTS = await menu('Choose TTS provider', [
|
|
242
239
|
['edge-tts', 'edge-tts — free, offline, 400+ voices. No key needed.'],
|
|
243
240
|
['elevenlabs', 'ElevenLabs — best quality + voice cloning. elevenlabs.io'],
|
|
241
|
+
['voxforge', 'VoxForge — self-hosted voice cloning. github.com/amg555/voxforge'],
|
|
244
242
|
['azure', 'Azure Cognitive Speech. portal.azure.com'],
|
|
245
243
|
['openai', 'OpenAI TTS (uses OPENAI_API_KEY above)'],
|
|
246
|
-
['webhook', 'Custom/
|
|
244
|
+
['webhook', 'Custom/any OpenAI-compatible TTS endpoint'],
|
|
247
245
|
], 'edge-tts');
|
|
248
246
|
|
|
249
|
-
if (cfg.PROVIDER_TTS === '
|
|
247
|
+
if (cfg.PROVIDER_TTS === 'voxforge') {
|
|
248
|
+
cfg.VOXFORGE_URL = await ask(' VoxForge base URL', 'https://api.voxforge.io');
|
|
249
|
+
cfg.VOXFORGE_KEY = await askSecret(' VoxForge API key (leave blank if self-hosted without auth)');
|
|
250
|
+
cfg.VOXFORGE_VOICE = await ask(' Voice ID or cloned speaker ID', 'narrator');
|
|
251
|
+
cfg.VOXFORGE_MODEL = await ask(' Model ID (leave blank for default)', 'voxforge-v1');
|
|
252
|
+
console.log(' ℹ️ Self-hosted? Set VOXFORGE_URL to your instance URL (e.g. http://localhost:8080).');
|
|
253
|
+
} else if (cfg.PROVIDER_TTS === 'elevenlabs') {
|
|
250
254
|
cfg.ELEVENLABS_API_KEY = await askSecret(' ElevenLabs API key');
|
|
251
255
|
cfg.ELEVENLABS_VOICE_ID = await ask(' Voice ID (leave blank for default Adam)', 'pNInz6obpgDQGcFmaJgB');
|
|
252
256
|
console.log(' ℹ️ For voice cloning: set ELEVENLABS_VOICE_ID to your cloned voice ID.');
|
|
@@ -260,18 +264,19 @@ Press Enter to accept a default. Ctrl+C to cancel.
|
|
|
260
264
|
}
|
|
261
265
|
|
|
262
266
|
// ── Image ─────────────────────────────────────────────────────────────────
|
|
263
|
-
section('3 / 5 — Image
|
|
264
|
-
console.log('
|
|
265
|
-
console.log('
|
|
266
|
-
console.log('
|
|
267
|
+
section('3 / 5 — Visuals & Image Provider (optional)');
|
|
268
|
+
console.log(' Default: Code-driven animations, kinetic typography, and user-provided media.');
|
|
269
|
+
console.log(' Zero keys needed for high quality vector animation.');
|
|
270
|
+
console.log(' Select a diffusion provider only if you explicitly want AI-generated pictures.\n');
|
|
267
271
|
|
|
268
272
|
cfg.PROVIDER_IMAGE = await menu('Choose image provider', [
|
|
269
|
-
['
|
|
270
|
-
['
|
|
273
|
+
['none', 'None (Recommended) — code-driven animations & user media. Zero keys, best quality.'],
|
|
274
|
+
['pollinations', 'Pollinations — free AI images (no key needed, FLUX model)'],
|
|
275
|
+
['falai', 'fal.ai — fast cloud diffusion. Get key: fal.ai'],
|
|
271
276
|
['replicate', 'Replicate FLUX-schnell. Get key: replicate.com'],
|
|
272
|
-
['dalle', 'DALL-E 3 (uses OPENAI_API_KEY
|
|
277
|
+
['dalle', 'DALL-E 3 (uses OPENAI_API_KEY)'],
|
|
273
278
|
['webhook', 'Custom/external image API endpoint'],
|
|
274
|
-
], '
|
|
279
|
+
], 'none');
|
|
275
280
|
|
|
276
281
|
if (cfg.PROVIDER_IMAGE === 'falai') {
|
|
277
282
|
cfg.FALAI_API_KEY = await askSecret(' fal.ai API key');
|
|
@@ -326,7 +331,7 @@ Press Enter to accept a default. Ctrl+C to cancel.
|
|
|
326
331
|
|
|
327
332
|
const groups = {
|
|
328
333
|
'# ── LLM ─────────────────────────────────────────────────────────': ['PROVIDER_LLM','GROQ_API_KEY','CEREBRAS_API_KEY','OPENROUTER_API_KEY','ANTHROPIC_API_KEY','GEMINI_API_KEY','MISTRAL_API_KEY','TOGETHER_API_KEY','OPENAI_API_KEY','OLLAMA_BASE_URL','OLLAMA_MODEL','WEBHOOK_LLM_URL','WEBHOOK_LLM_HEADERS','WEBHOOK_LLM_RESPONSE_PATH','LLM_MODEL'],
|
|
329
|
-
'# ── TTS ─────────────────────────────────────────────────────────': ['PROVIDER_TTS','ELEVENLABS_API_KEY','ELEVENLABS_VOICE_ID','AZURE_SPEECH_KEY','AZURE_SPEECH_REGION','WEBHOOK_TTS_URL','WEBHOOK_TTS_RESPONSE','WEBHOOK_TTS_RESPONSE_PATH','TTS_VOICE','TTS_RATE'],
|
|
334
|
+
'# ── TTS ─────────────────────────────────────────────────────────': ['PROVIDER_TTS','VOXFORGE_URL','VOXFORGE_KEY','VOXFORGE_VOICE','VOXFORGE_MODEL','ELEVENLABS_API_KEY','ELEVENLABS_VOICE_ID','AZURE_SPEECH_KEY','AZURE_SPEECH_REGION','WEBHOOK_TTS_URL','WEBHOOK_TTS_RESPONSE','WEBHOOK_TTS_RESPONSE_PATH','TTS_VOICE','TTS_RATE'],
|
|
330
335
|
'# ── Image generation ─────────────────────────────────────────────': ['PROVIDER_IMAGE','FALAI_API_KEY','REPLICATE_API_KEY','WEBHOOK_IMAGE_URL','WEBHOOK_IMAGE_RESPONSE','WEBHOOK_IMAGE_RESPONSE_PATH','POLLINATIONS_MODEL'],
|
|
331
336
|
'# ── Delivery webhooks ────────────────────────────────────────────': ['PUBLISH_WEBHOOK_URL','REVIEW_NOTIFY_URL'],
|
|
332
337
|
'# ── External platform adapter ────────────────────────────────────': ['EXTERNAL_PLATFORM_NAME','EXTERNAL_PLATFORM_URL','EXTERNAL_PLATFORM_KEY','EXTERNAL_PLATFORM_INGEST_PATH','EXTERNAL_PLATFORM_RENDER_PATH','EXTERNAL_PLATFORM_STATUS_PATH'],
|
|
@@ -49,7 +49,11 @@ class FactCheckerReviewGate {
|
|
|
49
49
|
this.stateMachine = stateMachine;
|
|
50
50
|
this.notifyUrl = notifyUrl || process.env.REVIEW_NOTIFY_URL || '';
|
|
51
51
|
this.reviewsDir = reviewsDir;
|
|
52
|
-
|
|
52
|
+
try {
|
|
53
|
+
fs.mkdirSync(reviewsDir, { recursive: true });
|
|
54
|
+
} catch {
|
|
55
|
+
// Best-effort in restricted/sandbox environments
|
|
56
|
+
}
|
|
53
57
|
}
|
|
54
58
|
|
|
55
59
|
// ── Gate evaluation ───────────────────────────────────────────────────────
|