@tandryukha/aidemo 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/record-demo/SKILL.md +66 -0
- package/LICENSE +21 -0
- package/README.md +292 -0
- package/bin/aidemo.mjs +53 -0
- package/docs/AUTHORING.md +728 -0
- package/package.json +47 -0
- package/src/caption-render.ts +69 -0
- package/src/captions.ts +441 -0
- package/src/capture.ts +574 -0
- package/src/cards.ts +67 -0
- package/src/cli.ts +646 -0
- package/src/compose.ts +937 -0
- package/src/config.ts +222 -0
- package/src/cursor-overlay.ts +186 -0
- package/src/cursor.ts +134 -0
- package/src/distribute.ts +713 -0
- package/src/embed.ts +202 -0
- package/src/ffmpeg.ts +83 -0
- package/src/gif.ts +65 -0
- package/src/golden.ts +109 -0
- package/src/i18n.ts +86 -0
- package/src/mcp/jobs.ts +285 -0
- package/src/mcp/server.ts +1016 -0
- package/src/music.ts +153 -0
- package/src/params.ts +134 -0
- package/src/player.ts +1173 -0
- package/src/project.ts +287 -0
- package/src/recorder.ts +320 -0
- package/src/starter.ts +141 -0
- package/src/stills.ts +121 -0
- package/src/types.ts +660 -0
- package/src/util.ts +171 -0
- package/src/variants.ts +117 -0
- package/src/voice-local.ts +230 -0
- package/src/voice.ts +405 -0
- package/src/zoom.ts +183 -0
package/src/voice.ts
ADDED
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
import OpenAI from "openai";
|
|
2
|
+
import { promises as fs } from "node:fs";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import type { Storyboard, VoiceManifest, VoicePlan } from "./types.js";
|
|
5
|
+
import { VoiceManifestSchema } from "./types.js";
|
|
6
|
+
import { Project } from "./project.js";
|
|
7
|
+
import {
|
|
8
|
+
requireOpenAiKey,
|
|
9
|
+
openAiBaseUrl,
|
|
10
|
+
TTS_MODEL,
|
|
11
|
+
ttsProvider,
|
|
12
|
+
kokoroDtype,
|
|
13
|
+
requireElevenLabsKey,
|
|
14
|
+
ELEVENLABS_MODEL,
|
|
15
|
+
ELEVENLABS_VOICE,
|
|
16
|
+
explainAudioEndpointError,
|
|
17
|
+
} from "./config.js";
|
|
18
|
+
import { LocalVoiceProvider } from "./voice-local.js";
|
|
19
|
+
import { runFfmpeg, probeDurationMs } from "./ffmpeg.js";
|
|
20
|
+
import {
|
|
21
|
+
ensureDir,
|
|
22
|
+
writeJson,
|
|
23
|
+
readJson,
|
|
24
|
+
exists,
|
|
25
|
+
log,
|
|
26
|
+
ok,
|
|
27
|
+
step,
|
|
28
|
+
CanceledError,
|
|
29
|
+
type SceneProgress,
|
|
30
|
+
} from "./util.js";
|
|
31
|
+
import { dirname } from "node:path";
|
|
32
|
+
|
|
33
|
+
/** Silence inserted after each scene in the narration track. */
|
|
34
|
+
const GAP_MS = 300;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Swap-in point for other TTS backends (local Piper, …). The pipeline only
|
|
38
|
+
* depends on this interface, never on a concrete provider. Selection is via
|
|
39
|
+
* AIDEMO_TTS_PROVIDER (see config.ts); OpenAI remains the default.
|
|
40
|
+
*/
|
|
41
|
+
export interface VoiceProvider {
|
|
42
|
+
synthesize(input: {
|
|
43
|
+
text: string;
|
|
44
|
+
plan: VoicePlan;
|
|
45
|
+
}): Promise<Buffer>;
|
|
46
|
+
/** Release provider resources (e.g. the local provider's ONNX session).
|
|
47
|
+
* Called by generateVoice on providers it constructed itself. */
|
|
48
|
+
dispose?(): Promise<void>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export class OpenAIVoiceProvider implements VoiceProvider {
|
|
52
|
+
private client: OpenAI;
|
|
53
|
+
constructor() {
|
|
54
|
+
// baseURL undefined → api.openai.com; set → any OpenAI-compatible server.
|
|
55
|
+
this.client = new OpenAI({ apiKey: requireOpenAiKey(), baseURL: openAiBaseUrl() });
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async synthesize({ text, plan }: { text: string; plan: VoicePlan }): Promise<Buffer> {
|
|
59
|
+
const instructions = buildInstructions(plan);
|
|
60
|
+
const res = await this.client.audio.speech
|
|
61
|
+
.create({
|
|
62
|
+
model: TTS_MODEL(),
|
|
63
|
+
voice: plan.voiceId as OpenAI.Audio.SpeechCreateParams["voice"],
|
|
64
|
+
input: text,
|
|
65
|
+
instructions,
|
|
66
|
+
response_format: "mp3",
|
|
67
|
+
})
|
|
68
|
+
.catch((err) => explainAudioEndpointError(err, "tts"));
|
|
69
|
+
return Buffer.from(await res.arrayBuffer());
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export class ElevenLabsVoiceProvider implements VoiceProvider {
|
|
74
|
+
private apiKey: string;
|
|
75
|
+
private warnedInstructions = false;
|
|
76
|
+
constructor() {
|
|
77
|
+
this.apiKey = requireElevenLabsKey();
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async synthesize({ text, plan }: { text: string; plan: VoicePlan }): Promise<Buffer> {
|
|
81
|
+
if (plan.instructions && !this.warnedInstructions) {
|
|
82
|
+
this.warnedInstructions = true;
|
|
83
|
+
log(
|
|
84
|
+
"⚠ voice.instructions is a gpt-4o-mini-tts steering prompt — ElevenLabs ignores it"
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
// "marin" is the schema default (an OpenAI voice name); anything else the
|
|
88
|
+
// author chose deliberately and is passed to ElevenLabs verbatim.
|
|
89
|
+
const voiceId = plan.voiceId === "marin" ? ELEVENLABS_VOICE() : plan.voiceId;
|
|
90
|
+
const body: Record<string, unknown> = { text, model_id: ELEVENLABS_MODEL() };
|
|
91
|
+
if (plan.speed && plan.speed !== 1) {
|
|
92
|
+
// Storyboards allow 0.5–2; ElevenLabs accepts 0.7–1.2.
|
|
93
|
+
body.voice_settings = { speed: Math.min(1.2, Math.max(0.7, plan.speed)) };
|
|
94
|
+
}
|
|
95
|
+
const res = await fetch(
|
|
96
|
+
`https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(voiceId)}?output_format=mp3_44100_128`,
|
|
97
|
+
{
|
|
98
|
+
method: "POST",
|
|
99
|
+
headers: { "xi-api-key": this.apiKey, "content-type": "application/json" },
|
|
100
|
+
body: JSON.stringify(body),
|
|
101
|
+
}
|
|
102
|
+
);
|
|
103
|
+
if (!res.ok) {
|
|
104
|
+
const detail = await res.text().catch(() => "");
|
|
105
|
+
throw new Error(
|
|
106
|
+
`ElevenLabs TTS failed for voice "${voiceId}" (HTTP ${res.status}): ${detail.slice(0, 300)}`
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
return Buffer.from(await res.arrayBuffer());
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Fold the speed hint into the steering instructions (gpt-4o-mini-tts is prompt-steered). */
|
|
114
|
+
function buildInstructions(plan: VoicePlan): string {
|
|
115
|
+
const parts: string[] = [];
|
|
116
|
+
if (plan.instructions) parts.push(plan.instructions);
|
|
117
|
+
if (plan.speed && plan.speed > 1.05) parts.push("Speak at a brisk, energetic pace.");
|
|
118
|
+
else if (plan.speed && plan.speed < 0.95) parts.push("Speak slowly and deliberately.");
|
|
119
|
+
return parts.join(" ") || "Clear, friendly, natural narration.";
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function planFor(storyboard: Storyboard, sceneVoice?: VoicePlan): VoicePlan {
|
|
123
|
+
return {
|
|
124
|
+
voiceId: sceneVoice?.voiceId ?? storyboard.voice?.voiceId ?? "marin",
|
|
125
|
+
instructions: sceneVoice?.instructions ?? storyboard.voice?.instructions,
|
|
126
|
+
speed: sceneVoice?.speed ?? storyboard.voice?.speed,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Identity of a scene's audio = its narration + the resolved voice plan (+ the
|
|
132
|
+
* TTS provider when non-default; for the local provider that includes the
|
|
133
|
+
* model quantization; + the language code for a multi-language render). The
|
|
134
|
+
* provider and lang are folded in only when non-default so every pre-existing
|
|
135
|
+
* manifest hash stays valid — but switching providers (or requantizing the
|
|
136
|
+
* local model, or rendering a different language) still invalidates instead of
|
|
137
|
+
* silently reusing the wrong audio. (Per-language audio also lives in its own
|
|
138
|
+
* audio/<lang>/ dir, so caches can't physically collide; hashing the lang is
|
|
139
|
+
* belt-and-suspenders for two languages that happen to share narration text.)
|
|
140
|
+
*/
|
|
141
|
+
function voiceHash(
|
|
142
|
+
text: string,
|
|
143
|
+
plan: VoicePlan,
|
|
144
|
+
provider: string,
|
|
145
|
+
lang?: string
|
|
146
|
+
): string {
|
|
147
|
+
const base = provider === "openai" ? { text, plan } : { text, plan, provider };
|
|
148
|
+
const identity = lang ? { ...base, lang } : base;
|
|
149
|
+
return createHash("sha256")
|
|
150
|
+
.update(JSON.stringify(identity))
|
|
151
|
+
.digest("hex")
|
|
152
|
+
.slice(0, 16);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Local Kokoro ships English voices only (en-US + en-GB accents — verified from
|
|
157
|
+
* kokoro-js's voice table). A non-English `--lang` would still be read by an
|
|
158
|
+
* English voice and mispronounced, so warn (don't fail): the caller may have
|
|
159
|
+
* authored English-appropriate variants, or accept it for a smoke test. Real
|
|
160
|
+
* non-English speech needs AIDEMO_TTS_PROVIDER=elevenlabs (multilingual) or
|
|
161
|
+
* OpenAI TTS.
|
|
162
|
+
*/
|
|
163
|
+
function warnLocalLangCoverage(lang: string | undefined): void {
|
|
164
|
+
if (!lang) return;
|
|
165
|
+
const family = lang.toLowerCase().split(/[-_]/)[0];
|
|
166
|
+
if (family === "en") return;
|
|
167
|
+
log(
|
|
168
|
+
`⚠ local Kokoro TTS speaks English only (en-US + en-GB accents) — "${lang}" ` +
|
|
169
|
+
`narration will be read by an English voice and mispronounced. For genuine ` +
|
|
170
|
+
`${lang} speech, render with AIDEMO_TTS_PROVIDER=elevenlabs or OpenAI TTS.`
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export interface VoiceOptions extends SceneProgress {
|
|
175
|
+
provider?: VoiceProvider;
|
|
176
|
+
/** Regenerate only this scene id; reuse existing audio for the rest. */
|
|
177
|
+
only?: string;
|
|
178
|
+
/** Re-synthesize every scene even if its narration + voice are unchanged. */
|
|
179
|
+
force?: boolean;
|
|
180
|
+
/** Best-effort cancellation, checked between scenes (MCP job_cancel). */
|
|
181
|
+
signal?: AbortSignal;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Generates one narration clip per scene, assembles them (with inter-scene
|
|
186
|
+
* gaps) into narration.mp3, and writes voice.json — whose per-scene durations
|
|
187
|
+
* become the compose target length for each scene's video segment.
|
|
188
|
+
*
|
|
189
|
+
* Idempotent: a scene whose narration + voice plan hash matches the prior
|
|
190
|
+
* voice.json (and whose audio file still exists) is reused instead of re-voiced.
|
|
191
|
+
* This makes re-running the pipeline cheap and, crucially, doesn't silently
|
|
192
|
+
* discard a take you already approved. `force` re-voices everything; `only`
|
|
193
|
+
* targets a single scene (reusing the rest regardless of hash).
|
|
194
|
+
*/
|
|
195
|
+
export async function generateVoice(
|
|
196
|
+
project: Project,
|
|
197
|
+
storyboard: Storyboard,
|
|
198
|
+
opts: VoiceOptions = {}
|
|
199
|
+
): Promise<VoiceManifest> {
|
|
200
|
+
const providerName = ttsProvider();
|
|
201
|
+
// The audio identity key: local audio also changes with the quantization.
|
|
202
|
+
const providerKey =
|
|
203
|
+
providerName === "local" ? `local:${kokoroDtype()}` : providerName;
|
|
204
|
+
if (providerName === "local") warnLocalLangCoverage(project.lang);
|
|
205
|
+
const base = openAiBaseUrl();
|
|
206
|
+
step(
|
|
207
|
+
providerName === "elevenlabs"
|
|
208
|
+
? "Generating narration (ElevenLabs TTS)"
|
|
209
|
+
: providerName === "local"
|
|
210
|
+
? "Generating narration (local Kokoro-82M, in-process)"
|
|
211
|
+
: base
|
|
212
|
+
? `Generating narration (TTS @ ${base})`
|
|
213
|
+
: "Generating narration (OpenAI TTS)"
|
|
214
|
+
);
|
|
215
|
+
await ensureDir(dirname(project.narrationPath));
|
|
216
|
+
|
|
217
|
+
// Construct the provider lazily so a fully-cached run needs no API key
|
|
218
|
+
// (and, for local, doesn't import onnxruntime or touch the model cache).
|
|
219
|
+
let providerInst = opts.provider ?? null;
|
|
220
|
+
const provider = () =>
|
|
221
|
+
(providerInst ??=
|
|
222
|
+
providerName === "elevenlabs"
|
|
223
|
+
? new ElevenLabsVoiceProvider()
|
|
224
|
+
: providerName === "local"
|
|
225
|
+
? new LocalVoiceProvider()
|
|
226
|
+
: new OpenAIVoiceProvider());
|
|
227
|
+
|
|
228
|
+
// Prior manifest lets us reuse audio for unchanged scenes.
|
|
229
|
+
const prior = (await exists(project.voiceManifestPath))
|
|
230
|
+
? VoiceManifestSchema.safeParse(await readJson(project.voiceManifestPath))
|
|
231
|
+
: null;
|
|
232
|
+
const priorById = new Map(
|
|
233
|
+
prior?.success ? prior.data.scenes.map((s) => [s.id, s]) : []
|
|
234
|
+
);
|
|
235
|
+
|
|
236
|
+
const manifest: VoiceManifest = { gapMs: GAP_MS, scenes: [] };
|
|
237
|
+
let made = 0;
|
|
238
|
+
let reused = 0;
|
|
239
|
+
const sceneTotal = storyboard.scenes.length;
|
|
240
|
+
try {
|
|
241
|
+
for (let i = 0; i < storyboard.scenes.length; i++) {
|
|
242
|
+
const scene = storyboard.scenes[i];
|
|
243
|
+
opts.onSceneStart?.(scene.id, i, sceneTotal);
|
|
244
|
+
if (opts.signal?.aborted)
|
|
245
|
+
throw new CanceledError(`canceled before voicing scene ${scene.id}`);
|
|
246
|
+
const outPath = project.sceneAudioPath(scene.id);
|
|
247
|
+
const plan = planFor(storyboard, scene.voice);
|
|
248
|
+
const hash = voiceHash(scene.narration, plan, providerKey, project.lang);
|
|
249
|
+
const prev = priorById.get(scene.id);
|
|
250
|
+
const scoped = opts.only != null; // --scene mode
|
|
251
|
+
const targeted = scoped && scene.id === opts.only;
|
|
252
|
+
const unchanged = !opts.force && prev?.hash === hash;
|
|
253
|
+
const audioOk = await exists(outPath);
|
|
254
|
+
const reuse = audioOk && (scoped ? !targeted : unchanged);
|
|
255
|
+
|
|
256
|
+
if (reuse) {
|
|
257
|
+
const durationMs =
|
|
258
|
+
prev?.durationMs && prev.durationMs > 0
|
|
259
|
+
? prev.durationMs
|
|
260
|
+
: await probeDurationMs(outPath).catch(() => 0);
|
|
261
|
+
// In --scene mode keep whatever hash was recorded; otherwise it matched.
|
|
262
|
+
manifest.scenes.push({
|
|
263
|
+
id: scene.id,
|
|
264
|
+
file: outPath,
|
|
265
|
+
durationMs,
|
|
266
|
+
hash: scoped ? prev?.hash : hash,
|
|
267
|
+
});
|
|
268
|
+
log(`scene ${scene.id}: unchanged, reusing narration`);
|
|
269
|
+
reused++;
|
|
270
|
+
opts.onSceneComplete?.(scene.id, i, sceneTotal);
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
log(
|
|
275
|
+
`scene ${scene.id}: "${scene.narration.slice(0, 48)}${
|
|
276
|
+
scene.narration.length > 48 ? "…" : ""
|
|
277
|
+
}" (${plan.voiceId})`
|
|
278
|
+
);
|
|
279
|
+
const audio = await provider().synthesize({ text: scene.narration, plan });
|
|
280
|
+
await fs.writeFile(outPath, audio);
|
|
281
|
+
const durationMs = await probeDurationMs(outPath);
|
|
282
|
+
manifest.scenes.push({ id: scene.id, file: outPath, durationMs, hash });
|
|
283
|
+
made++;
|
|
284
|
+
opts.onSceneComplete?.(scene.id, i, sceneTotal);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
await assembleNarration(project, manifest);
|
|
288
|
+
await writeJson(project.voiceManifestPath, manifest);
|
|
289
|
+
const total = manifest.scenes.reduce((a, s) => a + s.durationMs + GAP_MS, 0);
|
|
290
|
+
ok(
|
|
291
|
+
`narration → ${project.narrationPath} (${manifest.scenes.length} scenes, ` +
|
|
292
|
+
`~${Math.round(total / 1000)}s; ${made} voiced, ${reused} reused)`
|
|
293
|
+
);
|
|
294
|
+
ok(`voice.json → ${project.voiceManifestPath}`);
|
|
295
|
+
// Existing captions were timed against the OLD narration; compose would burn
|
|
296
|
+
// mismatched text. Say so now, not after a wasted compose.
|
|
297
|
+
if (made > 0 && (await exists(project.captionsCuesPath))) {
|
|
298
|
+
log(`⚠ narration changed — captions are now stale; re-run: aidemo captions ${project.dir}`);
|
|
299
|
+
}
|
|
300
|
+
warnTargetLength(storyboard, manifest, total);
|
|
301
|
+
return manifest;
|
|
302
|
+
} finally {
|
|
303
|
+
// A provider we constructed is ours to clean up. The local provider holds
|
|
304
|
+
// an ONNX session that makes an abrupt process exit abort noisily.
|
|
305
|
+
if (!opts.provider && providerInst?.dispose) {
|
|
306
|
+
await providerInst.dispose().catch(() => {});
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** Whitespace word count — language-agnostic for the space-delimited scripts
|
|
312
|
+
* storyboards are authored in (including Estonian, Finnish, German, …). */
|
|
313
|
+
function countWords(text: string): number {
|
|
314
|
+
const trimmed = text.trim();
|
|
315
|
+
return trimmed ? trimmed.split(/\s+/).length : 0;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* `targetLengthSeconds` is an authoring budget nothing else in the pipeline
|
|
320
|
+
* checks — docs/AUTHORING.md's "≈2.5 words/sec" planning figure is
|
|
321
|
+
* English-centric, and non-English OpenAI TTS voices can read 30-40% slower
|
|
322
|
+
* (Estonian `marin` measured at ≈1.8 w/s: 131 planned words, budgeted at 52s,
|
|
323
|
+
* came out at 71.8s — see aidemo#15). voice.json is the first point the
|
|
324
|
+
* pipeline knows the REAL narration length, so warn here — before record +
|
|
325
|
+
* compose spend a full render finding out. Advisory only: never throws, never
|
|
326
|
+
* changes the exit code.
|
|
327
|
+
*/
|
|
328
|
+
function warnTargetLength(
|
|
329
|
+
storyboard: Storyboard,
|
|
330
|
+
manifest: VoiceManifest,
|
|
331
|
+
narrationTotalMs: number
|
|
332
|
+
): void {
|
|
333
|
+
const targetSec = storyboard.targetLengthSeconds;
|
|
334
|
+
if (targetSec == null) return;
|
|
335
|
+
|
|
336
|
+
const narrationSec = narrationTotalMs / 1000;
|
|
337
|
+
const introSec = (storyboard.intro?.durationMs ?? 0) / 1000;
|
|
338
|
+
const outroSec = (storyboard.outro?.durationMs ?? 0) / 1000;
|
|
339
|
+
const totalSec = narrationSec + introSec + outroSec;
|
|
340
|
+
|
|
341
|
+
const narrationById = new Map(storyboard.scenes.map((s) => [s.id, s.narration]));
|
|
342
|
+
const rows = manifest.scenes.map((s) => {
|
|
343
|
+
const words = countWords(narrationById.get(s.id) ?? "");
|
|
344
|
+
const sec = s.durationMs / 1000;
|
|
345
|
+
const wps = sec > 0 ? words / sec : 0;
|
|
346
|
+
return { id: s.id, words, sec, wps };
|
|
347
|
+
});
|
|
348
|
+
const totalWords = rows.reduce((a, r) => a + r.words, 0);
|
|
349
|
+
// Pace over the narration track itself (scenes + inter-scene gaps), which is
|
|
350
|
+
// what the author's word count actually bought them — cards are fixed
|
|
351
|
+
// overhead, not narration, so they're excluded from the rate.
|
|
352
|
+
const pace = narrationSec > 0 ? totalWords / narrationSec : 0;
|
|
353
|
+
|
|
354
|
+
const parts = [`narration ${narrationSec.toFixed(1)}s`];
|
|
355
|
+
if (introSec > 0) parts.push(`intro ${introSec.toFixed(1)}s`);
|
|
356
|
+
if (outroSec > 0) parts.push(`outro ${outroSec.toFixed(1)}s`);
|
|
357
|
+
const summary =
|
|
358
|
+
parts.length > 1 ? `${parts.join(" + ")} = ${totalSec.toFixed(1)}s` : parts[0];
|
|
359
|
+
|
|
360
|
+
const TOLERANCE = 1.05; // 5% slack before it's worth interrupting the author
|
|
361
|
+
if (totalSec > targetSec * TOLERANCE) {
|
|
362
|
+
const budgetWords = Math.round(targetSec * pace);
|
|
363
|
+
log(
|
|
364
|
+
`⚠ ${summary} exceeds targetLengthSeconds ${targetSec} ` +
|
|
365
|
+
`(measured pace ${pace.toFixed(1)} w/s — budget ≈ ${budgetWords} words)`
|
|
366
|
+
);
|
|
367
|
+
log(` ${"scene".padEnd(10)}${"words".padStart(7)}${"sec".padStart(7)}${"w/s".padStart(6)}`);
|
|
368
|
+
for (const r of rows) {
|
|
369
|
+
log(
|
|
370
|
+
` ${r.id.padEnd(10)}${String(r.words).padStart(7)}${r.sec.toFixed(1).padStart(7)}${r.wps.toFixed(1).padStart(6)}`
|
|
371
|
+
);
|
|
372
|
+
}
|
|
373
|
+
} else if (totalSec < targetSec) {
|
|
374
|
+
log(
|
|
375
|
+
`${summary} is under targetLengthSeconds ${targetSec} ` +
|
|
376
|
+
`(measured pace ${pace.toFixed(1)} w/s) — room to add narration if useful`
|
|
377
|
+
);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/** Concatenate scene clips with a trailing gap after each, in one ffmpeg pass. */
|
|
382
|
+
async function assembleNarration(project: Project, manifest: VoiceManifest): Promise<void> {
|
|
383
|
+
const gapSec = (manifest.gapMs / 1000).toFixed(3);
|
|
384
|
+
const inputs: string[] = [];
|
|
385
|
+
const filters: string[] = [];
|
|
386
|
+
const labels: string[] = [];
|
|
387
|
+
manifest.scenes.forEach((s, i) => {
|
|
388
|
+
inputs.push("-i", s.file);
|
|
389
|
+
filters.push(`[${i}:a]aresample=44100,apad=pad_dur=${gapSec}[a${i}]`);
|
|
390
|
+
labels.push(`[a${i}]`);
|
|
391
|
+
});
|
|
392
|
+
const n = manifest.scenes.length;
|
|
393
|
+
const filterComplex =
|
|
394
|
+
filters.join(";") + ";" + labels.join("") + `concat=n=${n}:v=0:a=1[out]`;
|
|
395
|
+
await runFfmpeg([
|
|
396
|
+
...inputs,
|
|
397
|
+
"-filter_complex",
|
|
398
|
+
filterComplex,
|
|
399
|
+
"-map",
|
|
400
|
+
"[out]",
|
|
401
|
+
"-ac",
|
|
402
|
+
"1",
|
|
403
|
+
project.narrationPath,
|
|
404
|
+
]);
|
|
405
|
+
}
|
package/src/zoom.ts
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import type { ZoomConfig } from "./types.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Screen-Studio-style auto-zoom, rendered at compose time with ffmpeg's
|
|
5
|
+
* `zoompan`. The player records focus events (clicks / typing / explicit
|
|
6
|
+
* `focus` actions); compose maps them into final-video time and this module
|
|
7
|
+
* turns them into eased zoom choreography:
|
|
8
|
+
*
|
|
9
|
+
* idle 1.0x → ease in to `scale` just before the click → hold → ease back
|
|
10
|
+
*
|
|
11
|
+
* Consecutive focus points closer together than a hold+ease don't bounce out
|
|
12
|
+
* and in again — the camera stays zoomed and *pans* between them.
|
|
13
|
+
*
|
|
14
|
+
* Everything is emitted as piecewise smoothstep ffmpeg expressions over output
|
|
15
|
+
* time (`on/fps`), one channel each for zoom, center-x and center-y.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** A focus point in FINAL video time, coordinates in output-video pixels. */
|
|
19
|
+
export interface ZoomEvent {
|
|
20
|
+
tMs: number;
|
|
21
|
+
x: number;
|
|
22
|
+
y: number;
|
|
23
|
+
scale?: number;
|
|
24
|
+
holdMs?: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
interface Key {
|
|
28
|
+
t: number; // seconds
|
|
29
|
+
v: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Append a keyframe, nudging time forward to keep the channel monotonic. */
|
|
33
|
+
function pushKey(keys: Key[], t: number, v: number): void {
|
|
34
|
+
const last = keys[keys.length - 1];
|
|
35
|
+
if (last && t <= last.t) t = last.t + 0.02;
|
|
36
|
+
keys.push({ t, v });
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const num = (n: number): string => {
|
|
40
|
+
const s = n.toFixed(4).replace(/\.?0+$/, "");
|
|
41
|
+
return s === "" || s === "-" ? "0" : s;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Piecewise expression over `T` (a time expression in seconds): holds the
|
|
46
|
+
* first value before the first key, smoothsteps between keys, holds the last
|
|
47
|
+
* value after the last key.
|
|
48
|
+
*/
|
|
49
|
+
function piecewise(keys: Key[], T: string): string {
|
|
50
|
+
let expr = num(keys[keys.length - 1].v);
|
|
51
|
+
for (let i = keys.length - 2; i >= 0; i--) {
|
|
52
|
+
const a = keys[i];
|
|
53
|
+
const b = keys[i + 1];
|
|
54
|
+
let seg: string;
|
|
55
|
+
if (a.v === b.v || b.t - a.t < 1e-6) {
|
|
56
|
+
seg = num(a.v);
|
|
57
|
+
} else {
|
|
58
|
+
const p = `((${T}-${num(a.t)})/${num(b.t - a.t)})`;
|
|
59
|
+
seg = `(${num(a.v)}+(${num(b.v - a.v)})*${p}*${p}*(3-2*${p}))`;
|
|
60
|
+
}
|
|
61
|
+
expr = `if(lt(${T},${num(b.t)}),${seg},${expr})`;
|
|
62
|
+
}
|
|
63
|
+
// Guard the span before the first key (p would be negative in seg 0).
|
|
64
|
+
return `if(lt(${T},${num(keys[0].t)}),${num(keys[0].v)},${expr})`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Build the zoom/pan keyframes for all three channels.
|
|
69
|
+
* Exported for tests/debugging; buildZoomFilter is the real entry point.
|
|
70
|
+
*/
|
|
71
|
+
export function planZoom(
|
|
72
|
+
events: ZoomEvent[],
|
|
73
|
+
cfg: ZoomConfig,
|
|
74
|
+
outW: number,
|
|
75
|
+
outH: number,
|
|
76
|
+
durMs: number
|
|
77
|
+
): { z: Key[]; cx: Key[]; cy: Key[] } | null {
|
|
78
|
+
const dur = durMs / 1000;
|
|
79
|
+
const ease = Math.max(0.15, cfg.easeMs / 1000);
|
|
80
|
+
const sorted = [...events]
|
|
81
|
+
.filter((e) => e.tMs >= 0 && e.tMs < durMs)
|
|
82
|
+
.sort((a, b) => a.tMs - b.tMs);
|
|
83
|
+
if (sorted.length === 0) return null;
|
|
84
|
+
|
|
85
|
+
const z: Key[] = [{ t: 0, v: 1 }];
|
|
86
|
+
const cx: Key[] = [{ t: 0, v: outW / 2 }];
|
|
87
|
+
const cy: Key[] = [{ t: 0, v: outH / 2 }];
|
|
88
|
+
|
|
89
|
+
let zoomed = false;
|
|
90
|
+
let lastZ = 1;
|
|
91
|
+
let lastX = outW / 2;
|
|
92
|
+
let lastY = outH / 2;
|
|
93
|
+
|
|
94
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
95
|
+
const e = sorted[i];
|
|
96
|
+
const t = e.tMs / 1000;
|
|
97
|
+
const scale = Math.min(4, Math.max(1.05, e.scale ?? cfg.scale));
|
|
98
|
+
const hold = Math.max(0.2, (e.holdMs ?? cfg.holdMs) / 1000);
|
|
99
|
+
const ex = Math.min(outW, Math.max(0, e.x));
|
|
100
|
+
const ey = Math.min(outH, Math.max(0, e.y));
|
|
101
|
+
|
|
102
|
+
if (!zoomed) {
|
|
103
|
+
// Ease in, starting a beat before the click so the click lands zoomed.
|
|
104
|
+
const inStart = Math.max(0, t - ease * 0.35);
|
|
105
|
+
const inEnd = inStart + ease;
|
|
106
|
+
pushKey(z, inStart, 1);
|
|
107
|
+
pushKey(z, inEnd, scale);
|
|
108
|
+
// While z=1 the center is invisible — snap it to the new focus point.
|
|
109
|
+
pushKey(cx, inStart, ex);
|
|
110
|
+
pushKey(cy, inStart, ey);
|
|
111
|
+
} else {
|
|
112
|
+
// Already zoomed: pan (and rescale if needed) to the new focus point.
|
|
113
|
+
const panDur = Math.min(ease, 0.8);
|
|
114
|
+
const panStart = Math.max(0, t - panDur * 0.5);
|
|
115
|
+
const panEnd = panStart + panDur;
|
|
116
|
+
pushKey(cx, panStart, lastX);
|
|
117
|
+
pushKey(cx, panEnd, ex);
|
|
118
|
+
pushKey(cy, panStart, lastY);
|
|
119
|
+
pushKey(cy, panEnd, ey);
|
|
120
|
+
if (scale !== lastZ) {
|
|
121
|
+
pushKey(z, panStart, lastZ);
|
|
122
|
+
pushKey(z, panEnd, scale);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Zoom out after the hold — unless the next focus point arrives before
|
|
127
|
+
// we'd finish bouncing out; then stay zoomed and let the pan handle it.
|
|
128
|
+
const holdEnd = t + hold;
|
|
129
|
+
const next = sorted[i + 1];
|
|
130
|
+
const nextInStart = next ? next.tMs / 1000 - ease * 0.35 : Infinity;
|
|
131
|
+
if (nextInStart < holdEnd + ease) {
|
|
132
|
+
zoomed = true;
|
|
133
|
+
} else {
|
|
134
|
+
const outStart = Math.min(holdEnd, Math.max(0, dur - ease));
|
|
135
|
+
pushKey(z, outStart, scale);
|
|
136
|
+
pushKey(z, outStart + ease, 1);
|
|
137
|
+
// Pin the center through the zoom-out so it doesn't drift mid-move.
|
|
138
|
+
pushKey(cx, outStart, ex);
|
|
139
|
+
pushKey(cy, outStart, ey);
|
|
140
|
+
zoomed = false;
|
|
141
|
+
}
|
|
142
|
+
lastZ = scale;
|
|
143
|
+
lastX = ex;
|
|
144
|
+
lastY = ey;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return { z, cx, cy };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Full `-vf` filter string applying the auto-zoom to a CFR video of
|
|
152
|
+
* outW x outH at `fps`, or null when there is nothing to zoom. For SD-ish
|
|
153
|
+
* sources the frame is upscaled 2x first so zoompan's integer-pixel crop
|
|
154
|
+
* doesn't shimmer and a 1.5x zoom stays sharp.
|
|
155
|
+
*/
|
|
156
|
+
export function buildZoomFilter(
|
|
157
|
+
events: ZoomEvent[],
|
|
158
|
+
cfg: ZoomConfig,
|
|
159
|
+
outW: number,
|
|
160
|
+
outH: number,
|
|
161
|
+
durMs: number,
|
|
162
|
+
fps: number
|
|
163
|
+
): string | null {
|
|
164
|
+
const plan = planZoom(events, cfg, outW, outH, durMs);
|
|
165
|
+
if (!plan) return null;
|
|
166
|
+
|
|
167
|
+
const up = outW < 1600 ? 2 : 1;
|
|
168
|
+
const T = `on/${fps}`;
|
|
169
|
+
const zExpr = piecewise(plan.z, T);
|
|
170
|
+
const scaleKeys = (keys: Key[]): Key[] =>
|
|
171
|
+
keys.map((k) => ({ t: k.t, v: k.v * up }));
|
|
172
|
+
const cxExpr = piecewise(scaleKeys(plan.cx), T);
|
|
173
|
+
const cyExpr = piecewise(scaleKeys(plan.cy), T);
|
|
174
|
+
|
|
175
|
+
const xExpr = `clip(${cxExpr}-(iw/zoom)/2,0,iw-iw/zoom)`;
|
|
176
|
+
const yExpr = `clip(${cyExpr}-(ih/zoom)/2,0,ih-ih/zoom)`;
|
|
177
|
+
const pre = up > 1 ? `scale=${outW * up}:${outH * up}:flags=lanczos,` : "";
|
|
178
|
+
return (
|
|
179
|
+
pre +
|
|
180
|
+
`zoompan=z='${zExpr}':x='${xExpr}':y='${yExpr}'` +
|
|
181
|
+
`:d=1:s=${outW}x${outH}:fps=${fps}`
|
|
182
|
+
);
|
|
183
|
+
}
|