@tandryukha/aidemo 0.8.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/record-demo/SKILL.md +5 -2
- package/README.md +16 -4
- package/docs/AUTHORING.md +609 -45
- package/package.json +3 -2
- package/src/anchors.ts +128 -0
- package/src/attention.ts +365 -0
- package/src/caption-render.ts +17 -8
- package/src/captions.ts +70 -6
- package/src/capture.ts +113 -0
- package/src/cards.ts +15 -6
- package/src/cli.ts +580 -22
- package/src/compose.ts +1006 -93
- package/src/cursor-overlay.ts +7 -113
- package/src/cursor.ts +9 -3
- package/src/distribute.ts +37 -4
- package/src/expr.ts +130 -0
- package/src/ffmpeg.ts +156 -0
- package/src/frame.ts +205 -0
- package/src/frames.ts +159 -0
- package/src/golden.ts +27 -0
- package/src/guide.ts +102 -0
- package/src/i18n.ts +7 -4
- package/src/import-trace.ts +629 -0
- package/src/inspect.ts +307 -0
- package/src/lint.ts +601 -0
- package/src/login.ts +143 -0
- package/src/mcp/jobs.ts +4 -1
- package/src/mcp/server.ts +403 -26
- package/src/player.ts +975 -32
- package/src/profile.ts +128 -0
- package/src/project.ts +12 -2
- package/src/recorder.ts +252 -15
- package/src/setup.ts +248 -0
- package/src/starter.ts +244 -0
- package/src/stills.ts +15 -4
- package/src/types.ts +643 -5
- package/src/variants.ts +1 -1
- package/src/voice-local.ts +7 -1
- package/src/voice.ts +60 -1
- package/src/walkthrough.ts +250 -0
- package/src/zoom.ts +85 -39
package/src/compose.ts
CHANGED
|
@@ -1,22 +1,63 @@
|
|
|
1
1
|
import { promises as fs } from "node:fs";
|
|
2
|
-
import { resolve, isAbsolute } from "node:path";
|
|
2
|
+
import { resolve, isAbsolute, relative } from "node:path";
|
|
3
3
|
import { Project } from "./project.js";
|
|
4
|
-
import type {
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
import type {
|
|
5
|
+
Storyboard,
|
|
6
|
+
Timeline,
|
|
7
|
+
IdleSpan,
|
|
8
|
+
Card,
|
|
9
|
+
Output,
|
|
10
|
+
Loudness,
|
|
11
|
+
ComposeReport,
|
|
12
|
+
ComposeSceneReport,
|
|
13
|
+
ComposeWarning,
|
|
14
|
+
} from "./types.js";
|
|
15
|
+
import {
|
|
16
|
+
TimelineSchema,
|
|
17
|
+
VoiceManifestSchema,
|
|
18
|
+
LoudnessSchema,
|
|
19
|
+
HoldSchema,
|
|
20
|
+
AutoIdleSchema,
|
|
21
|
+
} from "./types.js";
|
|
22
|
+
import { sceneWordsFor, type Cue } from "./captions.js";
|
|
23
|
+
import { wordStartMs } from "./anchors.js";
|
|
7
24
|
import { renderCaptionPngs } from "./caption-render.js";
|
|
8
25
|
import { renderCardPng } from "./cards.js";
|
|
9
26
|
import { renderCursorPng } from "./cursor.js";
|
|
10
|
-
import { buildZoomFilter, type ZoomEvent } from "./zoom.js";
|
|
27
|
+
import { buildZoomFilter, clampScaleForWidth, type ZoomEvent } from "./zoom.js";
|
|
11
28
|
import {
|
|
12
29
|
buildCursorFilter,
|
|
13
30
|
type CursorPoint,
|
|
14
31
|
type HideWindow,
|
|
15
32
|
} from "./cursor-overlay.js";
|
|
16
|
-
import {
|
|
33
|
+
import {
|
|
34
|
+
renderAttentionPngs,
|
|
35
|
+
renderKeyChipPngs,
|
|
36
|
+
renderClickRingPngs,
|
|
37
|
+
buildRedactFilter,
|
|
38
|
+
REDACT_BATCH,
|
|
39
|
+
KEY_CHIP_MS,
|
|
40
|
+
DEFAULT_ACCENT,
|
|
41
|
+
type OverlayItem,
|
|
42
|
+
type PlacedAttention,
|
|
43
|
+
type PlacedKey,
|
|
44
|
+
type PlacedClick,
|
|
45
|
+
type PlacedRedact,
|
|
46
|
+
} from "./attention.js";
|
|
47
|
+
import { renderFramePng } from "./frame.js";
|
|
48
|
+
import { OUTPUT_PRESETS, type CaptionsConfig } from "./types.js";
|
|
49
|
+
import {
|
|
50
|
+
runFfmpeg,
|
|
51
|
+
probeDurationMs,
|
|
52
|
+
probeFlatTailMs,
|
|
53
|
+
probeVideoDims,
|
|
54
|
+
probeFreezeSpans,
|
|
55
|
+
type FreezeSpan,
|
|
56
|
+
} from "./ffmpeg.js";
|
|
17
57
|
import {
|
|
18
58
|
ensureDir,
|
|
19
59
|
readJson,
|
|
60
|
+
writeJson,
|
|
20
61
|
exists,
|
|
21
62
|
log,
|
|
22
63
|
ok,
|
|
@@ -43,14 +84,26 @@ export async function compose(
|
|
|
43
84
|
project: Project,
|
|
44
85
|
storyboard: Storyboard,
|
|
45
86
|
opts: SceneProgress = {}
|
|
46
|
-
): Promise<
|
|
87
|
+
): Promise<ComposeReport> {
|
|
47
88
|
step("Composing final video (ffmpeg)");
|
|
48
89
|
|
|
49
90
|
const timeline = TimelineSchema.parse(await readJson(project.timelinePath));
|
|
50
91
|
const voice = VoiceManifestSchema.parse(
|
|
51
92
|
await readJson(project.voiceManifestPath)
|
|
52
93
|
);
|
|
53
|
-
|
|
94
|
+
// Structured twin of the log lines below — see ComposeReportSchema.
|
|
95
|
+
const warnings: ComposeWarning[] = [];
|
|
96
|
+
const sceneReports: ComposeSceneReport[] = [];
|
|
97
|
+
const warn = (code: string, message: string, scene?: string): void => {
|
|
98
|
+
warnings.push({ code, message, ...(scene ? { scene } : {}) });
|
|
99
|
+
};
|
|
100
|
+
const hold = HoldSchema.parse(storyboard.hold ?? {});
|
|
101
|
+
if (await warnStaleCaptions(project)) {
|
|
102
|
+
warn(
|
|
103
|
+
"stale-captions",
|
|
104
|
+
`narration.mp3 is newer than the caption files — re-run: aidemo captions ${project.dir}`
|
|
105
|
+
);
|
|
106
|
+
}
|
|
54
107
|
const tmp = project.composeTmpDir;
|
|
55
108
|
await fs.rm(tmp, { recursive: true, force: true });
|
|
56
109
|
await ensureDir(tmp);
|
|
@@ -63,6 +116,41 @@ export async function compose(
|
|
|
63
116
|
const { width: outW, height: outH } = await probeVideoDims(rawVideo);
|
|
64
117
|
const pxScale = outW / storyboard.video.width;
|
|
65
118
|
|
|
119
|
+
// --- autoIdle / multi-source plumbing -------------------------------------
|
|
120
|
+
// A resumed take splices scenes from more than one raw file; every source
|
|
121
|
+
// must share the main take's pixel size or all the overlay math (captions,
|
|
122
|
+
// zoom, cursor — all scaled by pxScale) silently misaligns on those scenes.
|
|
123
|
+
const dimsChecked = new Set<string>([rawVideo]);
|
|
124
|
+
const assertSourceDims = async (file: string): Promise<void> => {
|
|
125
|
+
if (dimsChecked.has(file)) return;
|
|
126
|
+
dimsChecked.add(file);
|
|
127
|
+
const d = await probeVideoDims(file);
|
|
128
|
+
if (d.width !== outW || d.height !== outH) {
|
|
129
|
+
throw new Error(
|
|
130
|
+
`resumed take mixes recordings of different sizes: ${relative(project.dir, file)} is ` +
|
|
131
|
+
`${d.width}x${d.height}, the current take is ${outW}x${outH} — re-record the whole demo ` +
|
|
132
|
+
`(a resume must use the same capture mode and viewport as the take it continues)`
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
const autoIdleCfg = AutoIdleSchema.parse(
|
|
137
|
+
typeof storyboard.autoIdle === "object" ? storyboard.autoIdle : {}
|
|
138
|
+
);
|
|
139
|
+
const autoIdleOn =
|
|
140
|
+
storyboard.autoIdle === true ||
|
|
141
|
+
(typeof storyboard.autoIdle === "object" && storyboard.autoIdle.enabled !== false);
|
|
142
|
+
const autoIdleFor = (scene: { autoIdle?: boolean } | undefined): boolean =>
|
|
143
|
+
scene?.autoIdle ?? autoIdleOn;
|
|
144
|
+
const freezeCache = new Map<string, Promise<FreezeSpan[]>>();
|
|
145
|
+
const freezeSpansFor = (file: string): Promise<FreezeSpan[]> => {
|
|
146
|
+
let p = freezeCache.get(file);
|
|
147
|
+
if (!p) {
|
|
148
|
+
p = probeFreezeSpans(file, autoIdleCfg.minMs ?? 1500, autoIdleCfg.noise ?? 0.003);
|
|
149
|
+
freezeCache.set(file, p);
|
|
150
|
+
}
|
|
151
|
+
return p;
|
|
152
|
+
};
|
|
153
|
+
|
|
66
154
|
// Map scene id -> narration duration; compose target = narration + gap.
|
|
67
155
|
const voiceById = new Map(voice.scenes.map((s) => [s.id, s.durationMs]));
|
|
68
156
|
const sceneById = new Map(storyboard.scenes.map((s) => [s.id, s]));
|
|
@@ -80,6 +168,18 @@ export async function compose(
|
|
|
80
168
|
const cursorPts: CursorPoint[] = [];
|
|
81
169
|
const hideWindows: HideWindow[] = [];
|
|
82
170
|
let cursorSampleCount = 0;
|
|
171
|
+
// Attention layer accumulators (all opt-in; empty for a plain storyboard).
|
|
172
|
+
const accent = storyboard.attention?.color ?? storyboard.brand?.accent ?? DEFAULT_ACCENT;
|
|
173
|
+
const attentionPlaced: PlacedAttention[] = [];
|
|
174
|
+
const keysPlaced: PlacedKey[] = [];
|
|
175
|
+
const clicksPlaced: PlacedClick[] = [];
|
|
176
|
+
const redactPlaced: PlacedRedact[] = [];
|
|
177
|
+
const clickRings = !!(storyboard.attention?.clicks && cursorCfg);
|
|
178
|
+
/** Per-scene caption placement windows (content ms, before the intro shift). */
|
|
179
|
+
const captionWindows: Array<{ a: number; b: number; position: "top" | "bottom" }> = [];
|
|
180
|
+
// Dot cursors are centered on the point; the arrow PNG's tip is its top-left.
|
|
181
|
+
const dotOffset =
|
|
182
|
+
cursorCfg?.style === "dot" ? Math.round(16 * pxScale * (cursorCfg.scale ?? 1)) : 0;
|
|
83
183
|
|
|
84
184
|
const sceneVideos: string[] = [];
|
|
85
185
|
const sceneTotal = timeline.scenes.length;
|
|
@@ -90,22 +190,133 @@ export async function compose(
|
|
|
90
190
|
const targetMs = narrMs + voice.gapMs;
|
|
91
191
|
if (targetMs <= 0) {
|
|
92
192
|
log(`scene ${tl.id}: no narration, skipping`);
|
|
193
|
+
warn("scene-no-narration", `scene ${tl.id} has no narration audio and was skipped`, tl.id);
|
|
93
194
|
opts.onSceneComplete?.(tl.id, i, sceneTotal);
|
|
94
195
|
continue;
|
|
95
196
|
}
|
|
96
197
|
|
|
97
|
-
|
|
198
|
+
let blankTrimMs = 0;
|
|
199
|
+
// A resumed take's reused scenes live in an earlier raw file with its own lead-in.
|
|
200
|
+
const leadInMs = tl.leadInMs ?? timeline.leadInMs;
|
|
201
|
+
const sceneRaw = tl.source ? resolve(project.dir, tl.source) : rawVideo;
|
|
202
|
+
await assertSourceDims(sceneRaw);
|
|
203
|
+
// `autoIdle`: motionless spans of the raw take count as idle too, so dead
|
|
204
|
+
// air the storyboard never annotated gets the same cap as a `waitFor`.
|
|
205
|
+
const autoSpans = autoIdleFor(sceneById.get(tl.id))
|
|
206
|
+
? sceneAutoIdle(await freezeSpansFor(sceneRaw), tl, leadInMs)
|
|
207
|
+
: [];
|
|
208
|
+
if (autoSpans.length) {
|
|
209
|
+
const dead = autoSpans.reduce((a, sp) => a + (sp.endMs - sp.startMs), 0);
|
|
210
|
+
log(`scene ${tl.id}: autoIdle found ${autoSpans.length} motionless span(s), ${dead}ms`);
|
|
211
|
+
}
|
|
212
|
+
const keeps = keepIntervals(
|
|
213
|
+
autoSpans.length ? { ...tl, idleSpans: mergeIdle([...tl.idleSpans, ...autoSpans]) } : tl,
|
|
214
|
+
leadInMs
|
|
215
|
+
);
|
|
98
216
|
const rawSegPath = resolve(tmp, `scene-${i}-raw.mp4`);
|
|
99
|
-
await extractAndConcat(
|
|
217
|
+
await extractAndConcat(sceneRaw, keeps, rawSegPath, tmp, i);
|
|
100
218
|
|
|
101
|
-
|
|
219
|
+
let srcMs = await probeDurationMs(rawSegPath);
|
|
220
|
+
// A scene that ends right before a `goto` can trail off into the browser's
|
|
221
|
+
// white pre-paint. Freeze-holding THAT frame paints seconds of solid white
|
|
222
|
+
// (issue #36), so drop a flat tail before deciding how long to hold —
|
|
223
|
+
// but only when we would actually hold, and never the whole segment.
|
|
224
|
+
if (srcMs > 0 && targetMs > srcMs + 400) {
|
|
225
|
+
const flatMs = await probeFlatTailMs(rawSegPath, srcMs);
|
|
226
|
+
const trimTo = srcMs - flatMs;
|
|
227
|
+
if (flatMs > 200 && trimTo > 500) {
|
|
228
|
+
blankTrimMs = flatMs;
|
|
229
|
+
srcMs = trimTo;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
102
232
|
// Retime the segment toward the narration length, but only within
|
|
103
233
|
// [MIN_FACTOR, MAX_STRETCH]. If narration still needs more time, hold
|
|
104
234
|
// (freeze) the last frame for the remainder — natural for a static page,
|
|
105
235
|
// and far better than 3x slow-motion.
|
|
106
236
|
const ratio = targetMs / Math.max(srcMs, 1);
|
|
107
|
-
const
|
|
108
|
-
|
|
237
|
+
const uniformFactor = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, ratio));
|
|
238
|
+
|
|
239
|
+
// Narration anchors → piecewise retime. Each anchored action is a raw
|
|
240
|
+
// offset (inside the kept spans) that must land on the ms its word is
|
|
241
|
+
// spoken; between anchors the factor is whatever gets there (clamped to
|
|
242
|
+
// the same [MIN_FACTOR, MAX_STRETCH], so a beat that can't be reached is
|
|
243
|
+
// reported, never faked). No anchors → one piece, the uniform factor —
|
|
244
|
+
// arithmetically identical to the pre-anchor path.
|
|
245
|
+
const sceneDef = sceneById.get(tl.id);
|
|
246
|
+
const anchorDefs = sceneDef?.anchors ?? {};
|
|
247
|
+
const anchorPts: Array<{ name: string; raw: number; target: number }> = [];
|
|
248
|
+
if (Object.keys(anchorDefs).length && (tl.anchorEvents ?? []).length) {
|
|
249
|
+
const words = await sceneWordsFor(project, tl.id, sceneDef?.narration ?? "", narrMs);
|
|
250
|
+
for (const ev of tl.anchorEvents) {
|
|
251
|
+
const wi = anchorDefs[ev.name];
|
|
252
|
+
if (wi == null) continue;
|
|
253
|
+
const target = wordStartMs(words, sceneDef?.narration ?? "", wi);
|
|
254
|
+
if (target == null) continue;
|
|
255
|
+
const raw = offsetInKeeps(keeps, ev.tMs + leadInMs);
|
|
256
|
+
if (raw > srcMs - 40) {
|
|
257
|
+
warn(
|
|
258
|
+
"anchor-unreachable",
|
|
259
|
+
`scene ${tl.id}: anchor "${ev.name}" fires at the very end of the take (${Math.round(raw)}ms of ${Math.round(srcMs)}ms) — add a pause or a later action after it so the beat has room`,
|
|
260
|
+
tl.id
|
|
261
|
+
);
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
anchorPts.push({ name: ev.name, raw, target });
|
|
265
|
+
}
|
|
266
|
+
anchorPts.sort((a, b) => a.raw - b.raw);
|
|
267
|
+
}
|
|
268
|
+
const pieces: RetimePiece[] = [];
|
|
269
|
+
const anchorReport: NonNullable<ComposeSceneReport["anchors"]> = [];
|
|
270
|
+
if (anchorPts.length) {
|
|
271
|
+
let out = 0;
|
|
272
|
+
let prevRaw = 0;
|
|
273
|
+
const bounds = [
|
|
274
|
+
...anchorPts.map((a) => ({ raw: a.raw, target: a.target, name: a.name as string | null })),
|
|
275
|
+
{ raw: srcMs, target: targetMs, name: null as string | null },
|
|
276
|
+
];
|
|
277
|
+
for (const bnd of bounds) {
|
|
278
|
+
const len = bnd.raw - prevRaw;
|
|
279
|
+
if (len >= 40) {
|
|
280
|
+
const want = bnd.target - out;
|
|
281
|
+
const f = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, want / len));
|
|
282
|
+
pieces.push({ a: prevRaw, b: bnd.raw, f, out });
|
|
283
|
+
out += len * f;
|
|
284
|
+
prevRaw = bnd.raw;
|
|
285
|
+
}
|
|
286
|
+
if (bnd.name) {
|
|
287
|
+
const offMs = Math.round(out - bnd.target);
|
|
288
|
+
anchorReport.push({
|
|
289
|
+
name: bnd.name,
|
|
290
|
+
targetMs: Math.round(bnd.target),
|
|
291
|
+
landedMs: Math.round(out),
|
|
292
|
+
offMs,
|
|
293
|
+
});
|
|
294
|
+
if (Math.abs(offMs) > 150) {
|
|
295
|
+
warn(
|
|
296
|
+
"anchor-unreachable",
|
|
297
|
+
`scene ${tl.id}: anchor "${bnd.name}" lands ${Math.abs(offMs)}ms ${offMs > 0 ? "after" : "before"} its word even at ${offMs > 0 ? "2x speed" : "x1.6 slow-motion"} — ${
|
|
298
|
+
offMs > 0
|
|
299
|
+
? "trim what happens before the action (mark waits idle, drop a scroll) or move the {{@" + bnd.name + "}} marker later"
|
|
300
|
+
: "give the action more lead-in (a pause, a hover) or move the {{@" + bnd.name + "}} marker earlier"
|
|
301
|
+
}`,
|
|
302
|
+
tl.id
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
if (!pieces.length) pieces.push({ a: 0, b: srcMs, f: uniformFactor, out: 0 });
|
|
309
|
+
const anchored = anchorPts.length > 0;
|
|
310
|
+
let stretchedMs = anchored
|
|
311
|
+
? pieces[pieces.length - 1].out + (pieces[pieces.length - 1].b - pieces[pieces.length - 1].a) * pieces[pieces.length - 1].f
|
|
312
|
+
: uniformFactor * srcMs;
|
|
313
|
+
const factor = anchored ? stretchedMs / Math.max(srcMs, 1) : uniformFactor;
|
|
314
|
+
/** Kept-span offset (raw ms inside the concatenated keeps) → stretched scene ms. */
|
|
315
|
+
const stretchLocal = (local: number): number => {
|
|
316
|
+
let pc = pieces[0];
|
|
317
|
+
for (const q of pieces) if (local >= q.a) pc = q;
|
|
318
|
+
return pc.out + (local - pc.a) * pc.f;
|
|
319
|
+
};
|
|
109
320
|
// A scene whose active video exceeds 2x its narration would OVERRUN its
|
|
110
321
|
// narration slot — the narration track is fixed, so every later scene's
|
|
111
322
|
// video drifts behind its audio and the finale slides past the end of the
|
|
@@ -118,20 +329,60 @@ export async function compose(
|
|
|
118
329
|
stretchedMs = targetMs;
|
|
119
330
|
}
|
|
120
331
|
const holdMs = Math.max(0, targetMs - stretchedMs);
|
|
121
|
-
|
|
332
|
+
const holding = holdMs > 40;
|
|
333
|
+
// `hold.backoffMs` takes the held frame from slightly before the segment
|
|
334
|
+
// end (a mid-flight hover, a half-painted transition); the hold grows by
|
|
335
|
+
// the same amount so the scene still fills its narration slot exactly.
|
|
336
|
+
const backoffMs = holding
|
|
337
|
+
? Math.min(hold.backoffMs, Math.max(0, stretchedMs - 500))
|
|
338
|
+
: 0;
|
|
339
|
+
const driftHold = holding && hold.mode === "drift";
|
|
340
|
+
let vf = "";
|
|
341
|
+
if (blankTrimMs > 0) {
|
|
342
|
+
vf += `trim=duration=${(srcMs / 1000).toFixed(3)},setpts=PTS-STARTPTS,`;
|
|
343
|
+
}
|
|
344
|
+
// Tail (shared by the uniform and the piecewise path): overrun trim,
|
|
345
|
+
// backoff trim, freeze hold.
|
|
346
|
+
let tail = "";
|
|
122
347
|
if (overrunTrimMs > 0) {
|
|
123
|
-
|
|
348
|
+
tail += `,trim=duration=${(targetMs / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
|
|
124
349
|
}
|
|
125
|
-
if (
|
|
126
|
-
|
|
350
|
+
if (backoffMs > 0) {
|
|
351
|
+
tail += `,trim=duration=${((stretchedMs - backoffMs) / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
|
|
352
|
+
}
|
|
353
|
+
if (holding && !driftHold) {
|
|
354
|
+
tail += `,tpad=stop_mode=clone:stop_duration=${((holdMs + backoffMs) / 1000).toFixed(3)}`;
|
|
355
|
+
}
|
|
356
|
+
let filterArgs: string[];
|
|
357
|
+
if (anchored) {
|
|
358
|
+
// split → per-piece trim + setpts → concat. Same source, same format,
|
|
359
|
+
// so concat is a plain join; everything stays in the core filter set.
|
|
360
|
+
const n = pieces.length;
|
|
361
|
+
const splitLabels = pieces.map((_, k) => `[s${k}]`).join("");
|
|
362
|
+
const pieceFilters = pieces
|
|
363
|
+
.map(
|
|
364
|
+
(pc, k) =>
|
|
365
|
+
`[s${k}]trim=start=${(pc.a / 1000).toFixed(3)}:end=${(pc.b / 1000).toFixed(3)},` +
|
|
366
|
+
`setpts=(PTS-STARTPTS)*${pc.f.toFixed(6)}[p${k}]`
|
|
367
|
+
)
|
|
368
|
+
.join(";");
|
|
369
|
+
const concatIn = pieces.map((_, k) => `[p${k}]`).join("");
|
|
370
|
+
const fc =
|
|
371
|
+
`[0:v]${vf}split=${n}${splitLabels};${pieceFilters};` +
|
|
372
|
+
`${concatIn}concat=n=${n}:v=1:a=0,setpts=PTS-STARTPTS${tail}[vout]`;
|
|
373
|
+
filterArgs = ["-filter_complex", fc, "-map", "[vout]"];
|
|
374
|
+
} else {
|
|
375
|
+
filterArgs = ["-vf", `${vf}setpts=${factor.toFixed(6)}*PTS${tail}`];
|
|
127
376
|
}
|
|
128
377
|
|
|
129
378
|
// Map this scene's focus events into final-video time for the zoom pass:
|
|
130
379
|
// raw time → offset inside the kept spans → stretched → + scene offset.
|
|
380
|
+
let sceneFocus = 0;
|
|
131
381
|
if (zoomCfg && sceneById.get(tl.id)?.zoom !== false) {
|
|
132
382
|
for (const ev of tl.focusEvents ?? []) {
|
|
133
|
-
|
|
134
|
-
const
|
|
383
|
+
sceneFocus++;
|
|
384
|
+
const rawT = ev.tMs + leadInMs;
|
|
385
|
+
const local = stretchLocal(offsetInKeeps(keeps, rawT));
|
|
135
386
|
zoomEvents.push({
|
|
136
387
|
tMs: outCursorMs + Math.min(local, stretchedMs),
|
|
137
388
|
x: ev.x * pxScale,
|
|
@@ -148,12 +399,12 @@ export async function compose(
|
|
|
148
399
|
if (cursorCfg) {
|
|
149
400
|
const sceneStart = outCursorMs;
|
|
150
401
|
for (const s of tl.cursorSamples ?? []) {
|
|
151
|
-
const rawT = s.tMs +
|
|
152
|
-
const local = offsetInKeeps(keeps, rawT)
|
|
402
|
+
const rawT = s.tMs + leadInMs;
|
|
403
|
+
const local = stretchLocal(offsetInKeeps(keeps, rawT));
|
|
153
404
|
cursorPts.push({
|
|
154
405
|
t: (outCursorMs + Math.min(local, stretchedMs)) / 1000,
|
|
155
|
-
x: s.x * pxScale,
|
|
156
|
-
y: s.y * pxScale,
|
|
406
|
+
x: s.x * pxScale - dotOffset,
|
|
407
|
+
y: s.y * pxScale - dotOffset,
|
|
157
408
|
});
|
|
158
409
|
}
|
|
159
410
|
cursorSampleCount += (tl.cursorSamples ?? []).length;
|
|
@@ -163,14 +414,49 @@ export async function compose(
|
|
|
163
414
|
}
|
|
164
415
|
}
|
|
165
416
|
|
|
417
|
+
// Attention beats, key chips, click rings and redact spans: same raw →
|
|
418
|
+
// content-time remap; holds are content-time and clipped to the scene.
|
|
419
|
+
{
|
|
420
|
+
const sceneStart = outCursorMs;
|
|
421
|
+
const sceneEnd = outCursorMs + stretchedMs + (holdMs > 40 ? holdMs : 0);
|
|
422
|
+
const toContent = (rawMs: number): number =>
|
|
423
|
+
sceneStart + Math.min(stretchLocal(offsetInKeeps(keeps, rawMs + leadInMs)), stretchedMs);
|
|
424
|
+
for (const ev of tl.attentionEvents ?? []) {
|
|
425
|
+
const a = toContent(ev.tMs);
|
|
426
|
+
const b = Math.min(sceneEnd, a + ev.holdMs);
|
|
427
|
+
if (b - a < 80) continue;
|
|
428
|
+
attentionPlaced.push({ ...ev, a: a / 1000, b: b / 1000 });
|
|
429
|
+
}
|
|
430
|
+
for (const k of tl.keyEvents ?? []) {
|
|
431
|
+
const a = toContent(k.tMs);
|
|
432
|
+
const b = Math.min(sceneEnd, a + KEY_CHIP_MS);
|
|
433
|
+
if (b - a < 80) continue;
|
|
434
|
+
keysPlaced.push({ keys: k.keys, a: a / 1000, b: b / 1000 });
|
|
435
|
+
}
|
|
436
|
+
if (clickRings) {
|
|
437
|
+
for (const f of tl.focusEvents ?? []) {
|
|
438
|
+
if (f.kind !== "click") continue;
|
|
439
|
+
clicksPlaced.push({ x: f.x, y: f.y, t: toContent(f.tMs) / 1000 });
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
for (const r of tl.redactSpans ?? []) {
|
|
443
|
+
const a = toContent(r.startMs);
|
|
444
|
+
// A span open at the scene end covers the hold too.
|
|
445
|
+
const b = r.endMs >= tl.endMs - 5 ? sceneEnd : toContent(r.endMs);
|
|
446
|
+
if (b - a < 40) continue;
|
|
447
|
+
redactPlaced.push({ ...r, a: a / 1000, b: b / 1000 });
|
|
448
|
+
}
|
|
449
|
+
const pos = sceneById.get(tl.id)?.captions?.position;
|
|
450
|
+
if (pos) captionWindows.push({ a: sceneStart, b: sceneEnd, position: pos });
|
|
451
|
+
}
|
|
452
|
+
|
|
166
453
|
outCursorMs += stretchedMs + (holdMs > 40 ? holdMs : 0);
|
|
167
454
|
|
|
168
|
-
|
|
455
|
+
let scenePath = resolve(tmp, `scene-${i}.mp4`);
|
|
169
456
|
await runFfmpeg([
|
|
170
457
|
"-i",
|
|
171
458
|
rawSegPath,
|
|
172
|
-
|
|
173
|
-
vf,
|
|
459
|
+
...filterArgs,
|
|
174
460
|
"-an",
|
|
175
461
|
"-r",
|
|
176
462
|
String(FPS),
|
|
@@ -184,13 +470,75 @@ export async function compose(
|
|
|
184
470
|
"yuv420p",
|
|
185
471
|
scenePath,
|
|
186
472
|
]);
|
|
473
|
+
if (driftHold) {
|
|
474
|
+
scenePath = await appendDriftHold(
|
|
475
|
+
scenePath,
|
|
476
|
+
holdMs + backoffMs,
|
|
477
|
+
hold.driftScale,
|
|
478
|
+
outW,
|
|
479
|
+
outH,
|
|
480
|
+
tmp,
|
|
481
|
+
i
|
|
482
|
+
);
|
|
483
|
+
}
|
|
187
484
|
sceneVideos.push(scenePath);
|
|
485
|
+
const holdPct = holding ? holdMs / targetMs : 0;
|
|
188
486
|
log(
|
|
189
487
|
`scene ${tl.id}: ${srcMs}ms -> ${targetMs}ms ` +
|
|
190
|
-
`(x${factor.toFixed(2)}${
|
|
488
|
+
`(x${factor.toFixed(2)}${
|
|
489
|
+
holding
|
|
490
|
+
? ` + ${Math.round(holdMs)}ms ${hold.mode === "drift" ? "drift " : ""}hold` +
|
|
491
|
+
(backoffMs > 0 ? ` (backoff ${backoffMs}ms)` : "")
|
|
492
|
+
: ""
|
|
493
|
+
}${
|
|
191
494
|
overrunTrimMs > 0 ? ` - ${Math.round(overrunTrimMs)}ms tail trim` : ""
|
|
192
|
-
}, ${keeps.length} span(s)
|
|
495
|
+
}${blankTrimMs > 0 ? ` - ${Math.round(blankTrimMs)}ms blank tail` : ""}, ${keeps.length} span(s)${
|
|
496
|
+
anchored
|
|
497
|
+
? `, ${pieces.length} piece(s): ` +
|
|
498
|
+
anchorReport.map((a) => `${a.name} ${a.offMs >= 0 ? "+" : ""}${a.offMs}ms`).join(" ")
|
|
499
|
+
: ""
|
|
500
|
+
})`
|
|
193
501
|
);
|
|
502
|
+
sceneReports.push({
|
|
503
|
+
id: tl.id,
|
|
504
|
+
srcMs: Math.round(srcMs),
|
|
505
|
+
targetMs: Math.round(targetMs),
|
|
506
|
+
factor: Number(factor.toFixed(3)),
|
|
507
|
+
holdMs: Math.round(holding ? holdMs : 0),
|
|
508
|
+
holdPct: Number(holdPct.toFixed(3)),
|
|
509
|
+
tailTrimMs: Math.round(overrunTrimMs),
|
|
510
|
+
blankTrimMs: Math.round(blankTrimMs),
|
|
511
|
+
autoIdleMs: Math.round(autoSpans.reduce((a, sp) => a + (sp.endMs - sp.startMs), 0)),
|
|
512
|
+
spans: keeps.length,
|
|
513
|
+
focusEvents: sceneFocus,
|
|
514
|
+
...(anchorReport.length ? { anchors: anchorReport } : {}),
|
|
515
|
+
});
|
|
516
|
+
if (holdPct > FREEZE_WARN_PCT) {
|
|
517
|
+
warn(
|
|
518
|
+
"scene-freeze",
|
|
519
|
+
`scene ${tl.id}: ${Math.round(holdPct * 100)}% of its ${(targetMs / 1000).toFixed(1)}s ` +
|
|
520
|
+
`is a held frame (${(srcMs / 1000).toFixed(1)}s of action for ` +
|
|
521
|
+
`${(targetMs / 1000).toFixed(1)}s of narration) — add on-screen action ` +
|
|
522
|
+
`(hover/scroll/focus beats, a pause) or shorten the narration`,
|
|
523
|
+
tl.id
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
if (overrunTrimMs > 0) {
|
|
527
|
+
warn(
|
|
528
|
+
"overrun-trim",
|
|
529
|
+
`scene ${tl.id}: ${Math.round(overrunTrimMs)}ms of recorded action was cut from ` +
|
|
530
|
+
`the tail — the action outran the narration even at 2x; mark waits idle or ` +
|
|
531
|
+
`lengthen the narration`,
|
|
532
|
+
tl.id
|
|
533
|
+
);
|
|
534
|
+
}
|
|
535
|
+
if (blankTrimMs > 0) {
|
|
536
|
+
warn(
|
|
537
|
+
"blank-tail",
|
|
538
|
+
`scene ${tl.id}: dropped a ${Math.round(blankTrimMs)}ms solid-color tail before holding`,
|
|
539
|
+
tl.id
|
|
540
|
+
);
|
|
541
|
+
}
|
|
194
542
|
opts.onSceneComplete?.(tl.id, i, sceneTotal);
|
|
195
543
|
}
|
|
196
544
|
|
|
@@ -219,6 +567,66 @@ export async function compose(
|
|
|
219
567
|
);
|
|
220
568
|
}
|
|
221
569
|
|
|
570
|
+
// Redaction first: blur boxes ride every later pass (cursor, zoom, blur).
|
|
571
|
+
if (redactPlaced.length) {
|
|
572
|
+
let src = content;
|
|
573
|
+
let passes = 0;
|
|
574
|
+
for (let i = 0; i < redactPlaced.length; i += REDACT_BATCH) {
|
|
575
|
+
const batch = redactPlaced.slice(i, i + REDACT_BATCH);
|
|
576
|
+
const filter = buildRedactFilter(batch, outW, outH, pxScale);
|
|
577
|
+
if (!filter) continue;
|
|
578
|
+
const out = resolve(tmp, `content-redact-${passes}.mp4`);
|
|
579
|
+
await runFfmpeg([
|
|
580
|
+
"-i",
|
|
581
|
+
src,
|
|
582
|
+
"-filter_complex",
|
|
583
|
+
filter,
|
|
584
|
+
"-map",
|
|
585
|
+
"[vout]",
|
|
586
|
+
"-an",
|
|
587
|
+
"-r",
|
|
588
|
+
String(FPS),
|
|
589
|
+
"-c:v",
|
|
590
|
+
"libx264",
|
|
591
|
+
"-preset",
|
|
592
|
+
"veryfast",
|
|
593
|
+
"-crf",
|
|
594
|
+
"20",
|
|
595
|
+
"-pix_fmt",
|
|
596
|
+
"yuv420p",
|
|
597
|
+
out,
|
|
598
|
+
]);
|
|
599
|
+
src = out;
|
|
600
|
+
passes++;
|
|
601
|
+
}
|
|
602
|
+
content = src;
|
|
603
|
+
log(`redact: ${redactPlaced.length} span(s) blurred in ${passes} pass(es)`);
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// Attention overlays (highlight / spotlight / callout / click rings) go on
|
|
607
|
+
// BEFORE the cursor and the zoom, so they ride the camera like page content.
|
|
608
|
+
let attentionItems: OverlayItem[] = [];
|
|
609
|
+
if (attentionPlaced.length || clicksPlaced.length) {
|
|
610
|
+
attentionItems = [
|
|
611
|
+
...(await renderAttentionPngs(
|
|
612
|
+
attentionPlaced,
|
|
613
|
+
resolve(tmp, "attention"),
|
|
614
|
+
storyboard.video.width,
|
|
615
|
+
storyboard.video.height,
|
|
616
|
+
pxScale,
|
|
617
|
+
accent
|
|
618
|
+
)),
|
|
619
|
+
...(await renderClickRingPngs(clicksPlaced, resolve(tmp, "attention"), pxScale, accent)),
|
|
620
|
+
];
|
|
621
|
+
const r = await overlayPngBatches(content, attentionItems, tmp, "attention");
|
|
622
|
+
content = r.video;
|
|
623
|
+
log(
|
|
624
|
+
`attention: ${attentionPlaced.length} beat(s)` +
|
|
625
|
+
(clicksPlaced.length ? `, ${clicksPlaced.length} click ring(s)` : "") +
|
|
626
|
+
` in ${r.passes} pass(es)`
|
|
627
|
+
);
|
|
628
|
+
}
|
|
629
|
+
|
|
222
630
|
// Compose-time cursor overlay (opt-in `cursor` block). Draw the recorded
|
|
223
631
|
// cursor path onto the (cursor-free) content BEFORE the zoom pass, so the
|
|
224
632
|
// cursor zooms and pans with the frame exactly like a baked one would.
|
|
@@ -230,6 +638,10 @@ export async function compose(
|
|
|
230
638
|
"has no recorded cursor path — re-run `record`/`render` with the cursor " +
|
|
231
639
|
"block present (the current recording may still show a baked cursor)."
|
|
232
640
|
);
|
|
641
|
+
warn(
|
|
642
|
+
"cursor-missing",
|
|
643
|
+
"cursor block present but the take has no recorded cursor path — re-record with the block"
|
|
644
|
+
);
|
|
233
645
|
} else if (cursorCfg.hidden) {
|
|
234
646
|
log("cursor: hidden (no overlay drawn)");
|
|
235
647
|
} else {
|
|
@@ -237,7 +649,12 @@ export async function compose(
|
|
|
237
649
|
if (filter) {
|
|
238
650
|
const cScale = cursorCfg.scale ?? 1;
|
|
239
651
|
const png = resolve(tmp, "cursor.png");
|
|
240
|
-
await renderCursorPng(
|
|
652
|
+
await renderCursorPng(
|
|
653
|
+
png,
|
|
654
|
+
Math.round(32 * pxScale * cScale),
|
|
655
|
+
cursorCfg.style ?? "arrow",
|
|
656
|
+
cursorCfg.color
|
|
657
|
+
);
|
|
241
658
|
const withCursor = resolve(tmp, "content-cursor.mp4");
|
|
242
659
|
await runFfmpeg([
|
|
243
660
|
"-i",
|
|
@@ -272,11 +689,40 @@ export async function compose(
|
|
|
272
689
|
}
|
|
273
690
|
|
|
274
691
|
// Auto-zoom pass over the content (never over the cards or captions).
|
|
692
|
+
let focusDropped = 0;
|
|
275
693
|
if (zoomCfg && zoomEvents.length > 0) {
|
|
276
694
|
const contentMs = await probeDurationMs(content);
|
|
277
|
-
const filter = buildZoomFilter(
|
|
695
|
+
const filter = buildZoomFilter(
|
|
696
|
+
zoomEvents,
|
|
697
|
+
zoomCfg,
|
|
698
|
+
outW,
|
|
699
|
+
outH,
|
|
700
|
+
contentMs,
|
|
701
|
+
FPS,
|
|
702
|
+
storyboard.video.width,
|
|
703
|
+
(dropped, total) => {
|
|
704
|
+
focusDropped = dropped;
|
|
705
|
+
log(
|
|
706
|
+
` ! auto-zoom: ${dropped} of ${total} focus point(s) dropped to fit ` +
|
|
707
|
+
`ffmpeg's expression budget — the rest are spread evenly. Set ` +
|
|
708
|
+
`"zoom": false on scenes that don't need the camera to choose which.`
|
|
709
|
+
);
|
|
710
|
+
warn(
|
|
711
|
+
"focus-dropped",
|
|
712
|
+
`auto-zoom dropped ${dropped} of ${total} focus point(s) to fit ffmpeg's ` +
|
|
713
|
+
`expression budget — set "zoom": false on scenes that don't need the camera`
|
|
714
|
+
);
|
|
715
|
+
}
|
|
716
|
+
);
|
|
278
717
|
if (filter) {
|
|
279
|
-
|
|
718
|
+
const capped = clampScaleForWidth(zoomCfg.scale, storyboard.video.width);
|
|
719
|
+
log(
|
|
720
|
+
`auto-zoom: ${zoomEvents.length} focus event(s)` +
|
|
721
|
+
(capped < zoomCfg.scale
|
|
722
|
+
? ` (scale ${zoomCfg.scale} capped to ${capped.toFixed(2)} for a ` +
|
|
723
|
+
`${storyboard.video.width}px-wide viewport)`
|
|
724
|
+
: "")
|
|
725
|
+
);
|
|
280
726
|
const zoomed = resolve(tmp, "content-zoomed.mp4");
|
|
281
727
|
await runFfmpeg([
|
|
282
728
|
"-i",
|
|
@@ -327,19 +773,133 @@ export async function compose(
|
|
|
327
773
|
log(`motion blur: tmix ${frames} frames`);
|
|
328
774
|
}
|
|
329
775
|
|
|
776
|
+
// Keystroke chips are screen-fixed UI: after the zoom/blur, before cards.
|
|
777
|
+
let keyItems: OverlayItem[] = [];
|
|
778
|
+
if (keysPlaced.length) {
|
|
779
|
+
keyItems = await renderKeyChipPngs(
|
|
780
|
+
keysPlaced,
|
|
781
|
+
resolve(tmp, "attention"),
|
|
782
|
+
storyboard.video.width,
|
|
783
|
+
storyboard.video.height,
|
|
784
|
+
pxScale
|
|
785
|
+
);
|
|
786
|
+
const r = await overlayPngBatches(content, keyItems, tmp, "keys");
|
|
787
|
+
content = r.video;
|
|
788
|
+
log(`keystrokes: ${keysPlaced.length} chip(s) in ${r.passes} pass(es)`);
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
// Produced-look frame (opt-in `frame` block): pad the content onto a canvas
|
|
792
|
+
// and overlay the rasterized frame PNG (background, shadow, chrome, rounded
|
|
793
|
+
// hole). Everything after this — cards, watermark, captions — renders at
|
|
794
|
+
// the canvas size, so captions land in the padding band.
|
|
795
|
+
let canvasW = outW;
|
|
796
|
+
let canvasH = outH;
|
|
797
|
+
const outCfg = resolveOutputSizing(storyboard.output);
|
|
798
|
+
if (storyboard.frame) {
|
|
799
|
+
const framePng = resolve(tmp, "frame.png");
|
|
800
|
+
const layout = await renderFramePng(
|
|
801
|
+
storyboard.frame,
|
|
802
|
+
storyboard.brand,
|
|
803
|
+
framePng,
|
|
804
|
+
storyboard.video.width,
|
|
805
|
+
storyboard.video.height,
|
|
806
|
+
pxScale,
|
|
807
|
+
outCfg.width != null && outCfg.height != null ? outCfg.width / outCfg.height : undefined
|
|
808
|
+
);
|
|
809
|
+
const framed = resolve(tmp, "content-framed.mp4");
|
|
810
|
+
await runFfmpeg([
|
|
811
|
+
"-i",
|
|
812
|
+
content,
|
|
813
|
+
"-i",
|
|
814
|
+
framePng,
|
|
815
|
+
"-filter_complex",
|
|
816
|
+
`[0:v]pad=${layout.canvasW}:${layout.canvasH}:${layout.offsetX}:${layout.offsetY}:color=${layout.padColor}[p];` +
|
|
817
|
+
`[p][1:v]overlay=0:0:format=auto[vout]`,
|
|
818
|
+
"-map",
|
|
819
|
+
"[vout]",
|
|
820
|
+
"-an",
|
|
821
|
+
"-r",
|
|
822
|
+
String(FPS),
|
|
823
|
+
"-c:v",
|
|
824
|
+
"libx264",
|
|
825
|
+
"-preset",
|
|
826
|
+
"veryfast",
|
|
827
|
+
"-crf",
|
|
828
|
+
"20",
|
|
829
|
+
"-pix_fmt",
|
|
830
|
+
"yuv420p",
|
|
831
|
+
framed,
|
|
832
|
+
]);
|
|
833
|
+
content = framed;
|
|
834
|
+
canvasW = layout.canvasW;
|
|
835
|
+
canvasH = layout.canvasH;
|
|
836
|
+
log(
|
|
837
|
+
`frame: ${layout.canvasW}x${layout.canvasH} canvas (padding ${storyboard.frame.padding ?? 48}` +
|
|
838
|
+
`${storyboard.frame.chrome && storyboard.frame.chrome !== "none" ? `, ${storyboard.frame.chrome} chrome` : ""})`
|
|
839
|
+
);
|
|
840
|
+
}
|
|
841
|
+
const logicalW = canvasW / pxScale;
|
|
842
|
+
const logicalH = canvasH / pxScale;
|
|
843
|
+
|
|
844
|
+
// Brand watermark (opt-in `brand.logo`): scaled logo in a corner over the
|
|
845
|
+
// content (not the cards — the cards carry the logo in their layout).
|
|
846
|
+
const logoPath = storyboard.brand?.logo
|
|
847
|
+
? resolve(project.dir, storyboard.brand.logo)
|
|
848
|
+
: null;
|
|
849
|
+
if (logoPath && storyboard.brand?.watermark?.enabled !== false) {
|
|
850
|
+
if (!(await exists(logoPath))) {
|
|
851
|
+
warn("brand-logo-missing", `brand.logo not found at ${logoPath} — watermark and card logo skipped`);
|
|
852
|
+
} else {
|
|
853
|
+
const wm = storyboard.brand?.watermark ?? {};
|
|
854
|
+
const wmW = Math.round(canvasW * (wm.scale ?? 0.11));
|
|
855
|
+
const margin = Math.round(24 * pxScale);
|
|
856
|
+
const pos = wm.position ?? "bottom-right";
|
|
857
|
+
const x = pos.endsWith("left") ? margin : canvasW - wmW - margin;
|
|
858
|
+
const yExpr = pos.startsWith("top") ? `${margin}` : `${canvasH}-h-${margin}`;
|
|
859
|
+
const marked = resolve(tmp, "content-marked.mp4");
|
|
860
|
+
await runFfmpeg([
|
|
861
|
+
"-i",
|
|
862
|
+
content,
|
|
863
|
+
"-i",
|
|
864
|
+
logoPath,
|
|
865
|
+
"-filter_complex",
|
|
866
|
+
`[1:v]scale=${wmW}:-1:flags=lanczos,format=rgba,colorchannelmixer=aa=${(wm.opacity ?? 0.85).toFixed(2)}[lg];` +
|
|
867
|
+
`[0:v][lg]overlay=${x}:${yExpr}[vout]`,
|
|
868
|
+
"-map",
|
|
869
|
+
"[vout]",
|
|
870
|
+
"-an",
|
|
871
|
+
"-r",
|
|
872
|
+
String(FPS),
|
|
873
|
+
"-c:v",
|
|
874
|
+
"libx264",
|
|
875
|
+
"-preset",
|
|
876
|
+
"veryfast",
|
|
877
|
+
"-crf",
|
|
878
|
+
"20",
|
|
879
|
+
"-pix_fmt",
|
|
880
|
+
"yuv420p",
|
|
881
|
+
marked,
|
|
882
|
+
]);
|
|
883
|
+
content = marked;
|
|
884
|
+
log(`watermark: ${storyboard.brand?.logo} at ${pos}`);
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
const logoDataUri =
|
|
888
|
+
logoPath && (await exists(logoPath)) ? await fileToDataUri(logoPath) : undefined;
|
|
889
|
+
|
|
330
890
|
// Intro/outro title cards. Music runs under them; narration and captions
|
|
331
891
|
// shift right by the intro's length.
|
|
332
892
|
const segments: string[] = [];
|
|
333
893
|
const introMs = storyboard.intro ? storyboard.intro.durationMs : 0;
|
|
334
894
|
if (storyboard.intro) {
|
|
335
895
|
segments.push(
|
|
336
|
-
await cardSegment(storyboard.intro, "intro", tmp, storyboard,
|
|
896
|
+
await cardSegment(storyboard.intro, "intro", tmp, storyboard, logicalW, logicalH, pxScale, logoDataUri)
|
|
337
897
|
);
|
|
338
898
|
}
|
|
339
899
|
segments.push(content);
|
|
340
900
|
if (storyboard.outro) {
|
|
341
901
|
segments.push(
|
|
342
|
-
await cardSegment(storyboard.outro, "outro", tmp, storyboard,
|
|
902
|
+
await cardSegment(storyboard.outro, "outro", tmp, storyboard, logicalW, logicalH, pxScale, logoDataUri)
|
|
343
903
|
);
|
|
344
904
|
}
|
|
345
905
|
// Re-encode when assembling cards: the card segments carry mp4 edit lists
|
|
@@ -352,29 +912,188 @@ export async function compose(
|
|
|
352
912
|
? content
|
|
353
913
|
: await concatSegments(segments, resolve(tmp, "video.mp4"), tmp, "full", true);
|
|
354
914
|
|
|
355
|
-
|
|
915
|
+
// Caption band collisions: any attention box / key chip that reaches into
|
|
916
|
+
// the bottom strip flips the caption to the top for the cues it overlaps.
|
|
917
|
+
const bandTop = canvasH - Math.round((CAPTION_BOTTOM_GAP + 80) * pxScale);
|
|
918
|
+
const collisions = [...attentionItems, ...keyItems]
|
|
919
|
+
.filter((it) => {
|
|
920
|
+
// Spotlights are full-frame PNGs; use the hole instead of the canvas.
|
|
921
|
+
return it.y + pngHeightHint(it, attentionPlaced, pxScale) > bandTop;
|
|
922
|
+
})
|
|
923
|
+
.map((it) => ({ a: it.a * 1000, b: it.b * 1000 }));
|
|
924
|
+
const captionsResult = await burnCaptions(
|
|
356
925
|
project,
|
|
357
926
|
storyboard,
|
|
358
927
|
fullVideo,
|
|
359
928
|
tmp,
|
|
360
929
|
introMs,
|
|
361
|
-
pxScale
|
|
930
|
+
pxScale,
|
|
931
|
+
{
|
|
932
|
+
defaultPosition: storyboard.captions?.position ?? "bottom",
|
|
933
|
+
windows: captionWindows,
|
|
934
|
+
collisions,
|
|
935
|
+
look: { ...(storyboard.captions ?? {}), font: storyboard.captions?.font ?? storyboard.brand?.font },
|
|
936
|
+
logicalWidth: logicalW,
|
|
937
|
+
}
|
|
362
938
|
);
|
|
939
|
+
const captioned = captionsResult.video;
|
|
363
940
|
// Optional final resize/reframe (opt-in). Applied last, over the whole
|
|
364
941
|
// composed frame (cards + captions included), so a 1280x720 take can render
|
|
365
942
|
// as a vertical social clip. Audio is muxed after, so mux stays -c:v copy.
|
|
366
943
|
// `output` may carry only a `loudness` override (no width/height) — resize
|
|
367
944
|
// just when both dimensions are set.
|
|
368
945
|
const finalVideo =
|
|
369
|
-
|
|
370
|
-
? await resizeOutput(captioned,
|
|
946
|
+
outCfg.width != null && outCfg.height != null
|
|
947
|
+
? await resizeOutput(captioned, outCfg, tmp)
|
|
371
948
|
: captioned;
|
|
372
|
-
|
|
949
|
+
const chapters = storyboard.output?.chapters
|
|
950
|
+
? await writeChapters(tmp, [
|
|
951
|
+
...(storyboard.intro
|
|
952
|
+
? [{ id: "intro", title: storyboard.intro.title, durationMs: introMs }]
|
|
953
|
+
: []),
|
|
954
|
+
...sceneReports.map((r) => ({
|
|
955
|
+
id: r.id,
|
|
956
|
+
title: sceneById.get(r.id)?.title ?? r.id,
|
|
957
|
+
durationMs: r.targetMs,
|
|
958
|
+
})),
|
|
959
|
+
...(storyboard.outro
|
|
960
|
+
? [{ id: "outro", title: storyboard.outro.title, durationMs: storyboard.outro.durationMs }]
|
|
961
|
+
: []),
|
|
962
|
+
])
|
|
963
|
+
: null;
|
|
964
|
+
await muxAudio(project, storyboard, finalVideo, introMs, chapters);
|
|
373
965
|
ok(`final video → ${project.outputPath}`);
|
|
966
|
+
let poster: string | undefined;
|
|
967
|
+
if (storyboard.output?.poster) {
|
|
968
|
+
poster = project.posterPath;
|
|
969
|
+
await runFfmpeg([
|
|
970
|
+
"-ss",
|
|
971
|
+
((introMs + 500) / 1000).toFixed(3),
|
|
972
|
+
"-i",
|
|
973
|
+
project.outputPath,
|
|
974
|
+
"-frames:v",
|
|
975
|
+
"1",
|
|
976
|
+
"-update",
|
|
977
|
+
"1",
|
|
978
|
+
poster,
|
|
979
|
+
]);
|
|
980
|
+
log(`poster → ${poster}`);
|
|
981
|
+
}
|
|
374
982
|
// AIDEMO_KEEP_TMP=1 keeps .compose-tmp for debugging intermediates.
|
|
375
983
|
if (!process.env.AIDEMO_KEEP_TMP) {
|
|
376
984
|
await fs.rm(tmp, { recursive: true, force: true });
|
|
377
985
|
}
|
|
986
|
+
|
|
987
|
+
const report: ComposeReport = {
|
|
988
|
+
output: project.outputPath,
|
|
989
|
+
durationMs: await probeDurationMs(project.outputPath).catch(() => 0),
|
|
990
|
+
scenes: sceneReports,
|
|
991
|
+
zoom: { focusTotal: zoomEvents.length, focusDropped },
|
|
992
|
+
cursorPoints: cursorPts.length,
|
|
993
|
+
captions: { cues: captionsResult.cues, passes: captionsResult.passes },
|
|
994
|
+
hold: hold.mode,
|
|
995
|
+
...(poster ? { poster } : {}),
|
|
996
|
+
...(attentionPlaced.length || keysPlaced.length || clicksPlaced.length || redactPlaced.length
|
|
997
|
+
? {
|
|
998
|
+
attention: {
|
|
999
|
+
events: attentionPlaced.length,
|
|
1000
|
+
keys: keysPlaced.length,
|
|
1001
|
+
clicks: clicksPlaced.length,
|
|
1002
|
+
redactSpans: redactPlaced.length,
|
|
1003
|
+
captionsFlipped: captionsResult.flipped,
|
|
1004
|
+
},
|
|
1005
|
+
}
|
|
1006
|
+
: {}),
|
|
1007
|
+
warnings,
|
|
1008
|
+
};
|
|
1009
|
+
await writeJson(project.reportPath, report);
|
|
1010
|
+
const frozen = sceneReports.filter((s) => s.holdPct > FREEZE_WARN_PCT).length;
|
|
1011
|
+
log(
|
|
1012
|
+
`report → ${project.reportPath}` +
|
|
1013
|
+
(frozen ? ` (${frozen} of ${sceneReports.length} scene(s) mostly held)` : "")
|
|
1014
|
+
);
|
|
1015
|
+
if (warnings.length) {
|
|
1016
|
+
log(`⚠ ${warnings.length} compose warning(s):`);
|
|
1017
|
+
for (const w of warnings) log(` - [${w.code}] ${w.message}`);
|
|
1018
|
+
}
|
|
1019
|
+
return report;
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
/** Share of a scene that may be a held frame before compose flags it. */
|
|
1023
|
+
const FREEZE_WARN_PCT = 0.4;
|
|
1024
|
+
|
|
1025
|
+
/**
|
|
1026
|
+
* Hold mode "drift": instead of cloning the scene's last frame for `holdMs`,
|
|
1027
|
+
* push in on it very slowly (1 → driftScale over the hold), so a long
|
|
1028
|
+
* narration over a static page reads as a deliberate dwell, not a stall.
|
|
1029
|
+
* The frame is pulled from the encoded scene's tail, rendered as a zoompan
|
|
1030
|
+
* still (pre-upscaled 2x so integer-pixel crops don't shimmer — the same rule
|
|
1031
|
+
* the auto-zoom pass follows), and re-encoded onto the scene. One tiny
|
|
1032
|
+
* expression, so the zoom budget is untouched; `zoompan`/`scale` are core.
|
|
1033
|
+
*/
|
|
1034
|
+
async function appendDriftHold(
|
|
1035
|
+
scenePath: string,
|
|
1036
|
+
holdMs: number,
|
|
1037
|
+
driftScale: number,
|
|
1038
|
+
outW: number,
|
|
1039
|
+
outH: number,
|
|
1040
|
+
tmp: string,
|
|
1041
|
+
sceneIdx: number
|
|
1042
|
+
): Promise<string> {
|
|
1043
|
+
const durMs = await probeDurationMs(scenePath);
|
|
1044
|
+
const framePng = resolve(tmp, `scene-${sceneIdx}-hold.png`);
|
|
1045
|
+
await runFfmpeg([
|
|
1046
|
+
"-ss",
|
|
1047
|
+
(Math.max(0, durMs - 60) / 1000).toFixed(3),
|
|
1048
|
+
"-i",
|
|
1049
|
+
scenePath,
|
|
1050
|
+
"-frames:v",
|
|
1051
|
+
"1",
|
|
1052
|
+
"-update",
|
|
1053
|
+
"1",
|
|
1054
|
+
framePng,
|
|
1055
|
+
]);
|
|
1056
|
+
const holdSec = holdMs / 1000;
|
|
1057
|
+
const frames = Math.max(1, Math.round(holdSec * FPS));
|
|
1058
|
+
const holdMp4 = resolve(tmp, `scene-${sceneIdx}-hold.mp4`);
|
|
1059
|
+
// Ease the creep (ease-out quad) so it starts moving right away and settles.
|
|
1060
|
+
const z = `'1+(${driftScale.toFixed(4)}-1)*(1-pow(1-min(on/${frames}\,1)\,2))'`;
|
|
1061
|
+
await runFfmpeg([
|
|
1062
|
+
"-loop",
|
|
1063
|
+
"1",
|
|
1064
|
+
"-framerate",
|
|
1065
|
+
String(FPS),
|
|
1066
|
+
"-i",
|
|
1067
|
+
framePng,
|
|
1068
|
+
"-t",
|
|
1069
|
+
holdSec.toFixed(3),
|
|
1070
|
+
"-vf",
|
|
1071
|
+
`scale=iw*2:ih*2:flags=lanczos,` +
|
|
1072
|
+
`zoompan=z=${z}:x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=1:s=${outW}x${outH}:fps=${FPS},` +
|
|
1073
|
+
`setsar=1`,
|
|
1074
|
+
"-r",
|
|
1075
|
+
String(FPS),
|
|
1076
|
+
"-an",
|
|
1077
|
+
"-c:v",
|
|
1078
|
+
"libx264",
|
|
1079
|
+
"-preset",
|
|
1080
|
+
"veryfast",
|
|
1081
|
+
"-crf",
|
|
1082
|
+
"20",
|
|
1083
|
+
"-pix_fmt",
|
|
1084
|
+
"yuv420p",
|
|
1085
|
+
holdMp4,
|
|
1086
|
+
]);
|
|
1087
|
+
// The PNG-sourced hold carries an edit list; re-encode the join so the pts
|
|
1088
|
+
// start clean at 0 (a stream-copy concat would leak negative timestamps
|
|
1089
|
+
// into the overlay passes — see AGENTS.md).
|
|
1090
|
+
return concatSegments(
|
|
1091
|
+
[scenePath, holdMp4],
|
|
1092
|
+
resolve(tmp, `scene-${sceneIdx}-held.mp4`),
|
|
1093
|
+
tmp,
|
|
1094
|
+
`scene-${sceneIdx}-held`,
|
|
1095
|
+
true
|
|
1096
|
+
);
|
|
378
1097
|
}
|
|
379
1098
|
|
|
380
1099
|
/**
|
|
@@ -384,7 +1103,7 @@ export async function compose(
|
|
|
384
1103
|
* needs an API key so compose can't regenerate them itself; warn loudly and
|
|
385
1104
|
* name the fix instead.
|
|
386
1105
|
*/
|
|
387
|
-
async function warnStaleCaptions(project: Project): Promise<
|
|
1106
|
+
async function warnStaleCaptions(project: Project): Promise<boolean> {
|
|
388
1107
|
try {
|
|
389
1108
|
const [cap, narr] = await Promise.all([
|
|
390
1109
|
fs.stat(project.captionsCuesPath),
|
|
@@ -396,10 +1115,12 @@ async function warnStaleCaptions(project: Project): Promise<void> {
|
|
|
396
1115
|
`the burned captions will not match the audio. ` +
|
|
397
1116
|
`Fix: aidemo captions ${project.dir}` + ` (then re-run compose).`
|
|
398
1117
|
);
|
|
1118
|
+
return true;
|
|
399
1119
|
}
|
|
400
1120
|
} catch {
|
|
401
1121
|
/* captions or narration absent — burnCaptions handles missing captions */
|
|
402
1122
|
}
|
|
1123
|
+
return false;
|
|
403
1124
|
}
|
|
404
1125
|
|
|
405
1126
|
/**
|
|
@@ -564,18 +1285,23 @@ async function cardSegment(
|
|
|
564
1285
|
name: string,
|
|
565
1286
|
tmp: string,
|
|
566
1287
|
storyboard: Storyboard,
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
pxScale: number
|
|
1288
|
+
logicalW: number,
|
|
1289
|
+
logicalH: number,
|
|
1290
|
+
pxScale: number,
|
|
1291
|
+
logoDataUri?: string
|
|
570
1292
|
): Promise<string> {
|
|
571
1293
|
const png = resolve(tmp, `${name}.png`);
|
|
572
1294
|
await renderCardPng(
|
|
573
1295
|
card,
|
|
574
1296
|
png,
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
pxScale
|
|
1297
|
+
Math.round(logicalW),
|
|
1298
|
+
Math.round(logicalH),
|
|
1299
|
+
pxScale,
|
|
1300
|
+
storyboard.brand,
|
|
1301
|
+
logoDataUri
|
|
578
1302
|
);
|
|
1303
|
+
const outW = Math.round(logicalW * pxScale) & ~1;
|
|
1304
|
+
const outH = Math.round(logicalH * pxScale) & ~1;
|
|
579
1305
|
const durSec = card.durationMs / 1000;
|
|
580
1306
|
const fadeSec = Math.min(card.fadeMs / 1000, durSec / 3);
|
|
581
1307
|
const out = resolve(tmp, `${name}.mp4`);
|
|
@@ -609,6 +1335,42 @@ async function cardSegment(
|
|
|
609
1335
|
return out;
|
|
610
1336
|
}
|
|
611
1337
|
|
|
1338
|
+
/** Union of overlapping/adjacent idle spans (auto + annotated), sorted. */
|
|
1339
|
+
function mergeIdle(spans: IdleSpan[]): IdleSpan[] {
|
|
1340
|
+
const sorted = [...spans].sort((a, b) => a.startMs - b.startMs);
|
|
1341
|
+
const out: IdleSpan[] = [];
|
|
1342
|
+
for (const s of sorted) {
|
|
1343
|
+
const last = out[out.length - 1];
|
|
1344
|
+
if (last && s.startMs <= last.endMs) last.endMs = Math.max(last.endMs, s.endMs);
|
|
1345
|
+
else out.push({ ...s });
|
|
1346
|
+
}
|
|
1347
|
+
return out;
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
/**
|
|
1351
|
+
* Motionless spans of the raw file (video time) → idle spans for one scene
|
|
1352
|
+
* (timeline time, the same base the player writes). Spans are clipped to the
|
|
1353
|
+
* scene, and the first/last 300 ms are left alone: the frame right after a
|
|
1354
|
+
* click and the one the scene ends on are the beats a viewer reads, and
|
|
1355
|
+
* trimming into them makes cuts feel clipped.
|
|
1356
|
+
*/
|
|
1357
|
+
function sceneAutoIdle(
|
|
1358
|
+
freezes: FreezeSpan[],
|
|
1359
|
+
tl: Timeline["scenes"][number],
|
|
1360
|
+
leadInMs: number
|
|
1361
|
+
): IdleSpan[] {
|
|
1362
|
+
const EDGE_MS = 300;
|
|
1363
|
+
const lo = tl.startMs + leadInMs + EDGE_MS;
|
|
1364
|
+
const hi = tl.endMs + leadInMs - EDGE_MS;
|
|
1365
|
+
const out: IdleSpan[] = [];
|
|
1366
|
+
for (const f of freezes) {
|
|
1367
|
+
const a = Math.max(f.startMs, lo);
|
|
1368
|
+
const b = Math.min(f.endMs, hi);
|
|
1369
|
+
if (b - a > 200) out.push({ startMs: a - leadInMs, endMs: b - leadInMs, label: "auto-idle" });
|
|
1370
|
+
}
|
|
1371
|
+
return out;
|
|
1372
|
+
}
|
|
1373
|
+
|
|
612
1374
|
/**
|
|
613
1375
|
* Build the list of [startMs,endMs] video-time spans to keep for a scene:
|
|
614
1376
|
* non-idle content in full, each idle span capped to IDLE_CAP_MS.
|
|
@@ -641,6 +1403,14 @@ function keepIntervals(
|
|
|
641
1403
|
return keeps.filter(([a, b]) => b - a > 20);
|
|
642
1404
|
}
|
|
643
1405
|
|
|
1406
|
+
/** One retime piece: kept-span offsets [a,b) play at factor f, starting at stretched ms `out`. */
|
|
1407
|
+
interface RetimePiece {
|
|
1408
|
+
a: number;
|
|
1409
|
+
b: number;
|
|
1410
|
+
f: number;
|
|
1411
|
+
out: number;
|
|
1412
|
+
}
|
|
1413
|
+
|
|
644
1414
|
/**
|
|
645
1415
|
* Where a raw-video timestamp lands inside the concatenation of the kept
|
|
646
1416
|
* spans. Timestamps inside a trimmed gap clamp to the gap's start.
|
|
@@ -723,67 +1493,159 @@ async function extractAndConcat(
|
|
|
723
1493
|
*/
|
|
724
1494
|
const CAPTION_BOTTOM_GAP = 96;
|
|
725
1495
|
|
|
1496
|
+
/**
|
|
1497
|
+
* Max caption cues overlaid in a single ffmpeg pass. Each cue costs one input
|
|
1498
|
+
* and one filter link; ffmpeg fails to configure the graph somewhere past ~90
|
|
1499
|
+
* (issue #36), so long narrations are burned in several passes.
|
|
1500
|
+
*/
|
|
1501
|
+
const CAPTION_BATCH = 32;
|
|
1502
|
+
|
|
1503
|
+
/**
|
|
1504
|
+
* Overlay many time-gated PNGs onto `src` in passes of ≤ CAPTION_BATCH. Each
|
|
1505
|
+
* item is one extra input plus one overlay link; ffmpeg fails to configure a
|
|
1506
|
+
* graph somewhere past ~90 of them (issue #36), so long lists are burned in
|
|
1507
|
+
* several re-encoding passes. Shared by captions, attention beats, key chips.
|
|
1508
|
+
*/
|
|
1509
|
+
async function overlayPngBatches(
|
|
1510
|
+
src: string,
|
|
1511
|
+
items: OverlayItem[],
|
|
1512
|
+
tmp: string,
|
|
1513
|
+
label: string
|
|
1514
|
+
): Promise<{ video: string; passes: number }> {
|
|
1515
|
+
if (items.length === 0) return { video: src, passes: 0 };
|
|
1516
|
+
const batches: OverlayItem[][] = [];
|
|
1517
|
+
for (let i = 0; i < items.length; i += CAPTION_BATCH) {
|
|
1518
|
+
batches.push(items.slice(i, i + CAPTION_BATCH));
|
|
1519
|
+
}
|
|
1520
|
+
if (batches.length > 1) {
|
|
1521
|
+
log(`${label} overlay: ${batches.length} pass(es) of <= ${CAPTION_BATCH} item(s)`);
|
|
1522
|
+
}
|
|
1523
|
+
let cur = src;
|
|
1524
|
+
let out = src;
|
|
1525
|
+
for (let b = 0; b < batches.length; b++) {
|
|
1526
|
+
const batch = batches[b];
|
|
1527
|
+
const args: string[] = ["-i", cur];
|
|
1528
|
+
for (const it of batch) args.push("-i", it.png);
|
|
1529
|
+
const chain: string[] = [];
|
|
1530
|
+
let prev = "0:v";
|
|
1531
|
+
batch.forEach((it, i) => {
|
|
1532
|
+
const inp = `${i + 1}:v`;
|
|
1533
|
+
const lbl = i === batch.length - 1 ? "vout" : `o${i}`;
|
|
1534
|
+
// Escape the enable-expression commas so ffmpeg doesn't read them as
|
|
1535
|
+
// filter separators.
|
|
1536
|
+
chain.push(
|
|
1537
|
+
`[${prev}][${inp}]overlay=${it.x}:${it.y}:enable=between(t\\,${it.a.toFixed(3)}\\,${it.b.toFixed(3)})[${lbl}]`
|
|
1538
|
+
);
|
|
1539
|
+
prev = lbl;
|
|
1540
|
+
});
|
|
1541
|
+
out = resolve(tmp, `${label}-${b}.mp4`);
|
|
1542
|
+
await runFfmpeg([
|
|
1543
|
+
...args,
|
|
1544
|
+
"-filter_complex",
|
|
1545
|
+
chain.join(";"),
|
|
1546
|
+
"-map",
|
|
1547
|
+
"[vout]",
|
|
1548
|
+
"-an",
|
|
1549
|
+
"-c:v",
|
|
1550
|
+
"libx264",
|
|
1551
|
+
"-preset",
|
|
1552
|
+
"veryfast",
|
|
1553
|
+
"-crf",
|
|
1554
|
+
"20",
|
|
1555
|
+
"-pix_fmt",
|
|
1556
|
+
"yuv420p",
|
|
1557
|
+
out,
|
|
1558
|
+
]);
|
|
1559
|
+
cur = out;
|
|
1560
|
+
}
|
|
1561
|
+
return { video: out, passes: batches.length };
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
/** Height of an overlay PNG for the caption-collision check (spotlight → its hole). */
|
|
1565
|
+
function pngHeightHint(it: OverlayItem, placed: PlacedAttention[], pxScale: number): number {
|
|
1566
|
+
if (it.x === 0 && it.y === 0) {
|
|
1567
|
+
const ev = placed.find((p) => p.kind === "spotlight" && p.a === it.a && p.b === it.b);
|
|
1568
|
+
if (ev) return Math.round((ev.y + ev.h + (ev.padding ?? 10)) * pxScale);
|
|
1569
|
+
return 0;
|
|
1570
|
+
}
|
|
1571
|
+
return it.png.includes("/key-") ? 60 * pxScale : 48 * pxScale;
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
interface CaptionPlacement {
|
|
1575
|
+
defaultPosition: "top" | "bottom";
|
|
1576
|
+
/** Caption look (style/font/size/color/background), brand font already merged. */
|
|
1577
|
+
look?: CaptionsConfig & { font?: string };
|
|
1578
|
+
/** Logical (CSS px) width of the frame the strip must span. */
|
|
1579
|
+
logicalWidth?: number;
|
|
1580
|
+
/** Content-time windows (ms, before the intro shift) with a fixed position. */
|
|
1581
|
+
windows: Array<{ a: number; b: number; position: "top" | "bottom" }>;
|
|
1582
|
+
/** Content-time windows (ms) during which the bottom band is occupied. */
|
|
1583
|
+
collisions: Array<{ a: number; b: number }>;
|
|
1584
|
+
}
|
|
1585
|
+
|
|
726
1586
|
async function burnCaptions(
|
|
727
1587
|
project: Project,
|
|
728
1588
|
storyboard: Storyboard,
|
|
729
1589
|
silentVideo: string,
|
|
730
1590
|
tmp: string,
|
|
731
1591
|
introMs: number,
|
|
732
|
-
pxScale: number
|
|
733
|
-
|
|
734
|
-
|
|
1592
|
+
pxScale: number,
|
|
1593
|
+
placement: CaptionPlacement = { defaultPosition: "bottom", windows: [], collisions: [] }
|
|
1594
|
+
): Promise<{ video: string; cues: number; passes: number; flipped: number }> {
|
|
1595
|
+
const none = { video: silentVideo, cues: 0, passes: 0, flipped: 0 };
|
|
1596
|
+
if (placement.look?.style === "none") {
|
|
1597
|
+
log("captions: style none — not burned (SRT/VTT files still written)");
|
|
1598
|
+
return none;
|
|
1599
|
+
}
|
|
1600
|
+
if (!(await exists(project.captionsCuesPath))) return none;
|
|
735
1601
|
const cues = (await readJson<Cue[]>(project.captionsCuesPath)) ?? [];
|
|
736
|
-
if (cues.length === 0) return
|
|
1602
|
+
if (cues.length === 0) return none;
|
|
737
1603
|
|
|
738
1604
|
log(`rendering ${cues.length} caption image(s)`);
|
|
739
1605
|
const rendered = await renderCaptionPngs(
|
|
740
1606
|
cues,
|
|
741
1607
|
resolve(tmp, "captions"),
|
|
742
|
-
storyboard.video.width,
|
|
743
|
-
pxScale
|
|
1608
|
+
Math.round(placement.logicalWidth ?? storyboard.video.width),
|
|
1609
|
+
pxScale,
|
|
1610
|
+
placement.look ?? {}
|
|
744
1611
|
);
|
|
745
1612
|
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
//
|
|
750
|
-
//
|
|
751
|
-
const
|
|
752
|
-
const
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
const
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
);
|
|
765
|
-
|
|
1613
|
+
// Cue times are narration-relative; the intro card shifts them right. The
|
|
1614
|
+
// strip sits `gap` above the bottom edge (clear of an app's bottom input
|
|
1615
|
+
// bar) — or, per scene / on a collision with an attention overlay, at the
|
|
1616
|
+
// top: the PNG is a bottom-anchored strip, so "top" flips it vertically.
|
|
1617
|
+
// A "bar" strip is flush with the frame edge (no gap, no pill offset).
|
|
1618
|
+
const bar = placement.look?.style === "bar";
|
|
1619
|
+
const gap = bar ? 0 : Math.round(CAPTION_BOTTOM_GAP * pxScale);
|
|
1620
|
+
const { height: outH } = await probeVideoDims(silentVideo);
|
|
1621
|
+
let flipped = 0;
|
|
1622
|
+
const overlap = (a: number, b: number, w: { a: number; b: number }) => a < w.b && b > w.a;
|
|
1623
|
+
const items: OverlayItem[] = rendered.map((r) => {
|
|
1624
|
+
let pos = placement.defaultPosition;
|
|
1625
|
+
const win = placement.windows.find((w) => r.startMs >= w.a && r.startMs < w.b);
|
|
1626
|
+
if (win) pos = win.position;
|
|
1627
|
+
else if (pos === "bottom" && placement.collisions.some((c) => overlap(r.startMs, r.endMs, c))) {
|
|
1628
|
+
pos = "top";
|
|
1629
|
+
flipped++;
|
|
1630
|
+
}
|
|
1631
|
+
const stripH = Math.round(r.height * pxScale);
|
|
1632
|
+
// The strip PNG is bottom-anchored (pill bottom ≈ 22 px above the strip's
|
|
1633
|
+
// edge). For "top", slide the strip up so the pill's bottom lands ~gap+70
|
|
1634
|
+
// px from the top edge — a one- or two-line pill stays fully in frame.
|
|
1635
|
+
const topY = bar
|
|
1636
|
+
? -(stripH - Math.round(62 * pxScale))
|
|
1637
|
+
: gap + Math.round(70 * pxScale) - (stripH - Math.round(22 * pxScale));
|
|
1638
|
+
return {
|
|
1639
|
+
png: r.png,
|
|
1640
|
+
x: 0,
|
|
1641
|
+
y: pos === "top" ? topY : outH - stripH - gap,
|
|
1642
|
+
a: (r.startMs + introMs) / 1000,
|
|
1643
|
+
b: (r.endMs + introMs) / 1000,
|
|
1644
|
+
};
|
|
766
1645
|
});
|
|
767
|
-
|
|
768
|
-
const
|
|
769
|
-
|
|
770
|
-
...args,
|
|
771
|
-
"-filter_complex",
|
|
772
|
-
chain.join(";"),
|
|
773
|
-
"-map",
|
|
774
|
-
"[vout]",
|
|
775
|
-
"-an",
|
|
776
|
-
"-c:v",
|
|
777
|
-
"libx264",
|
|
778
|
-
"-preset",
|
|
779
|
-
"veryfast",
|
|
780
|
-
"-crf",
|
|
781
|
-
"20",
|
|
782
|
-
"-pix_fmt",
|
|
783
|
-
"yuv420p",
|
|
784
|
-
out,
|
|
785
|
-
]);
|
|
786
|
-
return out;
|
|
1646
|
+
if (flipped) log(`captions: ${flipped} cue(s) moved to the top (overlay in the bottom band)`);
|
|
1647
|
+
const r = await overlayPngBatches(silentVideo, items, tmp, "captioned");
|
|
1648
|
+
return { video: r.video, cues: cues.length, passes: r.passes, flipped };
|
|
787
1649
|
}
|
|
788
1650
|
|
|
789
1651
|
/**
|
|
@@ -795,11 +1657,51 @@ async function burnCaptions(
|
|
|
795
1657
|
* fixed -22dB bed). The bed is trimmed to the video, faded out at the end,
|
|
796
1658
|
* and the narration is delayed past the intro card.
|
|
797
1659
|
*/
|
|
1660
|
+
/** Output sizing with a preset's defaults filled in (explicit keys win). */
|
|
1661
|
+
function resolveOutputSizing(output: Output | undefined): Output {
|
|
1662
|
+
if (!output) return { fit: "contain" };
|
|
1663
|
+
const p = output.preset ? OUTPUT_PRESETS[output.preset] : null;
|
|
1664
|
+
return {
|
|
1665
|
+
...output,
|
|
1666
|
+
width: output.width ?? p?.width,
|
|
1667
|
+
height: output.height ?? p?.height,
|
|
1668
|
+
fit: output.width != null ? output.fit : (p?.fit ?? output.fit),
|
|
1669
|
+
};
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/** Write an ffmetadata file with one chapter per scene (content time + intro). */
|
|
1673
|
+
async function writeChapters(
|
|
1674
|
+
tmp: string,
|
|
1675
|
+
scenes: Array<{ id: string; title: string; durationMs: number }>
|
|
1676
|
+
): Promise<string> {
|
|
1677
|
+
const lines = [";FFMETADATA1"];
|
|
1678
|
+
let t = 0;
|
|
1679
|
+
for (const s of scenes) {
|
|
1680
|
+
const a = Math.round(t);
|
|
1681
|
+
const b = Math.round(t + s.durationMs);
|
|
1682
|
+
lines.push("[CHAPTER]", "TIMEBASE=1/1000", `START=${a}`, `END=${b}`, `title=${s.title.replace(/[\\=;#\n]/g, " ")}`);
|
|
1683
|
+
t += s.durationMs;
|
|
1684
|
+
}
|
|
1685
|
+
const path = resolve(tmp, "chapters.ffmeta");
|
|
1686
|
+
await fs.writeFile(path, lines.join("\n") + "\n");
|
|
1687
|
+
log(`chapters: ${scenes.length} marker(s)`);
|
|
1688
|
+
return path;
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
async function fileToDataUri(path: string): Promise<string> {
|
|
1692
|
+
const ext = path.toLowerCase().split(".").pop() ?? "png";
|
|
1693
|
+
const mime =
|
|
1694
|
+
ext === "svg" ? "image/svg+xml" : ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
|
|
1695
|
+
const buf = await fs.readFile(path);
|
|
1696
|
+
return `data:${mime};base64,${buf.toString("base64")}`;
|
|
1697
|
+
}
|
|
1698
|
+
|
|
798
1699
|
async function muxAudio(
|
|
799
1700
|
project: Project,
|
|
800
1701
|
storyboard: Storyboard,
|
|
801
1702
|
video: string,
|
|
802
|
-
introMs: number
|
|
1703
|
+
introMs: number,
|
|
1704
|
+
chaptersPath: string | null = null
|
|
803
1705
|
): Promise<void> {
|
|
804
1706
|
let musicPath = await resolveMusic(project, storyboard);
|
|
805
1707
|
const videoMs = await probeDurationMs(video);
|
|
@@ -819,9 +1721,15 @@ async function muxAudio(
|
|
|
819
1721
|
}
|
|
820
1722
|
}
|
|
821
1723
|
|
|
822
|
-
// Inputs, fixed order: 0=video, 1=narration, 2=music (if any)
|
|
1724
|
+
// Inputs, fixed order: 0=video, 1=narration, 2=music (if any), then the
|
|
1725
|
+
// chapters metadata file (mapped by index, never a stream).
|
|
823
1726
|
const args: string[] = ["-i", video, "-i", project.narrationPath];
|
|
824
1727
|
if (musicPath) args.push("-stream_loop", String(musicLoops), "-i", musicPath);
|
|
1728
|
+
let chaptersIdx = -1;
|
|
1729
|
+
if (chaptersPath) {
|
|
1730
|
+
chaptersIdx = musicPath ? 3 : 2;
|
|
1731
|
+
args.push("-i", chaptersPath);
|
|
1732
|
+
}
|
|
825
1733
|
|
|
826
1734
|
const delay = introMs > 0 ? `,adelay=${Math.round(introMs)}:all=1` : "";
|
|
827
1735
|
const filters: string[] = [];
|
|
@@ -892,6 +1800,7 @@ async function muxAudio(
|
|
|
892
1800
|
|
|
893
1801
|
args.push("-filter_complex", filters.join(";"));
|
|
894
1802
|
args.push("-map", "0:v", "-map", "[aout]");
|
|
1803
|
+
if (chaptersIdx >= 0) args.push("-map_metadata", String(chaptersIdx));
|
|
895
1804
|
args.push(
|
|
896
1805
|
"-c:v",
|
|
897
1806
|
"copy",
|
|
@@ -899,6 +1808,10 @@ async function muxAudio(
|
|
|
899
1808
|
"aac",
|
|
900
1809
|
"-b:a",
|
|
901
1810
|
"192k",
|
|
1811
|
+
// moov atom up front so the MP4 streams/scrubs in a browser tab or a PR
|
|
1812
|
+
// preview before it has fully downloaded (frames identical; container only).
|
|
1813
|
+
"-movflags",
|
|
1814
|
+
"+faststart",
|
|
902
1815
|
"-t",
|
|
903
1816
|
durSec,
|
|
904
1817
|
project.outputPath
|