@tandryukha/aidemo 0.8.0 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/compose.ts CHANGED
@@ -1,22 +1,63 @@
1
1
  import { promises as fs } from "node:fs";
2
- import { resolve, isAbsolute } from "node:path";
2
+ import { resolve, isAbsolute, relative } from "node:path";
3
3
  import { Project } from "./project.js";
4
- import type { Storyboard, Timeline, IdleSpan, Card, Output, Loudness } from "./types.js";
5
- import { TimelineSchema, VoiceManifestSchema, LoudnessSchema } from "./types.js";
6
- import type { Cue } from "./captions.js";
4
+ import type {
5
+ Storyboard,
6
+ Timeline,
7
+ IdleSpan,
8
+ Card,
9
+ Output,
10
+ Loudness,
11
+ ComposeReport,
12
+ ComposeSceneReport,
13
+ ComposeWarning,
14
+ } from "./types.js";
15
+ import {
16
+ TimelineSchema,
17
+ VoiceManifestSchema,
18
+ LoudnessSchema,
19
+ HoldSchema,
20
+ AutoIdleSchema,
21
+ } from "./types.js";
22
+ import { sceneWordsFor, type Cue } from "./captions.js";
23
+ import { wordStartMs } from "./anchors.js";
7
24
  import { renderCaptionPngs } from "./caption-render.js";
8
25
  import { renderCardPng } from "./cards.js";
9
26
  import { renderCursorPng } from "./cursor.js";
10
- import { buildZoomFilter, type ZoomEvent } from "./zoom.js";
27
+ import { buildZoomFilter, clampScaleForWidth, type ZoomEvent } from "./zoom.js";
11
28
  import {
12
29
  buildCursorFilter,
13
30
  type CursorPoint,
14
31
  type HideWindow,
15
32
  } from "./cursor-overlay.js";
16
- import { runFfmpeg, probeDurationMs, probeVideoDims } from "./ffmpeg.js";
33
+ import {
34
+ renderAttentionPngs,
35
+ renderKeyChipPngs,
36
+ renderClickRingPngs,
37
+ buildRedactFilter,
38
+ REDACT_BATCH,
39
+ KEY_CHIP_MS,
40
+ DEFAULT_ACCENT,
41
+ type OverlayItem,
42
+ type PlacedAttention,
43
+ type PlacedKey,
44
+ type PlacedClick,
45
+ type PlacedRedact,
46
+ } from "./attention.js";
47
+ import { renderFramePng } from "./frame.js";
48
+ import { OUTPUT_PRESETS, type CaptionsConfig } from "./types.js";
49
+ import {
50
+ runFfmpeg,
51
+ probeDurationMs,
52
+ probeFlatTailMs,
53
+ probeVideoDims,
54
+ probeFreezeSpans,
55
+ type FreezeSpan,
56
+ } from "./ffmpeg.js";
17
57
  import {
18
58
  ensureDir,
19
59
  readJson,
60
+ writeJson,
20
61
  exists,
21
62
  log,
22
63
  ok,
@@ -43,14 +84,26 @@ export async function compose(
43
84
  project: Project,
44
85
  storyboard: Storyboard,
45
86
  opts: SceneProgress = {}
46
- ): Promise<void> {
87
+ ): Promise<ComposeReport> {
47
88
  step("Composing final video (ffmpeg)");
48
89
 
49
90
  const timeline = TimelineSchema.parse(await readJson(project.timelinePath));
50
91
  const voice = VoiceManifestSchema.parse(
51
92
  await readJson(project.voiceManifestPath)
52
93
  );
53
- await warnStaleCaptions(project);
94
+ // Structured twin of the log lines below — see ComposeReportSchema.
95
+ const warnings: ComposeWarning[] = [];
96
+ const sceneReports: ComposeSceneReport[] = [];
97
+ const warn = (code: string, message: string, scene?: string): void => {
98
+ warnings.push({ code, message, ...(scene ? { scene } : {}) });
99
+ };
100
+ const hold = HoldSchema.parse(storyboard.hold ?? {});
101
+ if (await warnStaleCaptions(project)) {
102
+ warn(
103
+ "stale-captions",
104
+ `narration.mp3 is newer than the caption files — re-run: aidemo captions ${project.dir}`
105
+ );
106
+ }
54
107
  const tmp = project.composeTmpDir;
55
108
  await fs.rm(tmp, { recursive: true, force: true });
56
109
  await ensureDir(tmp);
@@ -63,6 +116,41 @@ export async function compose(
63
116
  const { width: outW, height: outH } = await probeVideoDims(rawVideo);
64
117
  const pxScale = outW / storyboard.video.width;
65
118
 
119
+ // --- autoIdle / multi-source plumbing -------------------------------------
120
+ // A resumed take splices scenes from more than one raw file; every source
121
+ // must share the main take's pixel size or all the overlay math (captions,
122
+ // zoom, cursor — all scaled by pxScale) silently misaligns on those scenes.
123
+ const dimsChecked = new Set<string>([rawVideo]);
124
+ const assertSourceDims = async (file: string): Promise<void> => {
125
+ if (dimsChecked.has(file)) return;
126
+ dimsChecked.add(file);
127
+ const d = await probeVideoDims(file);
128
+ if (d.width !== outW || d.height !== outH) {
129
+ throw new Error(
130
+ `resumed take mixes recordings of different sizes: ${relative(project.dir, file)} is ` +
131
+ `${d.width}x${d.height}, the current take is ${outW}x${outH} — re-record the whole demo ` +
132
+ `(a resume must use the same capture mode and viewport as the take it continues)`
133
+ );
134
+ }
135
+ };
136
+ const autoIdleCfg = AutoIdleSchema.parse(
137
+ typeof storyboard.autoIdle === "object" ? storyboard.autoIdle : {}
138
+ );
139
+ const autoIdleOn =
140
+ storyboard.autoIdle === true ||
141
+ (typeof storyboard.autoIdle === "object" && storyboard.autoIdle.enabled !== false);
142
+ const autoIdleFor = (scene: { autoIdle?: boolean } | undefined): boolean =>
143
+ scene?.autoIdle ?? autoIdleOn;
144
+ const freezeCache = new Map<string, Promise<FreezeSpan[]>>();
145
+ const freezeSpansFor = (file: string): Promise<FreezeSpan[]> => {
146
+ let p = freezeCache.get(file);
147
+ if (!p) {
148
+ p = probeFreezeSpans(file, autoIdleCfg.minMs ?? 1500, autoIdleCfg.noise ?? 0.003);
149
+ freezeCache.set(file, p);
150
+ }
151
+ return p;
152
+ };
153
+
66
154
  // Map scene id -> narration duration; compose target = narration + gap.
67
155
  const voiceById = new Map(voice.scenes.map((s) => [s.id, s.durationMs]));
68
156
  const sceneById = new Map(storyboard.scenes.map((s) => [s.id, s]));
@@ -80,6 +168,18 @@ export async function compose(
80
168
  const cursorPts: CursorPoint[] = [];
81
169
  const hideWindows: HideWindow[] = [];
82
170
  let cursorSampleCount = 0;
171
+ // Attention layer accumulators (all opt-in; empty for a plain storyboard).
172
+ const accent = storyboard.attention?.color ?? storyboard.brand?.accent ?? DEFAULT_ACCENT;
173
+ const attentionPlaced: PlacedAttention[] = [];
174
+ const keysPlaced: PlacedKey[] = [];
175
+ const clicksPlaced: PlacedClick[] = [];
176
+ const redactPlaced: PlacedRedact[] = [];
177
+ const clickRings = !!(storyboard.attention?.clicks && cursorCfg);
178
+ /** Per-scene caption placement windows (content ms, before the intro shift). */
179
+ const captionWindows: Array<{ a: number; b: number; position: "top" | "bottom" }> = [];
180
+ // Dot cursors are centered on the point; the arrow PNG's tip is its top-left.
181
+ const dotOffset =
182
+ cursorCfg?.style === "dot" ? Math.round(16 * pxScale * (cursorCfg.scale ?? 1)) : 0;
83
183
 
84
184
  const sceneVideos: string[] = [];
85
185
  const sceneTotal = timeline.scenes.length;
@@ -90,22 +190,133 @@ export async function compose(
90
190
  const targetMs = narrMs + voice.gapMs;
91
191
  if (targetMs <= 0) {
92
192
  log(`scene ${tl.id}: no narration, skipping`);
193
+ warn("scene-no-narration", `scene ${tl.id} has no narration audio and was skipped`, tl.id);
93
194
  opts.onSceneComplete?.(tl.id, i, sceneTotal);
94
195
  continue;
95
196
  }
96
197
 
97
- const keeps = keepIntervals(tl, timeline.leadInMs);
198
+ let blankTrimMs = 0;
199
+ // A resumed take's reused scenes live in an earlier raw file with its own lead-in.
200
+ const leadInMs = tl.leadInMs ?? timeline.leadInMs;
201
+ const sceneRaw = tl.source ? resolve(project.dir, tl.source) : rawVideo;
202
+ await assertSourceDims(sceneRaw);
203
+ // `autoIdle`: motionless spans of the raw take count as idle too, so dead
204
+ // air the storyboard never annotated gets the same cap as a `waitFor`.
205
+ const autoSpans = autoIdleFor(sceneById.get(tl.id))
206
+ ? sceneAutoIdle(await freezeSpansFor(sceneRaw), tl, leadInMs)
207
+ : [];
208
+ if (autoSpans.length) {
209
+ const dead = autoSpans.reduce((a, sp) => a + (sp.endMs - sp.startMs), 0);
210
+ log(`scene ${tl.id}: autoIdle found ${autoSpans.length} motionless span(s), ${dead}ms`);
211
+ }
212
+ const keeps = keepIntervals(
213
+ autoSpans.length ? { ...tl, idleSpans: mergeIdle([...tl.idleSpans, ...autoSpans]) } : tl,
214
+ leadInMs
215
+ );
98
216
  const rawSegPath = resolve(tmp, `scene-${i}-raw.mp4`);
99
- await extractAndConcat(rawVideo, keeps, rawSegPath, tmp, i);
217
+ await extractAndConcat(sceneRaw, keeps, rawSegPath, tmp, i);
100
218
 
101
- const srcMs = await probeDurationMs(rawSegPath);
219
+ let srcMs = await probeDurationMs(rawSegPath);
220
+ // A scene that ends right before a `goto` can trail off into the browser's
221
+ // white pre-paint. Freeze-holding THAT frame paints seconds of solid white
222
+ // (issue #36), so drop a flat tail before deciding how long to hold —
223
+ // but only when we would actually hold, and never the whole segment.
224
+ if (srcMs > 0 && targetMs > srcMs + 400) {
225
+ const flatMs = await probeFlatTailMs(rawSegPath, srcMs);
226
+ const trimTo = srcMs - flatMs;
227
+ if (flatMs > 200 && trimTo > 500) {
228
+ blankTrimMs = flatMs;
229
+ srcMs = trimTo;
230
+ }
231
+ }
102
232
  // Retime the segment toward the narration length, but only within
103
233
  // [MIN_FACTOR, MAX_STRETCH]. If narration still needs more time, hold
104
234
  // (freeze) the last frame for the remainder — natural for a static page,
105
235
  // and far better than 3x slow-motion.
106
236
  const ratio = targetMs / Math.max(srcMs, 1);
107
- const factor = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, ratio));
108
- let stretchedMs = factor * srcMs;
237
+ const uniformFactor = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, ratio));
238
+
239
+ // Narration anchors → piecewise retime. Each anchored action is a raw
240
+ // offset (inside the kept spans) that must land on the ms its word is
241
+ // spoken; between anchors the factor is whatever gets there (clamped to
242
+ // the same [MIN_FACTOR, MAX_STRETCH], so a beat that can't be reached is
243
+ // reported, never faked). No anchors → one piece, the uniform factor —
244
+ // arithmetically identical to the pre-anchor path.
245
+ const sceneDef = sceneById.get(tl.id);
246
+ const anchorDefs = sceneDef?.anchors ?? {};
247
+ const anchorPts: Array<{ name: string; raw: number; target: number }> = [];
248
+ if (Object.keys(anchorDefs).length && (tl.anchorEvents ?? []).length) {
249
+ const words = await sceneWordsFor(project, tl.id, sceneDef?.narration ?? "", narrMs);
250
+ for (const ev of tl.anchorEvents) {
251
+ const wi = anchorDefs[ev.name];
252
+ if (wi == null) continue;
253
+ const target = wordStartMs(words, sceneDef?.narration ?? "", wi);
254
+ if (target == null) continue;
255
+ const raw = offsetInKeeps(keeps, ev.tMs + leadInMs);
256
+ if (raw > srcMs - 40) {
257
+ warn(
258
+ "anchor-unreachable",
259
+ `scene ${tl.id}: anchor "${ev.name}" fires at the very end of the take (${Math.round(raw)}ms of ${Math.round(srcMs)}ms) — add a pause or a later action after it so the beat has room`,
260
+ tl.id
261
+ );
262
+ continue;
263
+ }
264
+ anchorPts.push({ name: ev.name, raw, target });
265
+ }
266
+ anchorPts.sort((a, b) => a.raw - b.raw);
267
+ }
268
+ const pieces: RetimePiece[] = [];
269
+ const anchorReport: NonNullable<ComposeSceneReport["anchors"]> = [];
270
+ if (anchorPts.length) {
271
+ let out = 0;
272
+ let prevRaw = 0;
273
+ const bounds = [
274
+ ...anchorPts.map((a) => ({ raw: a.raw, target: a.target, name: a.name as string | null })),
275
+ { raw: srcMs, target: targetMs, name: null as string | null },
276
+ ];
277
+ for (const bnd of bounds) {
278
+ const len = bnd.raw - prevRaw;
279
+ if (len >= 40) {
280
+ const want = bnd.target - out;
281
+ const f = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, want / len));
282
+ pieces.push({ a: prevRaw, b: bnd.raw, f, out });
283
+ out += len * f;
284
+ prevRaw = bnd.raw;
285
+ }
286
+ if (bnd.name) {
287
+ const offMs = Math.round(out - bnd.target);
288
+ anchorReport.push({
289
+ name: bnd.name,
290
+ targetMs: Math.round(bnd.target),
291
+ landedMs: Math.round(out),
292
+ offMs,
293
+ });
294
+ if (Math.abs(offMs) > 150) {
295
+ warn(
296
+ "anchor-unreachable",
297
+ `scene ${tl.id}: anchor "${bnd.name}" lands ${Math.abs(offMs)}ms ${offMs > 0 ? "after" : "before"} its word even at ${offMs > 0 ? "2x speed" : "x1.6 slow-motion"} — ${
298
+ offMs > 0
299
+ ? "trim what happens before the action (mark waits idle, drop a scroll) or move the {{@" + bnd.name + "}} marker later"
300
+ : "give the action more lead-in (a pause, a hover) or move the {{@" + bnd.name + "}} marker earlier"
301
+ }`,
302
+ tl.id
303
+ );
304
+ }
305
+ }
306
+ }
307
+ }
308
+ if (!pieces.length) pieces.push({ a: 0, b: srcMs, f: uniformFactor, out: 0 });
309
+ const anchored = anchorPts.length > 0;
310
+ let stretchedMs = anchored
311
+ ? pieces[pieces.length - 1].out + (pieces[pieces.length - 1].b - pieces[pieces.length - 1].a) * pieces[pieces.length - 1].f
312
+ : uniformFactor * srcMs;
313
+ const factor = anchored ? stretchedMs / Math.max(srcMs, 1) : uniformFactor;
314
+ /** Kept-span offset (raw ms inside the concatenated keeps) → stretched scene ms. */
315
+ const stretchLocal = (local: number): number => {
316
+ let pc = pieces[0];
317
+ for (const q of pieces) if (local >= q.a) pc = q;
318
+ return pc.out + (local - pc.a) * pc.f;
319
+ };
109
320
  // A scene whose active video exceeds 2x its narration would OVERRUN its
110
321
  // narration slot — the narration track is fixed, so every later scene's
111
322
  // video drifts behind its audio and the finale slides past the end of the
@@ -118,20 +329,60 @@ export async function compose(
118
329
  stretchedMs = targetMs;
119
330
  }
120
331
  const holdMs = Math.max(0, targetMs - stretchedMs);
121
- let vf = `setpts=${factor.toFixed(6)}*PTS`;
332
+ const holding = holdMs > 40;
333
+ // `hold.backoffMs` takes the held frame from slightly before the segment
334
+ // end (a mid-flight hover, a half-painted transition); the hold grows by
335
+ // the same amount so the scene still fills its narration slot exactly.
336
+ const backoffMs = holding
337
+ ? Math.min(hold.backoffMs, Math.max(0, stretchedMs - 500))
338
+ : 0;
339
+ const driftHold = holding && hold.mode === "drift";
340
+ let vf = "";
341
+ if (blankTrimMs > 0) {
342
+ vf += `trim=duration=${(srcMs / 1000).toFixed(3)},setpts=PTS-STARTPTS,`;
343
+ }
344
+ // Tail (shared by the uniform and the piecewise path): overrun trim,
345
+ // backoff trim, freeze hold.
346
+ let tail = "";
122
347
  if (overrunTrimMs > 0) {
123
- vf += `,trim=duration=${(targetMs / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
348
+ tail += `,trim=duration=${(targetMs / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
124
349
  }
125
- if (holdMs > 40) {
126
- vf += `,tpad=stop_mode=clone:stop_duration=${(holdMs / 1000).toFixed(3)}`;
350
+ if (backoffMs > 0) {
351
+ tail += `,trim=duration=${((stretchedMs - backoffMs) / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
352
+ }
353
+ if (holding && !driftHold) {
354
+ tail += `,tpad=stop_mode=clone:stop_duration=${((holdMs + backoffMs) / 1000).toFixed(3)}`;
355
+ }
356
+ let filterArgs: string[];
357
+ if (anchored) {
358
+ // split → per-piece trim + setpts → concat. Same source, same format,
359
+ // so concat is a plain join; everything stays in the core filter set.
360
+ const n = pieces.length;
361
+ const splitLabels = pieces.map((_, k) => `[s${k}]`).join("");
362
+ const pieceFilters = pieces
363
+ .map(
364
+ (pc, k) =>
365
+ `[s${k}]trim=start=${(pc.a / 1000).toFixed(3)}:end=${(pc.b / 1000).toFixed(3)},` +
366
+ `setpts=(PTS-STARTPTS)*${pc.f.toFixed(6)}[p${k}]`
367
+ )
368
+ .join(";");
369
+ const concatIn = pieces.map((_, k) => `[p${k}]`).join("");
370
+ const fc =
371
+ `[0:v]${vf}split=${n}${splitLabels};${pieceFilters};` +
372
+ `${concatIn}concat=n=${n}:v=1:a=0,setpts=PTS-STARTPTS${tail}[vout]`;
373
+ filterArgs = ["-filter_complex", fc, "-map", "[vout]"];
374
+ } else {
375
+ filterArgs = ["-vf", `${vf}setpts=${factor.toFixed(6)}*PTS${tail}`];
127
376
  }
128
377
 
129
378
  // Map this scene's focus events into final-video time for the zoom pass:
130
379
  // raw time → offset inside the kept spans → stretched → + scene offset.
380
+ let sceneFocus = 0;
131
381
  if (zoomCfg && sceneById.get(tl.id)?.zoom !== false) {
132
382
  for (const ev of tl.focusEvents ?? []) {
133
- const rawT = ev.tMs + timeline.leadInMs;
134
- const local = offsetInKeeps(keeps, rawT) * factor;
383
+ sceneFocus++;
384
+ const rawT = ev.tMs + leadInMs;
385
+ const local = stretchLocal(offsetInKeeps(keeps, rawT));
135
386
  zoomEvents.push({
136
387
  tMs: outCursorMs + Math.min(local, stretchedMs),
137
388
  x: ev.x * pxScale,
@@ -148,12 +399,12 @@ export async function compose(
148
399
  if (cursorCfg) {
149
400
  const sceneStart = outCursorMs;
150
401
  for (const s of tl.cursorSamples ?? []) {
151
- const rawT = s.tMs + timeline.leadInMs;
152
- const local = offsetInKeeps(keeps, rawT) * factor;
402
+ const rawT = s.tMs + leadInMs;
403
+ const local = stretchLocal(offsetInKeeps(keeps, rawT));
153
404
  cursorPts.push({
154
405
  t: (outCursorMs + Math.min(local, stretchedMs)) / 1000,
155
- x: s.x * pxScale,
156
- y: s.y * pxScale,
406
+ x: s.x * pxScale - dotOffset,
407
+ y: s.y * pxScale - dotOffset,
157
408
  });
158
409
  }
159
410
  cursorSampleCount += (tl.cursorSamples ?? []).length;
@@ -163,14 +414,49 @@ export async function compose(
163
414
  }
164
415
  }
165
416
 
417
+ // Attention beats, key chips, click rings and redact spans: same raw →
418
+ // content-time remap; holds are content-time and clipped to the scene.
419
+ {
420
+ const sceneStart = outCursorMs;
421
+ const sceneEnd = outCursorMs + stretchedMs + (holdMs > 40 ? holdMs : 0);
422
+ const toContent = (rawMs: number): number =>
423
+ sceneStart + Math.min(stretchLocal(offsetInKeeps(keeps, rawMs + leadInMs)), stretchedMs);
424
+ for (const ev of tl.attentionEvents ?? []) {
425
+ const a = toContent(ev.tMs);
426
+ const b = Math.min(sceneEnd, a + ev.holdMs);
427
+ if (b - a < 80) continue;
428
+ attentionPlaced.push({ ...ev, a: a / 1000, b: b / 1000 });
429
+ }
430
+ for (const k of tl.keyEvents ?? []) {
431
+ const a = toContent(k.tMs);
432
+ const b = Math.min(sceneEnd, a + KEY_CHIP_MS);
433
+ if (b - a < 80) continue;
434
+ keysPlaced.push({ keys: k.keys, a: a / 1000, b: b / 1000 });
435
+ }
436
+ if (clickRings) {
437
+ for (const f of tl.focusEvents ?? []) {
438
+ if (f.kind !== "click") continue;
439
+ clicksPlaced.push({ x: f.x, y: f.y, t: toContent(f.tMs) / 1000 });
440
+ }
441
+ }
442
+ for (const r of tl.redactSpans ?? []) {
443
+ const a = toContent(r.startMs);
444
+ // A span open at the scene end covers the hold too.
445
+ const b = r.endMs >= tl.endMs - 5 ? sceneEnd : toContent(r.endMs);
446
+ if (b - a < 40) continue;
447
+ redactPlaced.push({ ...r, a: a / 1000, b: b / 1000 });
448
+ }
449
+ const pos = sceneById.get(tl.id)?.captions?.position;
450
+ if (pos) captionWindows.push({ a: sceneStart, b: sceneEnd, position: pos });
451
+ }
452
+
166
453
  outCursorMs += stretchedMs + (holdMs > 40 ? holdMs : 0);
167
454
 
168
- const scenePath = resolve(tmp, `scene-${i}.mp4`);
455
+ let scenePath = resolve(tmp, `scene-${i}.mp4`);
169
456
  await runFfmpeg([
170
457
  "-i",
171
458
  rawSegPath,
172
- "-vf",
173
- vf,
459
+ ...filterArgs,
174
460
  "-an",
175
461
  "-r",
176
462
  String(FPS),
@@ -184,13 +470,75 @@ export async function compose(
184
470
  "yuv420p",
185
471
  scenePath,
186
472
  ]);
473
+ if (driftHold) {
474
+ scenePath = await appendDriftHold(
475
+ scenePath,
476
+ holdMs + backoffMs,
477
+ hold.driftScale,
478
+ outW,
479
+ outH,
480
+ tmp,
481
+ i
482
+ );
483
+ }
187
484
  sceneVideos.push(scenePath);
485
+ const holdPct = holding ? holdMs / targetMs : 0;
188
486
  log(
189
487
  `scene ${tl.id}: ${srcMs}ms -> ${targetMs}ms ` +
190
- `(x${factor.toFixed(2)}${holdMs > 40 ? ` + ${Math.round(holdMs)}ms hold` : ""}${
488
+ `(x${factor.toFixed(2)}${
489
+ holding
490
+ ? ` + ${Math.round(holdMs)}ms ${hold.mode === "drift" ? "drift " : ""}hold` +
491
+ (backoffMs > 0 ? ` (backoff ${backoffMs}ms)` : "")
492
+ : ""
493
+ }${
191
494
  overrunTrimMs > 0 ? ` - ${Math.round(overrunTrimMs)}ms tail trim` : ""
192
- }, ${keeps.length} span(s))`
495
+ }${blankTrimMs > 0 ? ` - ${Math.round(blankTrimMs)}ms blank tail` : ""}, ${keeps.length} span(s)${
496
+ anchored
497
+ ? `, ${pieces.length} piece(s): ` +
498
+ anchorReport.map((a) => `${a.name} ${a.offMs >= 0 ? "+" : ""}${a.offMs}ms`).join(" ")
499
+ : ""
500
+ })`
193
501
  );
502
+ sceneReports.push({
503
+ id: tl.id,
504
+ srcMs: Math.round(srcMs),
505
+ targetMs: Math.round(targetMs),
506
+ factor: Number(factor.toFixed(3)),
507
+ holdMs: Math.round(holding ? holdMs : 0),
508
+ holdPct: Number(holdPct.toFixed(3)),
509
+ tailTrimMs: Math.round(overrunTrimMs),
510
+ blankTrimMs: Math.round(blankTrimMs),
511
+ autoIdleMs: Math.round(autoSpans.reduce((a, sp) => a + (sp.endMs - sp.startMs), 0)),
512
+ spans: keeps.length,
513
+ focusEvents: sceneFocus,
514
+ ...(anchorReport.length ? { anchors: anchorReport } : {}),
515
+ });
516
+ if (holdPct > FREEZE_WARN_PCT) {
517
+ warn(
518
+ "scene-freeze",
519
+ `scene ${tl.id}: ${Math.round(holdPct * 100)}% of its ${(targetMs / 1000).toFixed(1)}s ` +
520
+ `is a held frame (${(srcMs / 1000).toFixed(1)}s of action for ` +
521
+ `${(targetMs / 1000).toFixed(1)}s of narration) — add on-screen action ` +
522
+ `(hover/scroll/focus beats, a pause) or shorten the narration`,
523
+ tl.id
524
+ );
525
+ }
526
+ if (overrunTrimMs > 0) {
527
+ warn(
528
+ "overrun-trim",
529
+ `scene ${tl.id}: ${Math.round(overrunTrimMs)}ms of recorded action was cut from ` +
530
+ `the tail — the action outran the narration even at 2x; mark waits idle or ` +
531
+ `lengthen the narration`,
532
+ tl.id
533
+ );
534
+ }
535
+ if (blankTrimMs > 0) {
536
+ warn(
537
+ "blank-tail",
538
+ `scene ${tl.id}: dropped a ${Math.round(blankTrimMs)}ms solid-color tail before holding`,
539
+ tl.id
540
+ );
541
+ }
194
542
  opts.onSceneComplete?.(tl.id, i, sceneTotal);
195
543
  }
196
544
 
@@ -219,6 +567,66 @@ export async function compose(
219
567
  );
220
568
  }
221
569
 
570
+ // Redaction first: blur boxes ride every later pass (cursor, zoom, blur).
571
+ if (redactPlaced.length) {
572
+ let src = content;
573
+ let passes = 0;
574
+ for (let i = 0; i < redactPlaced.length; i += REDACT_BATCH) {
575
+ const batch = redactPlaced.slice(i, i + REDACT_BATCH);
576
+ const filter = buildRedactFilter(batch, outW, outH, pxScale);
577
+ if (!filter) continue;
578
+ const out = resolve(tmp, `content-redact-${passes}.mp4`);
579
+ await runFfmpeg([
580
+ "-i",
581
+ src,
582
+ "-filter_complex",
583
+ filter,
584
+ "-map",
585
+ "[vout]",
586
+ "-an",
587
+ "-r",
588
+ String(FPS),
589
+ "-c:v",
590
+ "libx264",
591
+ "-preset",
592
+ "veryfast",
593
+ "-crf",
594
+ "20",
595
+ "-pix_fmt",
596
+ "yuv420p",
597
+ out,
598
+ ]);
599
+ src = out;
600
+ passes++;
601
+ }
602
+ content = src;
603
+ log(`redact: ${redactPlaced.length} span(s) blurred in ${passes} pass(es)`);
604
+ }
605
+
606
+ // Attention overlays (highlight / spotlight / callout / click rings) go on
607
+ // BEFORE the cursor and the zoom, so they ride the camera like page content.
608
+ let attentionItems: OverlayItem[] = [];
609
+ if (attentionPlaced.length || clicksPlaced.length) {
610
+ attentionItems = [
611
+ ...(await renderAttentionPngs(
612
+ attentionPlaced,
613
+ resolve(tmp, "attention"),
614
+ storyboard.video.width,
615
+ storyboard.video.height,
616
+ pxScale,
617
+ accent
618
+ )),
619
+ ...(await renderClickRingPngs(clicksPlaced, resolve(tmp, "attention"), pxScale, accent)),
620
+ ];
621
+ const r = await overlayPngBatches(content, attentionItems, tmp, "attention");
622
+ content = r.video;
623
+ log(
624
+ `attention: ${attentionPlaced.length} beat(s)` +
625
+ (clicksPlaced.length ? `, ${clicksPlaced.length} click ring(s)` : "") +
626
+ ` in ${r.passes} pass(es)`
627
+ );
628
+ }
629
+
222
630
  // Compose-time cursor overlay (opt-in `cursor` block). Draw the recorded
223
631
  // cursor path onto the (cursor-free) content BEFORE the zoom pass, so the
224
632
  // cursor zooms and pans with the frame exactly like a baked one would.
@@ -230,6 +638,10 @@ export async function compose(
230
638
  "has no recorded cursor path — re-run `record`/`render` with the cursor " +
231
639
  "block present (the current recording may still show a baked cursor)."
232
640
  );
641
+ warn(
642
+ "cursor-missing",
643
+ "cursor block present but the take has no recorded cursor path — re-record with the block"
644
+ );
233
645
  } else if (cursorCfg.hidden) {
234
646
  log("cursor: hidden (no overlay drawn)");
235
647
  } else {
@@ -237,7 +649,12 @@ export async function compose(
237
649
  if (filter) {
238
650
  const cScale = cursorCfg.scale ?? 1;
239
651
  const png = resolve(tmp, "cursor.png");
240
- await renderCursorPng(png, Math.round(32 * pxScale * cScale));
652
+ await renderCursorPng(
653
+ png,
654
+ Math.round(32 * pxScale * cScale),
655
+ cursorCfg.style ?? "arrow",
656
+ cursorCfg.color
657
+ );
241
658
  const withCursor = resolve(tmp, "content-cursor.mp4");
242
659
  await runFfmpeg([
243
660
  "-i",
@@ -272,11 +689,40 @@ export async function compose(
272
689
  }
273
690
 
274
691
  // Auto-zoom pass over the content (never over the cards or captions).
692
+ let focusDropped = 0;
275
693
  if (zoomCfg && zoomEvents.length > 0) {
276
694
  const contentMs = await probeDurationMs(content);
277
- const filter = buildZoomFilter(zoomEvents, zoomCfg, outW, outH, contentMs, FPS);
695
+ const filter = buildZoomFilter(
696
+ zoomEvents,
697
+ zoomCfg,
698
+ outW,
699
+ outH,
700
+ contentMs,
701
+ FPS,
702
+ storyboard.video.width,
703
+ (dropped, total) => {
704
+ focusDropped = dropped;
705
+ log(
706
+ ` ! auto-zoom: ${dropped} of ${total} focus point(s) dropped to fit ` +
707
+ `ffmpeg's expression budget — the rest are spread evenly. Set ` +
708
+ `"zoom": false on scenes that don't need the camera to choose which.`
709
+ );
710
+ warn(
711
+ "focus-dropped",
712
+ `auto-zoom dropped ${dropped} of ${total} focus point(s) to fit ffmpeg's ` +
713
+ `expression budget — set "zoom": false on scenes that don't need the camera`
714
+ );
715
+ }
716
+ );
278
717
  if (filter) {
279
- log(`auto-zoom: ${zoomEvents.length} focus event(s)`);
718
+ const capped = clampScaleForWidth(zoomCfg.scale, storyboard.video.width);
719
+ log(
720
+ `auto-zoom: ${zoomEvents.length} focus event(s)` +
721
+ (capped < zoomCfg.scale
722
+ ? ` (scale ${zoomCfg.scale} capped to ${capped.toFixed(2)} for a ` +
723
+ `${storyboard.video.width}px-wide viewport)`
724
+ : "")
725
+ );
280
726
  const zoomed = resolve(tmp, "content-zoomed.mp4");
281
727
  await runFfmpeg([
282
728
  "-i",
@@ -327,19 +773,133 @@ export async function compose(
327
773
  log(`motion blur: tmix ${frames} frames`);
328
774
  }
329
775
 
776
+ // Keystroke chips are screen-fixed UI: after the zoom/blur, before cards.
777
+ let keyItems: OverlayItem[] = [];
778
+ if (keysPlaced.length) {
779
+ keyItems = await renderKeyChipPngs(
780
+ keysPlaced,
781
+ resolve(tmp, "attention"),
782
+ storyboard.video.width,
783
+ storyboard.video.height,
784
+ pxScale
785
+ );
786
+ const r = await overlayPngBatches(content, keyItems, tmp, "keys");
787
+ content = r.video;
788
+ log(`keystrokes: ${keysPlaced.length} chip(s) in ${r.passes} pass(es)`);
789
+ }
790
+
791
+ // Produced-look frame (opt-in `frame` block): pad the content onto a canvas
792
+ // and overlay the rasterized frame PNG (background, shadow, chrome, rounded
793
+ // hole). Everything after this — cards, watermark, captions — renders at
794
+ // the canvas size, so captions land in the padding band.
795
+ let canvasW = outW;
796
+ let canvasH = outH;
797
+ const outCfg = resolveOutputSizing(storyboard.output);
798
+ if (storyboard.frame) {
799
+ const framePng = resolve(tmp, "frame.png");
800
+ const layout = await renderFramePng(
801
+ storyboard.frame,
802
+ storyboard.brand,
803
+ framePng,
804
+ storyboard.video.width,
805
+ storyboard.video.height,
806
+ pxScale,
807
+ outCfg.width != null && outCfg.height != null ? outCfg.width / outCfg.height : undefined
808
+ );
809
+ const framed = resolve(tmp, "content-framed.mp4");
810
+ await runFfmpeg([
811
+ "-i",
812
+ content,
813
+ "-i",
814
+ framePng,
815
+ "-filter_complex",
816
+ `[0:v]pad=${layout.canvasW}:${layout.canvasH}:${layout.offsetX}:${layout.offsetY}:color=${layout.padColor}[p];` +
817
+ `[p][1:v]overlay=0:0:format=auto[vout]`,
818
+ "-map",
819
+ "[vout]",
820
+ "-an",
821
+ "-r",
822
+ String(FPS),
823
+ "-c:v",
824
+ "libx264",
825
+ "-preset",
826
+ "veryfast",
827
+ "-crf",
828
+ "20",
829
+ "-pix_fmt",
830
+ "yuv420p",
831
+ framed,
832
+ ]);
833
+ content = framed;
834
+ canvasW = layout.canvasW;
835
+ canvasH = layout.canvasH;
836
+ log(
837
+ `frame: ${layout.canvasW}x${layout.canvasH} canvas (padding ${storyboard.frame.padding ?? 48}` +
838
+ `${storyboard.frame.chrome && storyboard.frame.chrome !== "none" ? `, ${storyboard.frame.chrome} chrome` : ""})`
839
+ );
840
+ }
841
+ const logicalW = canvasW / pxScale;
842
+ const logicalH = canvasH / pxScale;
843
+
844
+ // Brand watermark (opt-in `brand.logo`): scaled logo in a corner over the
845
+ // content (not the cards — the cards carry the logo in their layout).
846
+ const logoPath = storyboard.brand?.logo
847
+ ? resolve(project.dir, storyboard.brand.logo)
848
+ : null;
849
+ if (logoPath && storyboard.brand?.watermark?.enabled !== false) {
850
+ if (!(await exists(logoPath))) {
851
+ warn("brand-logo-missing", `brand.logo not found at ${logoPath} — watermark and card logo skipped`);
852
+ } else {
853
+ const wm = storyboard.brand?.watermark ?? {};
854
+ const wmW = Math.round(canvasW * (wm.scale ?? 0.11));
855
+ const margin = Math.round(24 * pxScale);
856
+ const pos = wm.position ?? "bottom-right";
857
+ const x = pos.endsWith("left") ? margin : canvasW - wmW - margin;
858
+ const yExpr = pos.startsWith("top") ? `${margin}` : `${canvasH}-h-${margin}`;
859
+ const marked = resolve(tmp, "content-marked.mp4");
860
+ await runFfmpeg([
861
+ "-i",
862
+ content,
863
+ "-i",
864
+ logoPath,
865
+ "-filter_complex",
866
+ `[1:v]scale=${wmW}:-1:flags=lanczos,format=rgba,colorchannelmixer=aa=${(wm.opacity ?? 0.85).toFixed(2)}[lg];` +
867
+ `[0:v][lg]overlay=${x}:${yExpr}[vout]`,
868
+ "-map",
869
+ "[vout]",
870
+ "-an",
871
+ "-r",
872
+ String(FPS),
873
+ "-c:v",
874
+ "libx264",
875
+ "-preset",
876
+ "veryfast",
877
+ "-crf",
878
+ "20",
879
+ "-pix_fmt",
880
+ "yuv420p",
881
+ marked,
882
+ ]);
883
+ content = marked;
884
+ log(`watermark: ${storyboard.brand?.logo} at ${pos}`);
885
+ }
886
+ }
887
+ const logoDataUri =
888
+ logoPath && (await exists(logoPath)) ? await fileToDataUri(logoPath) : undefined;
889
+
330
890
  // Intro/outro title cards. Music runs under them; narration and captions
331
891
  // shift right by the intro's length.
332
892
  const segments: string[] = [];
333
893
  const introMs = storyboard.intro ? storyboard.intro.durationMs : 0;
334
894
  if (storyboard.intro) {
335
895
  segments.push(
336
- await cardSegment(storyboard.intro, "intro", tmp, storyboard, outW, outH, pxScale)
896
+ await cardSegment(storyboard.intro, "intro", tmp, storyboard, logicalW, logicalH, pxScale, logoDataUri)
337
897
  );
338
898
  }
339
899
  segments.push(content);
340
900
  if (storyboard.outro) {
341
901
  segments.push(
342
- await cardSegment(storyboard.outro, "outro", tmp, storyboard, outW, outH, pxScale)
902
+ await cardSegment(storyboard.outro, "outro", tmp, storyboard, logicalW, logicalH, pxScale, logoDataUri)
343
903
  );
344
904
  }
345
905
  // Re-encode when assembling cards: the card segments carry mp4 edit lists
@@ -352,29 +912,188 @@ export async function compose(
352
912
  ? content
353
913
  : await concatSegments(segments, resolve(tmp, "video.mp4"), tmp, "full", true);
354
914
 
355
- const captioned = await burnCaptions(
915
+ // Caption band collisions: any attention box / key chip that reaches into
916
+ // the bottom strip flips the caption to the top for the cues it overlaps.
917
+ const bandTop = canvasH - Math.round((CAPTION_BOTTOM_GAP + 80) * pxScale);
918
+ const collisions = [...attentionItems, ...keyItems]
919
+ .filter((it) => {
920
+ // Spotlights are full-frame PNGs; use the hole instead of the canvas.
921
+ return it.y + pngHeightHint(it, attentionPlaced, pxScale) > bandTop;
922
+ })
923
+ .map((it) => ({ a: it.a * 1000, b: it.b * 1000 }));
924
+ const captionsResult = await burnCaptions(
356
925
  project,
357
926
  storyboard,
358
927
  fullVideo,
359
928
  tmp,
360
929
  introMs,
361
- pxScale
930
+ pxScale,
931
+ {
932
+ defaultPosition: storyboard.captions?.position ?? "bottom",
933
+ windows: captionWindows,
934
+ collisions,
935
+ look: { ...(storyboard.captions ?? {}), font: storyboard.captions?.font ?? storyboard.brand?.font },
936
+ logicalWidth: logicalW,
937
+ }
362
938
  );
939
+ const captioned = captionsResult.video;
363
940
  // Optional final resize/reframe (opt-in). Applied last, over the whole
364
941
  // composed frame (cards + captions included), so a 1280x720 take can render
365
942
  // as a vertical social clip. Audio is muxed after, so mux stays -c:v copy.
366
943
  // `output` may carry only a `loudness` override (no width/height) — resize
367
944
  // just when both dimensions are set.
368
945
  const finalVideo =
369
- storyboard.output?.width != null && storyboard.output?.height != null
370
- ? await resizeOutput(captioned, storyboard.output, tmp)
946
+ outCfg.width != null && outCfg.height != null
947
+ ? await resizeOutput(captioned, outCfg, tmp)
371
948
  : captioned;
372
- await muxAudio(project, storyboard, finalVideo, introMs);
949
+ const chapters = storyboard.output?.chapters
950
+ ? await writeChapters(tmp, [
951
+ ...(storyboard.intro
952
+ ? [{ id: "intro", title: storyboard.intro.title, durationMs: introMs }]
953
+ : []),
954
+ ...sceneReports.map((r) => ({
955
+ id: r.id,
956
+ title: sceneById.get(r.id)?.title ?? r.id,
957
+ durationMs: r.targetMs,
958
+ })),
959
+ ...(storyboard.outro
960
+ ? [{ id: "outro", title: storyboard.outro.title, durationMs: storyboard.outro.durationMs }]
961
+ : []),
962
+ ])
963
+ : null;
964
+ await muxAudio(project, storyboard, finalVideo, introMs, chapters);
373
965
  ok(`final video → ${project.outputPath}`);
966
+ let poster: string | undefined;
967
+ if (storyboard.output?.poster) {
968
+ poster = project.posterPath;
969
+ await runFfmpeg([
970
+ "-ss",
971
+ ((introMs + 500) / 1000).toFixed(3),
972
+ "-i",
973
+ project.outputPath,
974
+ "-frames:v",
975
+ "1",
976
+ "-update",
977
+ "1",
978
+ poster,
979
+ ]);
980
+ log(`poster → ${poster}`);
981
+ }
374
982
  // AIDEMO_KEEP_TMP=1 keeps .compose-tmp for debugging intermediates.
375
983
  if (!process.env.AIDEMO_KEEP_TMP) {
376
984
  await fs.rm(tmp, { recursive: true, force: true });
377
985
  }
986
+
987
+ const report: ComposeReport = {
988
+ output: project.outputPath,
989
+ durationMs: await probeDurationMs(project.outputPath).catch(() => 0),
990
+ scenes: sceneReports,
991
+ zoom: { focusTotal: zoomEvents.length, focusDropped },
992
+ cursorPoints: cursorPts.length,
993
+ captions: { cues: captionsResult.cues, passes: captionsResult.passes },
994
+ hold: hold.mode,
995
+ ...(poster ? { poster } : {}),
996
+ ...(attentionPlaced.length || keysPlaced.length || clicksPlaced.length || redactPlaced.length
997
+ ? {
998
+ attention: {
999
+ events: attentionPlaced.length,
1000
+ keys: keysPlaced.length,
1001
+ clicks: clicksPlaced.length,
1002
+ redactSpans: redactPlaced.length,
1003
+ captionsFlipped: captionsResult.flipped,
1004
+ },
1005
+ }
1006
+ : {}),
1007
+ warnings,
1008
+ };
1009
+ await writeJson(project.reportPath, report);
1010
+ const frozen = sceneReports.filter((s) => s.holdPct > FREEZE_WARN_PCT).length;
1011
+ log(
1012
+ `report → ${project.reportPath}` +
1013
+ (frozen ? ` (${frozen} of ${sceneReports.length} scene(s) mostly held)` : "")
1014
+ );
1015
+ if (warnings.length) {
1016
+ log(`⚠ ${warnings.length} compose warning(s):`);
1017
+ for (const w of warnings) log(` - [${w.code}] ${w.message}`);
1018
+ }
1019
+ return report;
1020
+ }
1021
+
1022
+ /** Share of a scene that may be a held frame before compose flags it. */
1023
+ const FREEZE_WARN_PCT = 0.4;
1024
+
1025
+ /**
1026
+ * Hold mode "drift": instead of cloning the scene's last frame for `holdMs`,
1027
+ * push in on it very slowly (1 → driftScale over the hold), so a long
1028
+ * narration over a static page reads as a deliberate dwell, not a stall.
1029
+ * The frame is pulled from the encoded scene's tail, rendered as a zoompan
1030
+ * still (pre-upscaled 2x so integer-pixel crops don't shimmer — the same rule
1031
+ * the auto-zoom pass follows), and re-encoded onto the scene. One tiny
1032
+ * expression, so the zoom budget is untouched; `zoompan`/`scale` are core.
1033
+ */
1034
+ async function appendDriftHold(
1035
+ scenePath: string,
1036
+ holdMs: number,
1037
+ driftScale: number,
1038
+ outW: number,
1039
+ outH: number,
1040
+ tmp: string,
1041
+ sceneIdx: number
1042
+ ): Promise<string> {
1043
+ const durMs = await probeDurationMs(scenePath);
1044
+ const framePng = resolve(tmp, `scene-${sceneIdx}-hold.png`);
1045
+ await runFfmpeg([
1046
+ "-ss",
1047
+ (Math.max(0, durMs - 60) / 1000).toFixed(3),
1048
+ "-i",
1049
+ scenePath,
1050
+ "-frames:v",
1051
+ "1",
1052
+ "-update",
1053
+ "1",
1054
+ framePng,
1055
+ ]);
1056
+ const holdSec = holdMs / 1000;
1057
+ const frames = Math.max(1, Math.round(holdSec * FPS));
1058
+ const holdMp4 = resolve(tmp, `scene-${sceneIdx}-hold.mp4`);
1059
+ // Ease the creep (ease-out quad) so it starts moving right away and settles.
1060
+ const z = `'1+(${driftScale.toFixed(4)}-1)*(1-pow(1-min(on/${frames}\,1)\,2))'`;
1061
+ await runFfmpeg([
1062
+ "-loop",
1063
+ "1",
1064
+ "-framerate",
1065
+ String(FPS),
1066
+ "-i",
1067
+ framePng,
1068
+ "-t",
1069
+ holdSec.toFixed(3),
1070
+ "-vf",
1071
+ `scale=iw*2:ih*2:flags=lanczos,` +
1072
+ `zoompan=z=${z}:x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=1:s=${outW}x${outH}:fps=${FPS},` +
1073
+ `setsar=1`,
1074
+ "-r",
1075
+ String(FPS),
1076
+ "-an",
1077
+ "-c:v",
1078
+ "libx264",
1079
+ "-preset",
1080
+ "veryfast",
1081
+ "-crf",
1082
+ "20",
1083
+ "-pix_fmt",
1084
+ "yuv420p",
1085
+ holdMp4,
1086
+ ]);
1087
+ // The PNG-sourced hold carries an edit list; re-encode the join so the pts
1088
+ // start clean at 0 (a stream-copy concat would leak negative timestamps
1089
+ // into the overlay passes — see AGENTS.md).
1090
+ return concatSegments(
1091
+ [scenePath, holdMp4],
1092
+ resolve(tmp, `scene-${sceneIdx}-held.mp4`),
1093
+ tmp,
1094
+ `scene-${sceneIdx}-held`,
1095
+ true
1096
+ );
378
1097
  }
379
1098
 
380
1099
  /**
@@ -384,7 +1103,7 @@ export async function compose(
384
1103
  * needs an API key so compose can't regenerate them itself; warn loudly and
385
1104
  * name the fix instead.
386
1105
  */
387
- async function warnStaleCaptions(project: Project): Promise<void> {
1106
+ async function warnStaleCaptions(project: Project): Promise<boolean> {
388
1107
  try {
389
1108
  const [cap, narr] = await Promise.all([
390
1109
  fs.stat(project.captionsCuesPath),
@@ -396,10 +1115,12 @@ async function warnStaleCaptions(project: Project): Promise<void> {
396
1115
  `the burned captions will not match the audio. ` +
397
1116
  `Fix: aidemo captions ${project.dir}` + ` (then re-run compose).`
398
1117
  );
1118
+ return true;
399
1119
  }
400
1120
  } catch {
401
1121
  /* captions or narration absent — burnCaptions handles missing captions */
402
1122
  }
1123
+ return false;
403
1124
  }
404
1125
 
405
1126
  /**
@@ -564,18 +1285,23 @@ async function cardSegment(
564
1285
  name: string,
565
1286
  tmp: string,
566
1287
  storyboard: Storyboard,
567
- outW: number,
568
- outH: number,
569
- pxScale: number
1288
+ logicalW: number,
1289
+ logicalH: number,
1290
+ pxScale: number,
1291
+ logoDataUri?: string
570
1292
  ): Promise<string> {
571
1293
  const png = resolve(tmp, `${name}.png`);
572
1294
  await renderCardPng(
573
1295
  card,
574
1296
  png,
575
- storyboard.video.width,
576
- storyboard.video.height,
577
- pxScale
1297
+ Math.round(logicalW),
1298
+ Math.round(logicalH),
1299
+ pxScale,
1300
+ storyboard.brand,
1301
+ logoDataUri
578
1302
  );
1303
+ const outW = Math.round(logicalW * pxScale) & ~1;
1304
+ const outH = Math.round(logicalH * pxScale) & ~1;
579
1305
  const durSec = card.durationMs / 1000;
580
1306
  const fadeSec = Math.min(card.fadeMs / 1000, durSec / 3);
581
1307
  const out = resolve(tmp, `${name}.mp4`);
@@ -609,6 +1335,42 @@ async function cardSegment(
609
1335
  return out;
610
1336
  }
611
1337
 
1338
+ /** Union of overlapping/adjacent idle spans (auto + annotated), sorted. */
1339
+ function mergeIdle(spans: IdleSpan[]): IdleSpan[] {
1340
+ const sorted = [...spans].sort((a, b) => a.startMs - b.startMs);
1341
+ const out: IdleSpan[] = [];
1342
+ for (const s of sorted) {
1343
+ const last = out[out.length - 1];
1344
+ if (last && s.startMs <= last.endMs) last.endMs = Math.max(last.endMs, s.endMs);
1345
+ else out.push({ ...s });
1346
+ }
1347
+ return out;
1348
+ }
1349
+
1350
+ /**
1351
+ * Motionless spans of the raw file (video time) → idle spans for one scene
1352
+ * (timeline time, the same base the player writes). Spans are clipped to the
1353
+ * scene, and the first/last 300 ms are left alone: the frame right after a
1354
+ * click and the one the scene ends on are the beats a viewer reads, and
1355
+ * trimming into them makes cuts feel clipped.
1356
+ */
1357
+ function sceneAutoIdle(
1358
+ freezes: FreezeSpan[],
1359
+ tl: Timeline["scenes"][number],
1360
+ leadInMs: number
1361
+ ): IdleSpan[] {
1362
+ const EDGE_MS = 300;
1363
+ const lo = tl.startMs + leadInMs + EDGE_MS;
1364
+ const hi = tl.endMs + leadInMs - EDGE_MS;
1365
+ const out: IdleSpan[] = [];
1366
+ for (const f of freezes) {
1367
+ const a = Math.max(f.startMs, lo);
1368
+ const b = Math.min(f.endMs, hi);
1369
+ if (b - a > 200) out.push({ startMs: a - leadInMs, endMs: b - leadInMs, label: "auto-idle" });
1370
+ }
1371
+ return out;
1372
+ }
1373
+
612
1374
  /**
613
1375
  * Build the list of [startMs,endMs] video-time spans to keep for a scene:
614
1376
  * non-idle content in full, each idle span capped to IDLE_CAP_MS.
@@ -641,6 +1403,14 @@ function keepIntervals(
641
1403
  return keeps.filter(([a, b]) => b - a > 20);
642
1404
  }
643
1405
 
1406
+ /** One retime piece: kept-span offsets [a,b) play at factor f, starting at stretched ms `out`. */
1407
+ interface RetimePiece {
1408
+ a: number;
1409
+ b: number;
1410
+ f: number;
1411
+ out: number;
1412
+ }
1413
+
644
1414
  /**
645
1415
  * Where a raw-video timestamp lands inside the concatenation of the kept
646
1416
  * spans. Timestamps inside a trimmed gap clamp to the gap's start.
@@ -723,67 +1493,159 @@ async function extractAndConcat(
723
1493
  */
724
1494
  const CAPTION_BOTTOM_GAP = 96;
725
1495
 
1496
+ /**
1497
+ * Max caption cues overlaid in a single ffmpeg pass. Each cue costs one input
1498
+ * and one filter link; ffmpeg fails to configure the graph somewhere past ~90
1499
+ * (issue #36), so long narrations are burned in several passes.
1500
+ */
1501
+ const CAPTION_BATCH = 32;
1502
+
1503
+ /**
1504
+ * Overlay many time-gated PNGs onto `src` in passes of ≤ CAPTION_BATCH. Each
1505
+ * item is one extra input plus one overlay link; ffmpeg fails to configure a
1506
+ * graph somewhere past ~90 of them (issue #36), so long lists are burned in
1507
+ * several re-encoding passes. Shared by captions, attention beats, key chips.
1508
+ */
1509
+ async function overlayPngBatches(
1510
+ src: string,
1511
+ items: OverlayItem[],
1512
+ tmp: string,
1513
+ label: string
1514
+ ): Promise<{ video: string; passes: number }> {
1515
+ if (items.length === 0) return { video: src, passes: 0 };
1516
+ const batches: OverlayItem[][] = [];
1517
+ for (let i = 0; i < items.length; i += CAPTION_BATCH) {
1518
+ batches.push(items.slice(i, i + CAPTION_BATCH));
1519
+ }
1520
+ if (batches.length > 1) {
1521
+ log(`${label} overlay: ${batches.length} pass(es) of <= ${CAPTION_BATCH} item(s)`);
1522
+ }
1523
+ let cur = src;
1524
+ let out = src;
1525
+ for (let b = 0; b < batches.length; b++) {
1526
+ const batch = batches[b];
1527
+ const args: string[] = ["-i", cur];
1528
+ for (const it of batch) args.push("-i", it.png);
1529
+ const chain: string[] = [];
1530
+ let prev = "0:v";
1531
+ batch.forEach((it, i) => {
1532
+ const inp = `${i + 1}:v`;
1533
+ const lbl = i === batch.length - 1 ? "vout" : `o${i}`;
1534
+ // Escape the enable-expression commas so ffmpeg doesn't read them as
1535
+ // filter separators.
1536
+ chain.push(
1537
+ `[${prev}][${inp}]overlay=${it.x}:${it.y}:enable=between(t\\,${it.a.toFixed(3)}\\,${it.b.toFixed(3)})[${lbl}]`
1538
+ );
1539
+ prev = lbl;
1540
+ });
1541
+ out = resolve(tmp, `${label}-${b}.mp4`);
1542
+ await runFfmpeg([
1543
+ ...args,
1544
+ "-filter_complex",
1545
+ chain.join(";"),
1546
+ "-map",
1547
+ "[vout]",
1548
+ "-an",
1549
+ "-c:v",
1550
+ "libx264",
1551
+ "-preset",
1552
+ "veryfast",
1553
+ "-crf",
1554
+ "20",
1555
+ "-pix_fmt",
1556
+ "yuv420p",
1557
+ out,
1558
+ ]);
1559
+ cur = out;
1560
+ }
1561
+ return { video: out, passes: batches.length };
1562
+ }
1563
+
1564
+ /** Height of an overlay PNG for the caption-collision check (spotlight → its hole). */
1565
+ function pngHeightHint(it: OverlayItem, placed: PlacedAttention[], pxScale: number): number {
1566
+ if (it.x === 0 && it.y === 0) {
1567
+ const ev = placed.find((p) => p.kind === "spotlight" && p.a === it.a && p.b === it.b);
1568
+ if (ev) return Math.round((ev.y + ev.h + (ev.padding ?? 10)) * pxScale);
1569
+ return 0;
1570
+ }
1571
+ return it.png.includes("/key-") ? 60 * pxScale : 48 * pxScale;
1572
+ }
1573
+
1574
+ interface CaptionPlacement {
1575
+ defaultPosition: "top" | "bottom";
1576
+ /** Caption look (style/font/size/color/background), brand font already merged. */
1577
+ look?: CaptionsConfig & { font?: string };
1578
+ /** Logical (CSS px) width of the frame the strip must span. */
1579
+ logicalWidth?: number;
1580
+ /** Content-time windows (ms, before the intro shift) with a fixed position. */
1581
+ windows: Array<{ a: number; b: number; position: "top" | "bottom" }>;
1582
+ /** Content-time windows (ms) during which the bottom band is occupied. */
1583
+ collisions: Array<{ a: number; b: number }>;
1584
+ }
1585
+
726
1586
  async function burnCaptions(
727
1587
  project: Project,
728
1588
  storyboard: Storyboard,
729
1589
  silentVideo: string,
730
1590
  tmp: string,
731
1591
  introMs: number,
732
- pxScale: number
733
- ): Promise<string> {
734
- if (!(await exists(project.captionsCuesPath))) return silentVideo;
1592
+ pxScale: number,
1593
+ placement: CaptionPlacement = { defaultPosition: "bottom", windows: [], collisions: [] }
1594
+ ): Promise<{ video: string; cues: number; passes: number; flipped: number }> {
1595
+ const none = { video: silentVideo, cues: 0, passes: 0, flipped: 0 };
1596
+ if (placement.look?.style === "none") {
1597
+ log("captions: style none — not burned (SRT/VTT files still written)");
1598
+ return none;
1599
+ }
1600
+ if (!(await exists(project.captionsCuesPath))) return none;
735
1601
  const cues = (await readJson<Cue[]>(project.captionsCuesPath)) ?? [];
736
- if (cues.length === 0) return silentVideo;
1602
+ if (cues.length === 0) return none;
737
1603
 
738
1604
  log(`rendering ${cues.length} caption image(s)`);
739
1605
  const rendered = await renderCaptionPngs(
740
1606
  cues,
741
1607
  resolve(tmp, "captions"),
742
- storyboard.video.width,
743
- pxScale
1608
+ Math.round(placement.logicalWidth ?? storyboard.video.width),
1609
+ pxScale,
1610
+ placement.look ?? {}
744
1611
  );
745
1612
 
746
- const args: string[] = ["-i", silentVideo];
747
- for (const r of rendered) args.push("-i", r.png);
748
-
749
- // Chain one overlay per cue, each enabled only during its time window.
750
- // Cue times are narration-relative; the intro card shifts them right.
751
- const gap = Math.round(CAPTION_BOTTOM_GAP * pxScale);
752
- const chain: string[] = [];
753
- let prev = "0:v";
754
- rendered.forEach((r, i) => {
755
- const inp = `${i + 1}:v`;
756
- const out = i === rendered.length - 1 ? "vout" : `o${i}`;
757
- const a = ((r.startMs + introMs) / 1000).toFixed(3);
758
- const b = ((r.endMs + introMs) / 1000).toFixed(3);
759
- // Escape the enable-expression commas so ffmpeg doesn't read them as filter
760
- // separators. Lift the strip clear of the app's bottom input bar (the
761
- // ChatGPT composer) so captions never sit on top of the prompt being typed.
762
- chain.push(
763
- `[${prev}][${inp}]overlay=0:H-h-${gap}:enable=between(t\\,${a}\\,${b})[${out}]`
764
- );
765
- prev = out;
1613
+ // Cue times are narration-relative; the intro card shifts them right. The
1614
+ // strip sits `gap` above the bottom edge (clear of an app's bottom input
1615
+ // bar) — or, per scene / on a collision with an attention overlay, at the
1616
+ // top: the PNG is a bottom-anchored strip, so "top" flips it vertically.
1617
+ // A "bar" strip is flush with the frame edge (no gap, no pill offset).
1618
+ const bar = placement.look?.style === "bar";
1619
+ const gap = bar ? 0 : Math.round(CAPTION_BOTTOM_GAP * pxScale);
1620
+ const { height: outH } = await probeVideoDims(silentVideo);
1621
+ let flipped = 0;
1622
+ const overlap = (a: number, b: number, w: { a: number; b: number }) => a < w.b && b > w.a;
1623
+ const items: OverlayItem[] = rendered.map((r) => {
1624
+ let pos = placement.defaultPosition;
1625
+ const win = placement.windows.find((w) => r.startMs >= w.a && r.startMs < w.b);
1626
+ if (win) pos = win.position;
1627
+ else if (pos === "bottom" && placement.collisions.some((c) => overlap(r.startMs, r.endMs, c))) {
1628
+ pos = "top";
1629
+ flipped++;
1630
+ }
1631
+ const stripH = Math.round(r.height * pxScale);
1632
+ // The strip PNG is bottom-anchored (pill bottom ≈ 22 px above the strip's
1633
+ // edge). For "top", slide the strip up so the pill's bottom lands ~gap+70
1634
+ // px from the top edge — a one- or two-line pill stays fully in frame.
1635
+ const topY = bar
1636
+ ? -(stripH - Math.round(62 * pxScale))
1637
+ : gap + Math.round(70 * pxScale) - (stripH - Math.round(22 * pxScale));
1638
+ return {
1639
+ png: r.png,
1640
+ x: 0,
1641
+ y: pos === "top" ? topY : outH - stripH - gap,
1642
+ a: (r.startMs + introMs) / 1000,
1643
+ b: (r.endMs + introMs) / 1000,
1644
+ };
766
1645
  });
767
-
768
- const out = resolve(tmp, "captioned.mp4");
769
- await runFfmpeg([
770
- ...args,
771
- "-filter_complex",
772
- chain.join(";"),
773
- "-map",
774
- "[vout]",
775
- "-an",
776
- "-c:v",
777
- "libx264",
778
- "-preset",
779
- "veryfast",
780
- "-crf",
781
- "20",
782
- "-pix_fmt",
783
- "yuv420p",
784
- out,
785
- ]);
786
- return out;
1646
+ if (flipped) log(`captions: ${flipped} cue(s) moved to the top (overlay in the bottom band)`);
1647
+ const r = await overlayPngBatches(silentVideo, items, tmp, "captioned");
1648
+ return { video: r.video, cues: cues.length, passes: r.passes, flipped };
787
1649
  }
788
1650
 
789
1651
  /**
@@ -795,11 +1657,51 @@ async function burnCaptions(
795
1657
  * fixed -22dB bed). The bed is trimmed to the video, faded out at the end,
796
1658
  * and the narration is delayed past the intro card.
797
1659
  */
1660
+ /** Output sizing with a preset's defaults filled in (explicit keys win). */
1661
+ function resolveOutputSizing(output: Output | undefined): Output {
1662
+ if (!output) return { fit: "contain" };
1663
+ const p = output.preset ? OUTPUT_PRESETS[output.preset] : null;
1664
+ return {
1665
+ ...output,
1666
+ width: output.width ?? p?.width,
1667
+ height: output.height ?? p?.height,
1668
+ fit: output.width != null ? output.fit : (p?.fit ?? output.fit),
1669
+ };
1670
+ }
1671
+
1672
+ /** Write an ffmetadata file with one chapter per scene (content time + intro). */
1673
+ async function writeChapters(
1674
+ tmp: string,
1675
+ scenes: Array<{ id: string; title: string; durationMs: number }>
1676
+ ): Promise<string> {
1677
+ const lines = [";FFMETADATA1"];
1678
+ let t = 0;
1679
+ for (const s of scenes) {
1680
+ const a = Math.round(t);
1681
+ const b = Math.round(t + s.durationMs);
1682
+ lines.push("[CHAPTER]", "TIMEBASE=1/1000", `START=${a}`, `END=${b}`, `title=${s.title.replace(/[\\=;#\n]/g, " ")}`);
1683
+ t += s.durationMs;
1684
+ }
1685
+ const path = resolve(tmp, "chapters.ffmeta");
1686
+ await fs.writeFile(path, lines.join("\n") + "\n");
1687
+ log(`chapters: ${scenes.length} marker(s)`);
1688
+ return path;
1689
+ }
1690
+
1691
+ async function fileToDataUri(path: string): Promise<string> {
1692
+ const ext = path.toLowerCase().split(".").pop() ?? "png";
1693
+ const mime =
1694
+ ext === "svg" ? "image/svg+xml" : ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
1695
+ const buf = await fs.readFile(path);
1696
+ return `data:${mime};base64,${buf.toString("base64")}`;
1697
+ }
1698
+
798
1699
  async function muxAudio(
799
1700
  project: Project,
800
1701
  storyboard: Storyboard,
801
1702
  video: string,
802
- introMs: number
1703
+ introMs: number,
1704
+ chaptersPath: string | null = null
803
1705
  ): Promise<void> {
804
1706
  let musicPath = await resolveMusic(project, storyboard);
805
1707
  const videoMs = await probeDurationMs(video);
@@ -819,9 +1721,15 @@ async function muxAudio(
819
1721
  }
820
1722
  }
821
1723
 
822
- // Inputs, fixed order: 0=video, 1=narration, 2=music (if any).
1724
+ // Inputs, fixed order: 0=video, 1=narration, 2=music (if any), then the
1725
+ // chapters metadata file (mapped by index, never a stream).
823
1726
  const args: string[] = ["-i", video, "-i", project.narrationPath];
824
1727
  if (musicPath) args.push("-stream_loop", String(musicLoops), "-i", musicPath);
1728
+ let chaptersIdx = -1;
1729
+ if (chaptersPath) {
1730
+ chaptersIdx = musicPath ? 3 : 2;
1731
+ args.push("-i", chaptersPath);
1732
+ }
825
1733
 
826
1734
  const delay = introMs > 0 ? `,adelay=${Math.round(introMs)}:all=1` : "";
827
1735
  const filters: string[] = [];
@@ -892,6 +1800,7 @@ async function muxAudio(
892
1800
 
893
1801
  args.push("-filter_complex", filters.join(";"));
894
1802
  args.push("-map", "0:v", "-map", "[aout]");
1803
+ if (chaptersIdx >= 0) args.push("-map_metadata", String(chaptersIdx));
895
1804
  args.push(
896
1805
  "-c:v",
897
1806
  "copy",
@@ -899,6 +1808,10 @@ async function muxAudio(
899
1808
  "aac",
900
1809
  "-b:a",
901
1810
  "192k",
1811
+ // moov atom up front so the MP4 streams/scrubs in a browser tab or a PR
1812
+ // preview before it has fully downloaded (frames identical; container only).
1813
+ "-movflags",
1814
+ "+faststart",
902
1815
  "-t",
903
1816
  durSec,
904
1817
  project.outputPath