@tandryukha/aidemo 0.8.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/compose.ts CHANGED
@@ -1,22 +1,64 @@
1
1
  import { promises as fs } from "node:fs";
2
- import { resolve, isAbsolute } from "node:path";
2
+ import { resolve, isAbsolute, relative } from "node:path";
3
3
  import { Project } from "./project.js";
4
- import type { Storyboard, Timeline, IdleSpan, Card, Output, Loudness } from "./types.js";
5
- import { TimelineSchema, VoiceManifestSchema, LoudnessSchema } from "./types.js";
6
- import type { Cue } from "./captions.js";
4
+ import type {
5
+ Storyboard,
6
+ Timeline,
7
+ IdleSpan,
8
+ Card,
9
+ Output,
10
+ Loudness,
11
+ ComposeReport,
12
+ ComposeSceneReport,
13
+ ComposeWarning,
14
+ } from "./types.js";
15
+ import {
16
+ TimelineSchema,
17
+ VoiceManifestSchema,
18
+ LoudnessSchema,
19
+ HoldSchema,
20
+ AutoIdleSchema,
21
+ } from "./types.js";
22
+ import { sceneWordsFor, type Cue } from "./captions.js";
23
+ import { wordStartMs } from "./anchors.js";
7
24
  import { renderCaptionPngs } from "./caption-render.js";
8
25
  import { renderCardPng } from "./cards.js";
9
26
  import { renderCursorPng } from "./cursor.js";
10
- import { buildZoomFilter, type ZoomEvent } from "./zoom.js";
27
+ import { buildZoomFilter, clampScaleForWidth, type ZoomEvent } from "./zoom.js";
11
28
  import {
12
29
  buildCursorFilter,
13
30
  type CursorPoint,
14
31
  type HideWindow,
15
32
  } from "./cursor-overlay.js";
16
- import { runFfmpeg, probeDurationMs, probeVideoDims } from "./ffmpeg.js";
33
+ import {
34
+ renderAttentionPngs,
35
+ renderKeyChipPngs,
36
+ renderClickRingPngs,
37
+ buildRedactFilter,
38
+ REDACT_BATCH,
39
+ KEY_CHIP_MS,
40
+ DEFAULT_ACCENT,
41
+ type OverlayItem,
42
+ type PlacedAttention,
43
+ type PlacedKey,
44
+ type PlacedClick,
45
+ type PlacedRedact,
46
+ } from "./attention.js";
47
+ import { renderFramePng } from "./frame.js";
48
+ import { OUTPUT_PRESETS, type CaptionsConfig } from "./types.js";
49
+ import {
50
+ runFfmpeg,
51
+ probeDurationMs,
52
+ probeFlatTailMs,
53
+ probeFlatLeadMs,
54
+ probeVideoDims,
55
+ probeFreezeSpans,
56
+ type FreezeSpan,
57
+ } from "./ffmpeg.js";
17
58
  import {
18
59
  ensureDir,
19
60
  readJson,
61
+ writeJson,
20
62
  exists,
21
63
  log,
22
64
  ok,
@@ -43,14 +85,33 @@ export async function compose(
43
85
  project: Project,
44
86
  storyboard: Storyboard,
45
87
  opts: SceneProgress = {}
46
- ): Promise<void> {
88
+ ): Promise<ComposeReport> {
47
89
  step("Composing final video (ffmpeg)");
48
90
 
49
91
  const timeline = TimelineSchema.parse(await readJson(project.timelinePath));
50
92
  const voice = VoiceManifestSchema.parse(
51
93
  await readJson(project.voiceManifestPath)
52
94
  );
53
- await warnStaleCaptions(project);
95
+ // Structured twin of the log lines below — see ComposeReportSchema.
96
+ const warnings: ComposeWarning[] = [];
97
+ const sceneReports: ComposeSceneReport[] = [];
98
+ const warn = (code: string, message: string, scene?: string): void => {
99
+ warnings.push({ code, message, ...(scene ? { scene } : {}) });
100
+ };
101
+ /**
102
+ * Frames to dump next to a warning (issue #59): an agent cannot look at the
103
+ * MP4, so "the reward is on screen for 0.3 s" is only actionable with the
104
+ * frame at that moment attached. Content-time ms; the intro shift is added
105
+ * when they are extracted, after the final mux.
106
+ */
107
+ const warningFrames: Array<{ name: string; atMs: number }> = [];
108
+ const hold = HoldSchema.parse(storyboard.hold ?? {});
109
+ if (await warnStaleCaptions(project)) {
110
+ warn(
111
+ "stale-captions",
112
+ `narration.mp3 is newer than the caption files — re-run: aidemo captions ${project.dir}`
113
+ );
114
+ }
54
115
  const tmp = project.composeTmpDir;
55
116
  await fs.rm(tmp, { recursive: true, force: true });
56
117
  await ensureDir(tmp);
@@ -63,6 +124,41 @@ export async function compose(
63
124
  const { width: outW, height: outH } = await probeVideoDims(rawVideo);
64
125
  const pxScale = outW / storyboard.video.width;
65
126
 
127
+ // --- autoIdle / multi-source plumbing -------------------------------------
128
+ // A resumed take splices scenes from more than one raw file; every source
129
+ // must share the main take's pixel size or all the overlay math (captions,
130
+ // zoom, cursor — all scaled by pxScale) silently misaligns on those scenes.
131
+ const dimsChecked = new Set<string>([rawVideo]);
132
+ const assertSourceDims = async (file: string): Promise<void> => {
133
+ if (dimsChecked.has(file)) return;
134
+ dimsChecked.add(file);
135
+ const d = await probeVideoDims(file);
136
+ if (d.width !== outW || d.height !== outH) {
137
+ throw new Error(
138
+ `resumed take mixes recordings of different sizes: ${relative(project.dir, file)} is ` +
139
+ `${d.width}x${d.height}, the current take is ${outW}x${outH} — re-record the whole demo ` +
140
+ `(a resume must use the same capture mode and viewport as the take it continues)`
141
+ );
142
+ }
143
+ };
144
+ const autoIdleCfg = AutoIdleSchema.parse(
145
+ typeof storyboard.autoIdle === "object" ? storyboard.autoIdle : {}
146
+ );
147
+ const autoIdleOn =
148
+ storyboard.autoIdle === true ||
149
+ (typeof storyboard.autoIdle === "object" && storyboard.autoIdle.enabled !== false);
150
+ const autoIdleFor = (scene: { autoIdle?: boolean } | undefined): boolean =>
151
+ scene?.autoIdle ?? autoIdleOn;
152
+ const freezeCache = new Map<string, Promise<FreezeSpan[]>>();
153
+ const freezeSpansFor = (file: string): Promise<FreezeSpan[]> => {
154
+ let p = freezeCache.get(file);
155
+ if (!p) {
156
+ p = probeFreezeSpans(file, autoIdleCfg.minMs ?? 1500, autoIdleCfg.noise ?? 0.003);
157
+ freezeCache.set(file, p);
158
+ }
159
+ return p;
160
+ };
161
+
66
162
  // Map scene id -> narration duration; compose target = narration + gap.
67
163
  const voiceById = new Map(voice.scenes.map((s) => [s.id, s.durationMs]));
68
164
  const sceneById = new Map(storyboard.scenes.map((s) => [s.id, s]));
@@ -80,8 +176,24 @@ export async function compose(
80
176
  const cursorPts: CursorPoint[] = [];
81
177
  const hideWindows: HideWindow[] = [];
82
178
  let cursorSampleCount = 0;
179
+ // Attention layer accumulators (all opt-in; empty for a plain storyboard).
180
+ const accent = storyboard.attention?.color ?? storyboard.brand?.accent ?? DEFAULT_ACCENT;
181
+ const attentionPlaced: PlacedAttention[] = [];
182
+ const keysPlaced: PlacedKey[] = [];
183
+ const clicksPlaced: PlacedClick[] = [];
184
+ const redactPlaced: PlacedRedact[] = [];
185
+ const clickRings = !!(storyboard.attention?.clicks && cursorCfg);
186
+ /** Per-scene caption placement windows (content ms, before the intro shift). */
187
+ const captionWindows: Array<{ a: number; b: number; position: "top" | "bottom" }> = [];
188
+ // Dot cursors are centered on the point; the arrow PNG's tip is its top-left.
189
+ const dotOffset =
190
+ cursorCfg?.style === "dot" ? Math.round(16 * pxScale * (cursorCfg.scale ?? 1)) : 0;
83
191
 
84
192
  const sceneVideos: string[] = [];
193
+ /** timeline scenes that actually produced a segment, parallel to sceneVideos. */
194
+ const composedTl: Timeline["scenes"] = [];
195
+ /** content-time span of each composed scene, for `frame.url: "auto"`. */
196
+ const sceneSpans: Array<{ a: number; b: number }> = [];
85
197
  const sceneTotal = timeline.scenes.length;
86
198
  for (let i = 0; i < timeline.scenes.length; i++) {
87
199
  const tl = timeline.scenes[i];
@@ -90,22 +202,161 @@ export async function compose(
90
202
  const targetMs = narrMs + voice.gapMs;
91
203
  if (targetMs <= 0) {
92
204
  log(`scene ${tl.id}: no narration, skipping`);
205
+ warn("scene-no-narration", `scene ${tl.id} has no narration audio and was skipped`, tl.id);
93
206
  opts.onSceneComplete?.(tl.id, i, sceneTotal);
94
207
  continue;
95
208
  }
96
209
 
97
- const keeps = keepIntervals(tl, timeline.leadInMs);
210
+ let blankTrimMs = 0;
211
+ // A resumed take's reused scenes live in an earlier raw file with its own lead-in.
212
+ const leadInMs = tl.leadInMs ?? timeline.leadInMs;
213
+ const sceneRaw = tl.source ? resolve(project.dir, tl.source) : rawVideo;
214
+ await assertSourceDims(sceneRaw);
215
+ // `autoIdle`: motionless spans of the raw take count as idle too, so dead
216
+ // air the storyboard never annotated gets the same cap as a `waitFor`.
217
+ const autoSpans = autoIdleFor(sceneById.get(tl.id))
218
+ ? sceneAutoIdle(await freezeSpansFor(sceneRaw), tl, leadInMs)
219
+ : [];
220
+ if (autoSpans.length) {
221
+ const dead = autoSpans.reduce((a, sp) => a + (sp.endMs - sp.startMs), 0);
222
+ log(`scene ${tl.id}: autoIdle found ${autoSpans.length} motionless span(s), ${dead}ms`);
223
+ }
224
+ const keeps = keepIntervals(
225
+ autoSpans.length ? { ...tl, idleSpans: mergeIdle([...tl.idleSpans, ...autoSpans]) } : tl,
226
+ leadInMs
227
+ );
98
228
  const rawSegPath = resolve(tmp, `scene-${i}-raw.mp4`);
99
- await extractAndConcat(rawVideo, keeps, rawSegPath, tmp, i);
229
+ await extractAndConcat(sceneRaw, keeps, rawSegPath, tmp, i);
100
230
 
101
- const srcMs = await probeDurationMs(rawSegPath);
231
+ let srcMs = await probeDurationMs(rawSegPath);
232
+ // #47: the take opens on the browser's white pre-paint (the `goto` load),
233
+ // so the first caption plays over a blank page and `poster` grabs white.
234
+ // Trim it down to a sliver — opt-in, so an unchanged storyboard renders
235
+ // byte-for-byte as before.
236
+ let leadTrimMs = 0;
237
+ if (i === 0 && storyboard.output?.trimLeadingBlank && srcMs > 1000) {
238
+ const cap = storyboard.output.blankOpenCapMs ?? 100;
239
+ const flatLead = await probeFlatLeadMs(rawSegPath);
240
+ if (flatLead > cap + 80) {
241
+ leadTrimMs = Math.min(flatLead - cap, srcMs - 500);
242
+ srcMs -= leadTrimMs;
243
+ log(
244
+ `scene ${tl.id}: trimmed ${Math.round(leadTrimMs)}ms of blank opening ` +
245
+ `(kept ${cap}ms of it)`
246
+ );
247
+ }
248
+ }
249
+ // A scene that ends right before a `goto` can trail off into the browser's
250
+ // white pre-paint. Freeze-holding THAT frame paints seconds of solid white
251
+ // (issue #36), so drop a flat tail before deciding how long to hold —
252
+ // but only when we would actually hold, and never the whole segment.
253
+ if (srcMs > 0 && targetMs > srcMs + 400) {
254
+ const flatMs = await probeFlatTailMs(rawSegPath, srcMs);
255
+ const trimTo = srcMs - flatMs;
256
+ if (flatMs > 200 && trimTo > 500) {
257
+ blankTrimMs = flatMs;
258
+ srcMs = trimTo;
259
+ }
260
+ }
261
+ /**
262
+ * Raw video ms → offset inside this scene's kept, head-trimmed footage.
263
+ * Every event remap goes through it, so the leading-blank trim shifts the
264
+ * whole scene's timeline consistently.
265
+ */
266
+ const localIn = (rawT: number): number =>
267
+ Math.max(0, offsetInKeeps(keeps, rawT) - leadTrimMs);
102
268
  // Retime the segment toward the narration length, but only within
103
269
  // [MIN_FACTOR, MAX_STRETCH]. If narration still needs more time, hold
104
270
  // (freeze) the last frame for the remainder — natural for a static page,
105
271
  // and far better than 3x slow-motion.
106
272
  const ratio = targetMs / Math.max(srcMs, 1);
107
- const factor = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, ratio));
108
- let stretchedMs = factor * srcMs;
273
+ const uniformFactor = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, ratio));
274
+
275
+ // Narration anchors → piecewise retime. Each anchored action is a raw
276
+ // offset (inside the kept spans) that must land on the ms its word is
277
+ // spoken; between anchors the factor is whatever gets there (clamped to
278
+ // the same [MIN_FACTOR, MAX_STRETCH], so a beat that can't be reached is
279
+ // reported, never faked). No anchors → one piece, the uniform factor —
280
+ // arithmetically identical to the pre-anchor path.
281
+ const sceneDef = sceneById.get(tl.id);
282
+ const anchorDefs = sceneDef?.anchors ?? {};
283
+ const anchorPts: Array<{ name: string; raw: number; target: number }> = [];
284
+ if (Object.keys(anchorDefs).length && (tl.anchorEvents ?? []).length) {
285
+ const words = await sceneWordsFor(project, tl.id, sceneDef?.narration ?? "", narrMs);
286
+ for (const ev of tl.anchorEvents) {
287
+ const wi = anchorDefs[ev.name];
288
+ if (wi == null) continue;
289
+ const target = wordStartMs(words, sceneDef?.narration ?? "", wi);
290
+ if (target == null) continue;
291
+ const raw = localIn(ev.tMs + leadInMs);
292
+ if (raw > srcMs - 40) {
293
+ warn(
294
+ "anchor-unreachable",
295
+ `scene ${tl.id}: anchor "${ev.name}" fires at the very end of the take (${Math.round(raw)}ms of ${Math.round(srcMs)}ms) — add a pause or a later action after it so the beat has room`,
296
+ tl.id
297
+ );
298
+ continue;
299
+ }
300
+ anchorPts.push({ name: ev.name, raw, target });
301
+ }
302
+ anchorPts.sort((a, b) => a.raw - b.raw);
303
+ }
304
+ const pieces: RetimePiece[] = [];
305
+ const anchorReport: NonNullable<ComposeSceneReport["anchors"]> = [];
306
+ if (anchorPts.length) {
307
+ let out = 0;
308
+ let prevRaw = 0;
309
+ const bounds = [
310
+ ...anchorPts.map((a) => ({ raw: a.raw, target: a.target, name: a.name as string | null })),
311
+ { raw: srcMs, target: targetMs, name: null as string | null },
312
+ ];
313
+ for (const bnd of bounds) {
314
+ const len = bnd.raw - prevRaw;
315
+ if (len >= 40) {
316
+ const want = bnd.target - out;
317
+ const f = Math.min(MAX_STRETCH, Math.max(MIN_FACTOR, want / len));
318
+ pieces.push({ a: prevRaw, b: bnd.raw, f, out });
319
+ out += len * f;
320
+ prevRaw = bnd.raw;
321
+ }
322
+ if (bnd.name) {
323
+ const offMs = Math.round(out - bnd.target);
324
+ anchorReport.push({
325
+ name: bnd.name,
326
+ targetMs: Math.round(bnd.target),
327
+ landedMs: Math.round(out),
328
+ offMs,
329
+ });
330
+ if (Math.abs(offMs) > 150) {
331
+ warningFrames.push({
332
+ name: `${tl.id}-${bnd.name}`,
333
+ atMs: outCursorMs + Math.max(0, Math.min(out, targetMs)),
334
+ });
335
+ warn(
336
+ "anchor-unreachable",
337
+ `scene ${tl.id}: anchor "${bnd.name}" lands ${Math.abs(offMs)}ms ${offMs > 0 ? "after" : "before"} its word even at ${offMs > 0 ? "2x speed" : "x1.6 slow-motion"} — ${
338
+ offMs > 0
339
+ ? "trim what happens before the action (mark waits idle, drop a scroll) or move the {{@" + bnd.name + "}} marker later"
340
+ : "give the action more lead-in (a pause, a hover) or move the {{@" + bnd.name + "}} marker earlier"
341
+ }`,
342
+ tl.id
343
+ );
344
+ }
345
+ }
346
+ }
347
+ }
348
+ if (!pieces.length) pieces.push({ a: 0, b: srcMs, f: uniformFactor, out: 0 });
349
+ const anchored = anchorPts.length > 0;
350
+ let stretchedMs = anchored
351
+ ? pieces[pieces.length - 1].out + (pieces[pieces.length - 1].b - pieces[pieces.length - 1].a) * pieces[pieces.length - 1].f
352
+ : uniformFactor * srcMs;
353
+ const factor = anchored ? stretchedMs / Math.max(srcMs, 1) : uniformFactor;
354
+ /** Kept-span offset (raw ms inside the concatenated keeps) → stretched scene ms. */
355
+ const stretchLocal = (local: number): number => {
356
+ let pc = pieces[0];
357
+ for (const q of pieces) if (local >= q.a) pc = q;
358
+ return pc.out + (local - pc.a) * pc.f;
359
+ };
109
360
  // A scene whose active video exceeds 2x its narration would OVERRUN its
110
361
  // narration slot — the narration track is fixed, so every later scene's
111
362
  // video drifts behind its audio and the finale slides past the end of the
@@ -118,20 +369,77 @@ export async function compose(
118
369
  stretchedMs = targetMs;
119
370
  }
120
371
  const holdMs = Math.max(0, targetMs - stretchedMs);
121
- let vf = `setpts=${factor.toFixed(6)}*PTS`;
372
+ const holding = holdMs > 40;
373
+ // `hold.backoffMs` takes the held frame from slightly before the segment
374
+ // end (a mid-flight hover, a half-painted transition); the hold grows by
375
+ // the same amount so the scene still fills its narration slot exactly.
376
+ const backoffMs = holding
377
+ ? Math.min(hold.backoffMs, Math.max(0, stretchedMs - 500))
378
+ : 0;
379
+ // Drift only where it reads as a camera move (issue #57): a short hold
380
+ // bobs, and inside a composited window chrome the page appears to slide
381
+ // within a frame that does not move with it.
382
+ const framedChrome =
383
+ !!storyboard.frame && (storyboard.frame.chrome ?? "none") !== "none";
384
+ const driftSuppressed =
385
+ holding && hold.mode === "drift"
386
+ ? holdMs < hold.minDriftMs
387
+ ? `hold is ${Math.round(holdMs)}ms (< hold.minDriftMs ${hold.minDriftMs}ms)`
388
+ : framedChrome && !hold.driftInFrame
389
+ ? `frame.chrome "${storyboard.frame!.chrome}" is static — the drift would read as the page sliding inside it (set hold.driftInFrame to override)`
390
+ : null
391
+ : null;
392
+ if (driftSuppressed) log(`scene ${tl.id}: drift hold → freeze (${driftSuppressed})`);
393
+ const driftHold = holding && hold.mode === "drift" && !driftSuppressed;
394
+ let vf = "";
395
+ if (leadTrimMs > 0) {
396
+ vf += `trim=start=${(leadTrimMs / 1000).toFixed(3)},setpts=PTS-STARTPTS,`;
397
+ }
398
+ if (blankTrimMs > 0) {
399
+ vf += `trim=duration=${(srcMs / 1000).toFixed(3)},setpts=PTS-STARTPTS,`;
400
+ }
401
+ // Tail (shared by the uniform and the piecewise path): overrun trim,
402
+ // backoff trim, freeze hold.
403
+ let tail = "";
122
404
  if (overrunTrimMs > 0) {
123
- vf += `,trim=duration=${(targetMs / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
405
+ tail += `,trim=duration=${(targetMs / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
406
+ }
407
+ if (backoffMs > 0) {
408
+ tail += `,trim=duration=${((stretchedMs - backoffMs) / 1000).toFixed(3)},setpts=PTS-STARTPTS`;
124
409
  }
125
- if (holdMs > 40) {
126
- vf += `,tpad=stop_mode=clone:stop_duration=${(holdMs / 1000).toFixed(3)}`;
410
+ if (holding && !driftHold) {
411
+ tail += `,tpad=stop_mode=clone:stop_duration=${((holdMs + backoffMs) / 1000).toFixed(3)}`;
412
+ }
413
+ let filterArgs: string[];
414
+ if (anchored) {
415
+ // split → per-piece trim + setpts → concat. Same source, same format,
416
+ // so concat is a plain join; everything stays in the core filter set.
417
+ const n = pieces.length;
418
+ const splitLabels = pieces.map((_, k) => `[s${k}]`).join("");
419
+ const pieceFilters = pieces
420
+ .map(
421
+ (pc, k) =>
422
+ `[s${k}]trim=start=${(pc.a / 1000).toFixed(3)}:end=${(pc.b / 1000).toFixed(3)},` +
423
+ `setpts=(PTS-STARTPTS)*${pc.f.toFixed(6)}[p${k}]`
424
+ )
425
+ .join(";");
426
+ const concatIn = pieces.map((_, k) => `[p${k}]`).join("");
427
+ const fc =
428
+ `[0:v]${vf}split=${n}${splitLabels};${pieceFilters};` +
429
+ `${concatIn}concat=n=${n}:v=1:a=0,setpts=PTS-STARTPTS${tail}[vout]`;
430
+ filterArgs = ["-filter_complex", fc, "-map", "[vout]"];
431
+ } else {
432
+ filterArgs = ["-vf", `${vf}setpts=${factor.toFixed(6)}*PTS${tail}`];
127
433
  }
128
434
 
129
435
  // Map this scene's focus events into final-video time for the zoom pass:
130
436
  // raw time → offset inside the kept spans → stretched → + scene offset.
437
+ let sceneFocus = 0;
131
438
  if (zoomCfg && sceneById.get(tl.id)?.zoom !== false) {
132
439
  for (const ev of tl.focusEvents ?? []) {
133
- const rawT = ev.tMs + timeline.leadInMs;
134
- const local = offsetInKeeps(keeps, rawT) * factor;
440
+ sceneFocus++;
441
+ const rawT = ev.tMs + leadInMs;
442
+ const local = stretchLocal(localIn(rawT));
135
443
  zoomEvents.push({
136
444
  tMs: outCursorMs + Math.min(local, stretchedMs),
137
445
  x: ev.x * pxScale,
@@ -148,12 +456,12 @@ export async function compose(
148
456
  if (cursorCfg) {
149
457
  const sceneStart = outCursorMs;
150
458
  for (const s of tl.cursorSamples ?? []) {
151
- const rawT = s.tMs + timeline.leadInMs;
152
- const local = offsetInKeeps(keeps, rawT) * factor;
459
+ const rawT = s.tMs + leadInMs;
460
+ const local = stretchLocal(localIn(rawT));
153
461
  cursorPts.push({
154
462
  t: (outCursorMs + Math.min(local, stretchedMs)) / 1000,
155
- x: s.x * pxScale,
156
- y: s.y * pxScale,
463
+ x: s.x * pxScale - dotOffset,
464
+ y: s.y * pxScale - dotOffset,
157
465
  });
158
466
  }
159
467
  cursorSampleCount += (tl.cursorSamples ?? []).length;
@@ -163,14 +471,50 @@ export async function compose(
163
471
  }
164
472
  }
165
473
 
474
+ // Attention beats, key chips, click rings and redact spans: same raw →
475
+ // content-time remap; holds are content-time and clipped to the scene.
476
+ {
477
+ const sceneStart = outCursorMs;
478
+ const sceneEnd = outCursorMs + stretchedMs + (holdMs > 40 ? holdMs : 0);
479
+ const toContent = (rawMs: number): number =>
480
+ sceneStart + Math.min(stretchLocal(localIn(rawMs + leadInMs)), stretchedMs);
481
+ for (const ev of tl.attentionEvents ?? []) {
482
+ const a = toContent(ev.tMs);
483
+ const b = Math.min(sceneEnd, a + ev.holdMs);
484
+ if (b - a < 80) continue;
485
+ attentionPlaced.push({ ...ev, a: a / 1000, b: b / 1000 });
486
+ }
487
+ for (const k of tl.keyEvents ?? []) {
488
+ const a = toContent(k.tMs);
489
+ const b = Math.min(sceneEnd, a + KEY_CHIP_MS);
490
+ if (b - a < 80) continue;
491
+ keysPlaced.push({ keys: k.keys, a: a / 1000, b: b / 1000 });
492
+ }
493
+ if (clickRings) {
494
+ for (const f of tl.focusEvents ?? []) {
495
+ if (f.kind !== "click") continue;
496
+ clicksPlaced.push({ x: f.x, y: f.y, t: toContent(f.tMs) / 1000 });
497
+ }
498
+ }
499
+ for (const r of tl.redactSpans ?? []) {
500
+ const a = toContent(r.startMs);
501
+ // A span open at the scene end covers the hold too.
502
+ const b = r.endMs >= tl.endMs - 5 ? sceneEnd : toContent(r.endMs);
503
+ if (b - a < 40) continue;
504
+ redactPlaced.push({ ...r, a: a / 1000, b: b / 1000 });
505
+ }
506
+ const pos = sceneById.get(tl.id)?.captions?.position;
507
+ if (pos) captionWindows.push({ a: sceneStart, b: sceneEnd, position: pos });
508
+ }
509
+
510
+ const sceneContentStartMs = outCursorMs;
166
511
  outCursorMs += stretchedMs + (holdMs > 40 ? holdMs : 0);
167
512
 
168
- const scenePath = resolve(tmp, `scene-${i}.mp4`);
513
+ let scenePath = resolve(tmp, `scene-${i}.mp4`);
169
514
  await runFfmpeg([
170
515
  "-i",
171
516
  rawSegPath,
172
- "-vf",
173
- vf,
517
+ ...filterArgs,
174
518
  "-an",
175
519
  "-r",
176
520
  String(FPS),
@@ -184,13 +528,89 @@ export async function compose(
184
528
  "yuv420p",
185
529
  scenePath,
186
530
  ]);
531
+ if (driftHold) {
532
+ scenePath = await appendDriftHold(
533
+ scenePath,
534
+ holdMs + backoffMs,
535
+ hold.driftScale,
536
+ outW,
537
+ outH,
538
+ tmp,
539
+ i
540
+ );
541
+ }
187
542
  sceneVideos.push(scenePath);
543
+ composedTl.push(tl);
544
+ sceneSpans.push({ a: sceneContentStartMs, b: outCursorMs });
545
+ const holdPct = holding ? holdMs / targetMs : 0;
188
546
  log(
189
547
  `scene ${tl.id}: ${srcMs}ms -> ${targetMs}ms ` +
190
- `(x${factor.toFixed(2)}${holdMs > 40 ? ` + ${Math.round(holdMs)}ms hold` : ""}${
548
+ `(x${factor.toFixed(2)}${
549
+ holding
550
+ ? ` + ${Math.round(holdMs)}ms ${driftHold ? "drift " : ""}hold` +
551
+ (backoffMs > 0 ? ` (backoff ${backoffMs}ms)` : "")
552
+ : ""
553
+ }${
191
554
  overrunTrimMs > 0 ? ` - ${Math.round(overrunTrimMs)}ms tail trim` : ""
192
- }, ${keeps.length} span(s))`
555
+ }${blankTrimMs > 0 ? ` - ${Math.round(blankTrimMs)}ms blank tail` : ""}, ${keeps.length} span(s)${
556
+ anchored
557
+ ? `, ${pieces.length} piece(s): ` +
558
+ anchorReport.map((a) => `${a.name} ${a.offMs >= 0 ? "+" : ""}${a.offMs}ms`).join(" ")
559
+ : ""
560
+ })`
193
561
  );
562
+ sceneReports.push({
563
+ id: tl.id,
564
+ srcMs: Math.round(srcMs),
565
+ targetMs: Math.round(targetMs),
566
+ factor: Number(factor.toFixed(3)),
567
+ holdMs: Math.round(holding ? holdMs : 0),
568
+ holdPct: Number(holdPct.toFixed(3)),
569
+ tailTrimMs: Math.round(overrunTrimMs),
570
+ blankTrimMs: Math.round(blankTrimMs),
571
+ autoIdleMs: Math.round(autoSpans.reduce((a, sp) => a + (sp.endMs - sp.startMs), 0)),
572
+ spans: keeps.length,
573
+ focusEvents: sceneFocus,
574
+ ...(anchorReport.length ? { anchors: anchorReport } : {}),
575
+ });
576
+ if (holdPct > FREEZE_WARN_PCT) {
577
+ // Name the exact frame being held (issue #55): when the wrong one is
578
+ // frozen — the next page's load, a half-painted card — the only way to
579
+ // see it today is to extract frames by hand.
580
+ const heldRaw = rawAtKeepOffset(
581
+ keeps,
582
+ leadTrimMs + Math.max(0, srcMs - backoffMs / Math.max(factor, 0.01))
583
+ );
584
+ warningFrames.push({
585
+ name: `${tl.id}-held`,
586
+ atMs: sceneContentStartMs + stretchedMs + 200,
587
+ });
588
+ warn(
589
+ "scene-freeze",
590
+ `scene ${tl.id}: ${Math.round(holdPct * 100)}% of its ${(targetMs / 1000).toFixed(1)}s ` +
591
+ `is a held frame (${(srcMs / 1000).toFixed(1)}s of action for ` +
592
+ `${(targetMs / 1000).toFixed(1)}s of narration; the held frame is ` +
593
+ `${relative(project.dir, sceneRaw)} @ ${stampMs(heldRaw)}) — add on-screen action ` +
594
+ `(hover/scroll/focus beats, a pause) or shorten the narration`,
595
+ tl.id
596
+ );
597
+ }
598
+ if (overrunTrimMs > 0) {
599
+ warn(
600
+ "overrun-trim",
601
+ `scene ${tl.id}: ${Math.round(overrunTrimMs)}ms of recorded action was cut from ` +
602
+ `the tail — the action outran the narration even at 2x; mark waits idle or ` +
603
+ `lengthen the narration`,
604
+ tl.id
605
+ );
606
+ }
607
+ if (blankTrimMs > 0) {
608
+ warn(
609
+ "blank-tail",
610
+ `scene ${tl.id}: dropped a ${Math.round(blankTrimMs)}ms solid-color tail before holding`,
611
+ tl.id
612
+ );
613
+ }
194
614
  opts.onSceneComplete?.(tl.id, i, sceneTotal);
195
615
  }
196
616
 
@@ -201,14 +621,43 @@ export async function compose(
201
621
  // scene boundaries (re-encodes the join; total duration is preserved).
202
622
  let content: string;
203
623
  if (storyboard.transition && sceneVideos.length >= 2) {
204
- const durMs = storyboard.transition.durationMs;
624
+ const baseDur = storyboard.transition.durationMs;
625
+ // Per-boundary duration; 0 = hard cut. A boundary that crosses a page
626
+ // navigation dissolves the old page over a still-painting new one — a
627
+ // ghost double-image (issue #49) — so cut those, and honour an explicit
628
+ // per-scene override either way.
629
+ const durations: number[] = [];
630
+ const autoCut: string[] = [];
631
+ for (let b = 1; b < sceneVideos.length; b++) {
632
+ const scene = sceneById.get(composedTl[b].id);
633
+ if (scene?.transition === false) {
634
+ durations.push(0);
635
+ continue;
636
+ }
637
+ if (scene?.transition) {
638
+ durations.push(scene.transition.durationMs);
639
+ continue;
640
+ }
641
+ if (crossesNavigation(composedTl[b - 1], composedTl[b])) {
642
+ durations.push(0);
643
+ autoCut.push(composedTl[b].id);
644
+ continue;
645
+ }
646
+ durations.push(baseDur);
647
+ }
648
+ const fades = durations.filter((d) => d > 0).length;
205
649
  log(
206
- `scene crossfade: ${durMs}ms x ${sceneVideos.length - 1} boundary/ies`
650
+ `scene crossfade: ${baseDur}ms x ${fades} boundary/ies` +
651
+ (durations.length - fades
652
+ ? `, ${durations.length - fades} hard cut(s)` +
653
+ (autoCut.length ? ` (navigation into ${autoCut.join(", ")})` : "")
654
+ : "")
207
655
  );
208
656
  content = await crossfadeScenes(
209
657
  sceneVideos,
210
- durMs,
211
- resolve(tmp, "content.mp4")
658
+ durations,
659
+ resolve(tmp, "content.mp4"),
660
+ tmp
212
661
  );
213
662
  } else {
214
663
  content = await concatSegments(
@@ -219,6 +668,66 @@ export async function compose(
219
668
  );
220
669
  }
221
670
 
671
+ // Redaction first: blur boxes ride every later pass (cursor, zoom, blur).
672
+ if (redactPlaced.length) {
673
+ let src = content;
674
+ let passes = 0;
675
+ for (let i = 0; i < redactPlaced.length; i += REDACT_BATCH) {
676
+ const batch = redactPlaced.slice(i, i + REDACT_BATCH);
677
+ const filter = buildRedactFilter(batch, outW, outH, pxScale);
678
+ if (!filter) continue;
679
+ const out = resolve(tmp, `content-redact-${passes}.mp4`);
680
+ await runFfmpeg([
681
+ "-i",
682
+ src,
683
+ "-filter_complex",
684
+ filter,
685
+ "-map",
686
+ "[vout]",
687
+ "-an",
688
+ "-r",
689
+ String(FPS),
690
+ "-c:v",
691
+ "libx264",
692
+ "-preset",
693
+ "veryfast",
694
+ "-crf",
695
+ "20",
696
+ "-pix_fmt",
697
+ "yuv420p",
698
+ out,
699
+ ]);
700
+ src = out;
701
+ passes++;
702
+ }
703
+ content = src;
704
+ log(`redact: ${redactPlaced.length} span(s) blurred in ${passes} pass(es)`);
705
+ }
706
+
707
+ // Attention overlays (highlight / spotlight / callout / click rings) go on
708
+ // BEFORE the cursor and the zoom, so they ride the camera like page content.
709
+ let attentionItems: OverlayItem[] = [];
710
+ if (attentionPlaced.length || clicksPlaced.length) {
711
+ attentionItems = [
712
+ ...(await renderAttentionPngs(
713
+ attentionPlaced,
714
+ resolve(tmp, "attention"),
715
+ storyboard.video.width,
716
+ storyboard.video.height,
717
+ pxScale,
718
+ accent
719
+ )),
720
+ ...(await renderClickRingPngs(clicksPlaced, resolve(tmp, "attention"), pxScale, accent)),
721
+ ];
722
+ const r = await overlayPngBatches(content, attentionItems, tmp, "attention");
723
+ content = r.video;
724
+ log(
725
+ `attention: ${attentionPlaced.length} beat(s)` +
726
+ (clicksPlaced.length ? `, ${clicksPlaced.length} click ring(s)` : "") +
727
+ ` in ${r.passes} pass(es)`
728
+ );
729
+ }
730
+
222
731
  // Compose-time cursor overlay (opt-in `cursor` block). Draw the recorded
223
732
  // cursor path onto the (cursor-free) content BEFORE the zoom pass, so the
224
733
  // cursor zooms and pans with the frame exactly like a baked one would.
@@ -230,6 +739,10 @@ export async function compose(
230
739
  "has no recorded cursor path — re-run `record`/`render` with the cursor " +
231
740
  "block present (the current recording may still show a baked cursor)."
232
741
  );
742
+ warn(
743
+ "cursor-missing",
744
+ "cursor block present but the take has no recorded cursor path — re-record with the block"
745
+ );
233
746
  } else if (cursorCfg.hidden) {
234
747
  log("cursor: hidden (no overlay drawn)");
235
748
  } else {
@@ -237,7 +750,17 @@ export async function compose(
237
750
  if (filter) {
238
751
  const cScale = cursorCfg.scale ?? 1;
239
752
  const png = resolve(tmp, "cursor.png");
240
- await renderCursorPng(png, Math.round(32 * pxScale * cScale));
753
+ await renderCursorPng(
754
+ png,
755
+ Math.round(32 * pxScale * cScale),
756
+ cursorCfg.style ?? "arrow",
757
+ cursorCfg.color,
758
+ {
759
+ ...(cursorCfg.outline ? { outline: cursorCfg.outline } : {}),
760
+ ...(cursorCfg.outlineWidth != null ? { outlineWidth: cursorCfg.outlineWidth } : {}),
761
+ ...(cursorCfg.halo ? { halo: true } : {}),
762
+ }
763
+ );
241
764
  const withCursor = resolve(tmp, "content-cursor.mp4");
242
765
  await runFfmpeg([
243
766
  "-i",
@@ -272,11 +795,40 @@ export async function compose(
272
795
  }
273
796
 
274
797
  // Auto-zoom pass over the content (never over the cards or captions).
798
+ let focusDropped = 0;
275
799
  if (zoomCfg && zoomEvents.length > 0) {
276
800
  const contentMs = await probeDurationMs(content);
277
- const filter = buildZoomFilter(zoomEvents, zoomCfg, outW, outH, contentMs, FPS);
801
+ const filter = buildZoomFilter(
802
+ zoomEvents,
803
+ zoomCfg,
804
+ outW,
805
+ outH,
806
+ contentMs,
807
+ FPS,
808
+ storyboard.video.width,
809
+ (dropped, total) => {
810
+ focusDropped = dropped;
811
+ log(
812
+ ` ! auto-zoom: ${dropped} of ${total} focus point(s) dropped to fit ` +
813
+ `ffmpeg's expression budget — the rest are spread evenly. Set ` +
814
+ `"zoom": false on scenes that don't need the camera to choose which.`
815
+ );
816
+ warn(
817
+ "focus-dropped",
818
+ `auto-zoom dropped ${dropped} of ${total} focus point(s) to fit ffmpeg's ` +
819
+ `expression budget — set "zoom": false on scenes that don't need the camera`
820
+ );
821
+ }
822
+ );
278
823
  if (filter) {
279
- log(`auto-zoom: ${zoomEvents.length} focus event(s)`);
824
+ const capped = clampScaleForWidth(zoomCfg.scale, storyboard.video.width);
825
+ log(
826
+ `auto-zoom: ${zoomEvents.length} focus event(s)` +
827
+ (capped < zoomCfg.scale
828
+ ? ` (scale ${zoomCfg.scale} capped to ${capped.toFixed(2)} for a ` +
829
+ `${storyboard.video.width}px-wide viewport)`
830
+ : "")
831
+ );
280
832
  const zoomed = resolve(tmp, "content-zoomed.mp4");
281
833
  await runFfmpeg([
282
834
  "-i",
@@ -327,19 +879,158 @@ export async function compose(
327
879
  log(`motion blur: tmix ${frames} frames`);
328
880
  }
329
881
 
882
+ // Keystroke chips are screen-fixed UI: after the zoom/blur, before cards.
883
+ let keyItems: OverlayItem[] = [];
884
+ if (keysPlaced.length) {
885
+ keyItems = await renderKeyChipPngs(
886
+ keysPlaced,
887
+ resolve(tmp, "attention"),
888
+ storyboard.video.width,
889
+ storyboard.video.height,
890
+ pxScale
891
+ );
892
+ const r = await overlayPngBatches(content, keyItems, tmp, "keys");
893
+ content = r.video;
894
+ log(`keystrokes: ${keysPlaced.length} chip(s) in ${r.passes} pass(es)`);
895
+ }
896
+
897
+ // Produced-look frame (opt-in `frame` block): pad the content onto a canvas
898
+ // and overlay the rasterized frame PNG (background, shadow, chrome, rounded
899
+ // hole). Everything after this — cards, watermark, captions — renders at
900
+ // the canvas size, so captions land in the padding band.
901
+ let canvasW = outW;
902
+ let canvasH = outH;
903
+ const outCfg = resolveOutputSizing(storyboard.output);
904
+ if (storyboard.frame) {
905
+ // `frame.url: "auto"` tracks the page: one frame PNG per distinct address,
906
+ // time-gated to the scenes that are on it (#52). Everything else renders a
907
+ // single static frame, exactly as before.
908
+ const autoUrl = storyboard.frame.url === "auto";
909
+ const spans = autoUrl ? frameUrlSpans(composedTl, sceneSpans) : [];
910
+ const labels = autoUrl ? [...new Set(spans.map((sp) => sp.label))] : [""];
911
+ const framePngs: string[] = [];
912
+ let layout!: Awaited<ReturnType<typeof renderFramePng>>;
913
+ for (let li = 0; li < labels.length; li++) {
914
+ const framePng = resolve(tmp, `frame-${li}.png`);
915
+ layout = await renderFramePng(
916
+ autoUrl ? { ...storyboard.frame, url: labels[li] || undefined } : storyboard.frame,
917
+ storyboard.brand,
918
+ framePng,
919
+ storyboard.video.width,
920
+ storyboard.video.height,
921
+ pxScale,
922
+ outCfg.width != null && outCfg.height != null ? outCfg.width / outCfg.height : undefined
923
+ );
924
+ framePngs.push(framePng);
925
+ }
926
+ const framed = resolve(tmp, "content-framed.mp4");
927
+ // Each frame PNG covers the whole canvas and the spans partition the
928
+ // timeline, so exactly one overlay is enabled at any instant.
929
+ const chain = framePngs
930
+ .map((_, li) => {
931
+ const win = spans
932
+ .filter((sp) => sp.label === labels[li])
933
+ .map((sp) => `between(t,${(sp.a / 1000).toFixed(3)},${(sp.b / 1000).toFixed(3)})`)
934
+ .join("+");
935
+ const gate = autoUrl && framePngs.length > 1 ? `:enable='${win}'` : "";
936
+ const src = li === 0 ? "[p]" : `[v${li - 1}]`;
937
+ const dst = li === framePngs.length - 1 ? "[vout]" : `[v${li}]`;
938
+ return `${src}[${li + 1}:v]overlay=0:0:format=auto${gate}${dst}`;
939
+ })
940
+ .join(";");
941
+ await runFfmpeg([
942
+ "-i",
943
+ content,
944
+ ...framePngs.flatMap((f) => ["-i", f]),
945
+ "-filter_complex",
946
+ `[0:v]pad=${layout.canvasW}:${layout.canvasH}:${layout.offsetX}:${layout.offsetY}:color=${layout.padColor}[p];` +
947
+ chain,
948
+ "-map",
949
+ "[vout]",
950
+ "-an",
951
+ "-r",
952
+ String(FPS),
953
+ "-c:v",
954
+ "libx264",
955
+ "-preset",
956
+ "veryfast",
957
+ "-crf",
958
+ "20",
959
+ "-pix_fmt",
960
+ "yuv420p",
961
+ framed,
962
+ ]);
963
+ content = framed;
964
+ canvasW = layout.canvasW;
965
+ canvasH = layout.canvasH;
966
+ log(
967
+ `frame: ${layout.canvasW}x${layout.canvasH} canvas (padding ${storyboard.frame.padding ?? 48}` +
968
+ `${storyboard.frame.chrome && storyboard.frame.chrome !== "none" ? `, ${storyboard.frame.chrome} chrome` : ""})` +
969
+ (autoUrl ? `, auto url: ${labels.filter(Boolean).join(" -> ") || "(none)"}` : "")
970
+ );
971
+ }
972
+ const logicalW = canvasW / pxScale;
973
+ const logicalH = canvasH / pxScale;
974
+
975
+ // Brand watermark (opt-in `brand.logo`): scaled logo in a corner over the
976
+ // content (not the cards — the cards carry the logo in their layout).
977
+ const logoPath = storyboard.brand?.logo
978
+ ? resolve(project.dir, storyboard.brand.logo)
979
+ : null;
980
+ if (logoPath && storyboard.brand?.watermark?.enabled !== false) {
981
+ if (!(await exists(logoPath))) {
982
+ warn("brand-logo-missing", `brand.logo not found at ${logoPath} — watermark and card logo skipped`);
983
+ } else {
984
+ const wm = storyboard.brand?.watermark ?? {};
985
+ const wmW = Math.round(canvasW * (wm.scale ?? 0.11));
986
+ const margin = Math.round(24 * pxScale);
987
+ const pos = wm.position ?? "bottom-right";
988
+ const x = pos.endsWith("left") ? margin : canvasW - wmW - margin;
989
+ const yExpr = pos.startsWith("top") ? `${margin}` : `${canvasH}-h-${margin}`;
990
+ const marked = resolve(tmp, "content-marked.mp4");
991
+ await runFfmpeg([
992
+ "-i",
993
+ content,
994
+ "-i",
995
+ logoPath,
996
+ "-filter_complex",
997
+ `[1:v]scale=${wmW}:-1:flags=lanczos,format=rgba,colorchannelmixer=aa=${(wm.opacity ?? 0.85).toFixed(2)}[lg];` +
998
+ `[0:v][lg]overlay=${x}:${yExpr}[vout]`,
999
+ "-map",
1000
+ "[vout]",
1001
+ "-an",
1002
+ "-r",
1003
+ String(FPS),
1004
+ "-c:v",
1005
+ "libx264",
1006
+ "-preset",
1007
+ "veryfast",
1008
+ "-crf",
1009
+ "20",
1010
+ "-pix_fmt",
1011
+ "yuv420p",
1012
+ marked,
1013
+ ]);
1014
+ content = marked;
1015
+ log(`watermark: ${storyboard.brand?.logo} at ${pos}`);
1016
+ }
1017
+ }
1018
+ const logoDataUri =
1019
+ logoPath && (await exists(logoPath)) ? await fileToDataUri(logoPath) : undefined;
1020
+
330
1021
  // Intro/outro title cards. Music runs under them; narration and captions
331
1022
  // shift right by the intro's length.
332
1023
  const segments: string[] = [];
333
1024
  const introMs = storyboard.intro ? storyboard.intro.durationMs : 0;
334
1025
  if (storyboard.intro) {
335
1026
  segments.push(
336
- await cardSegment(storyboard.intro, "intro", tmp, storyboard, outW, outH, pxScale)
1027
+ await cardSegment(storyboard.intro, "intro", tmp, storyboard, logicalW, logicalH, pxScale, logoDataUri)
337
1028
  );
338
1029
  }
339
1030
  segments.push(content);
340
1031
  if (storyboard.outro) {
341
1032
  segments.push(
342
- await cardSegment(storyboard.outro, "outro", tmp, storyboard, outW, outH, pxScale)
1033
+ await cardSegment(storyboard.outro, "outro", tmp, storyboard, logicalW, logicalH, pxScale, logoDataUri)
343
1034
  );
344
1035
  }
345
1036
  // Re-encode when assembling cards: the card segments carry mp4 edit lists
@@ -352,29 +1043,237 @@ export async function compose(
352
1043
  ? content
353
1044
  : await concatSegments(segments, resolve(tmp, "video.mp4"), tmp, "full", true);
354
1045
 
355
- const captioned = await burnCaptions(
1046
+ // Caption band collisions: any attention box / key chip that reaches into
1047
+ // the bottom strip flips the caption to the top for the cues it overlaps.
1048
+ const bandTop = canvasH - Math.round((CAPTION_BOTTOM_GAP + 80) * pxScale);
1049
+ const collisions = [...attentionItems, ...keyItems]
1050
+ .filter((it) => {
1051
+ // Spotlights are full-frame PNGs; use the hole instead of the canvas.
1052
+ return it.y + pngHeightHint(it, attentionPlaced, pxScale) > bandTop;
1053
+ })
1054
+ .map((it) => ({ a: it.a * 1000, b: it.b * 1000 }));
1055
+ const captionsResult = await burnCaptions(
356
1056
  project,
357
1057
  storyboard,
358
1058
  fullVideo,
359
1059
  tmp,
360
1060
  introMs,
361
- pxScale
1061
+ pxScale,
1062
+ {
1063
+ defaultPosition: storyboard.captions?.position ?? "bottom",
1064
+ windows: captionWindows,
1065
+ collisions,
1066
+ look: { ...(storyboard.captions ?? {}), font: storyboard.captions?.font ?? storyboard.brand?.font },
1067
+ logicalWidth: logicalW,
1068
+ }
362
1069
  );
1070
+ const captioned = captionsResult.video;
363
1071
  // Optional final resize/reframe (opt-in). Applied last, over the whole
364
1072
  // composed frame (cards + captions included), so a 1280x720 take can render
365
1073
  // as a vertical social clip. Audio is muxed after, so mux stays -c:v copy.
366
1074
  // `output` may carry only a `loudness` override (no width/height) — resize
367
1075
  // just when both dimensions are set.
368
1076
  const finalVideo =
369
- storyboard.output?.width != null && storyboard.output?.height != null
370
- ? await resizeOutput(captioned, storyboard.output, tmp)
1077
+ outCfg.width != null && outCfg.height != null
1078
+ ? await resizeOutput(captioned, outCfg, tmp)
371
1079
  : captioned;
372
- await muxAudio(project, storyboard, finalVideo, introMs);
1080
+ const chapters = storyboard.output?.chapters
1081
+ ? await writeChapters(tmp, [
1082
+ ...(storyboard.intro
1083
+ ? [{ id: "intro", title: storyboard.intro.title, durationMs: introMs }]
1084
+ : []),
1085
+ ...sceneReports.map((r) => ({
1086
+ id: r.id,
1087
+ title: sceneById.get(r.id)?.title ?? r.id,
1088
+ durationMs: r.targetMs,
1089
+ })),
1090
+ ...(storyboard.outro
1091
+ ? [{ id: "outro", title: storyboard.outro.title, durationMs: storyboard.outro.durationMs }]
1092
+ : []),
1093
+ ])
1094
+ : null;
1095
+ await muxAudio(project, storyboard, finalVideo, introMs, chapters);
373
1096
  ok(`final video → ${project.outputPath}`);
1097
+ // Blank-open measurement + poster (issue #47). The poster used to be taken
1098
+ // at a fixed offset, which on a demo that opens with a `goto` is exactly the
1099
+ // white load frame; pick the first frame that actually shows content instead.
1100
+ const blankOpenMs = await probeFlatLeadMs(project.outputPath, 4000, introMs).catch(() => 0);
1101
+ if (blankOpenMs > 250) {
1102
+ warn(
1103
+ "blank-open",
1104
+ `the video opens on ${Math.round(blankOpenMs)}ms of blank page${storyboard.intro ? " after the intro card" : ""} ` +
1105
+ `(the first scene's load) — ` +
1106
+ `set output.trimLeadingBlank, or wait on the page's content before the first beat`
1107
+ );
1108
+ }
1109
+ let poster: string | undefined;
1110
+ if (storyboard.output?.poster) {
1111
+ poster = project.posterPath;
1112
+ const at =
1113
+ typeof storyboard.output.poster === "number"
1114
+ ? storyboard.output.poster
1115
+ : introMs + Math.max(500, blankOpenMs + 200);
1116
+ await runFfmpeg([
1117
+ "-ss",
1118
+ (at / 1000).toFixed(3),
1119
+ "-i",
1120
+ project.outputPath,
1121
+ "-frames:v",
1122
+ "1",
1123
+ "-update",
1124
+ "1",
1125
+ poster,
1126
+ ]);
1127
+ log(`poster → ${poster} (@ ${stampMs(at)})`);
1128
+ }
1129
+ // Dump the queued warning frames (issue #59). Best-effort: a failed grab
1130
+ // must never fail a compose that otherwise produced a video.
1131
+ if (warningFrames.length) {
1132
+ const dir = project.p("output", "warnings");
1133
+ await ensureDir(dir);
1134
+ for (const f of warningFrames) {
1135
+ const png = resolve(dir, `${f.name}.png`);
1136
+ await runFfmpeg([
1137
+ "-y",
1138
+ "-ss",
1139
+ ((introMs + f.atMs) / 1000).toFixed(3),
1140
+ "-i",
1141
+ project.outputPath,
1142
+ "-frames:v",
1143
+ "1",
1144
+ "-update",
1145
+ "1",
1146
+ png,
1147
+ ]).catch(() => {});
1148
+ }
1149
+ log(`warning frames → ${relative(project.dir, dir)}/ (${warningFrames.length})`);
1150
+ }
374
1151
  // AIDEMO_KEEP_TMP=1 keeps .compose-tmp for debugging intermediates.
375
1152
  if (!process.env.AIDEMO_KEEP_TMP) {
376
1153
  await fs.rm(tmp, { recursive: true, force: true });
377
1154
  }
1155
+
1156
+ const report: ComposeReport = {
1157
+ output: project.outputPath,
1158
+ durationMs: await probeDurationMs(project.outputPath).catch(() => 0),
1159
+ scenes: sceneReports,
1160
+ zoom: { focusTotal: zoomEvents.length, focusDropped },
1161
+ cursorPoints: cursorPts.length,
1162
+ captions: { cues: captionsResult.cues, passes: captionsResult.passes },
1163
+ hold: hold.mode,
1164
+ ...(poster ? { poster } : {}),
1165
+ blankOpenMs: Math.round(blankOpenMs),
1166
+ ...(attentionPlaced.length || keysPlaced.length || clicksPlaced.length || redactPlaced.length
1167
+ ? {
1168
+ attention: {
1169
+ events: attentionPlaced.length,
1170
+ keys: keysPlaced.length,
1171
+ clicks: clicksPlaced.length,
1172
+ redactSpans: redactPlaced.length,
1173
+ captionsFlipped: captionsResult.flipped,
1174
+ },
1175
+ }
1176
+ : {}),
1177
+ warnings,
1178
+ };
1179
+ await writeJson(project.reportPath, report);
1180
+ const frozen = sceneReports.filter((s) => s.holdPct > FREEZE_WARN_PCT).length;
1181
+ log(
1182
+ `report → ${project.reportPath}` +
1183
+ (frozen ? ` (${frozen} of ${sceneReports.length} scene(s) mostly held)` : "")
1184
+ );
1185
+ if (warnings.length) {
1186
+ // Last thing on screen, grouped by scene (issue #59): in a 28-scene render
1187
+ // an ungrouped list scrolls away above the ✓ lines and is never read.
1188
+ log(`⚠ ${warnings.length} compose warning(s):`);
1189
+ const byScene = new Map<string, ComposeWarning[]>();
1190
+ for (const w of warnings) {
1191
+ const k = w.scene ?? "(whole video)";
1192
+ byScene.set(k, [...(byScene.get(k) ?? []), w]);
1193
+ }
1194
+ for (const [scene, list] of byScene) {
1195
+ log(` ${scene}:`);
1196
+ for (const w of list) log(` - [${w.code}] ${w.message}`);
1197
+ }
1198
+ }
1199
+ return report;
1200
+ }
1201
+
1202
+ /** Share of a scene that may be a held frame before compose flags it. */
1203
+ const FREEZE_WARN_PCT = 0.4;
1204
+
1205
+ /**
1206
+ * Hold mode "drift": instead of cloning the scene's last frame for `holdMs`,
1207
+ * push in on it very slowly (1 → driftScale over the hold), so a long
1208
+ * narration over a static page reads as a deliberate dwell, not a stall.
1209
+ * The frame is pulled from the encoded scene's tail, rendered as a zoompan
1210
+ * still (pre-upscaled 2x so integer-pixel crops don't shimmer — the same rule
1211
+ * the auto-zoom pass follows), and re-encoded onto the scene. One tiny
1212
+ * expression, so the zoom budget is untouched; `zoompan`/`scale` are core.
1213
+ */
1214
+ async function appendDriftHold(
1215
+ scenePath: string,
1216
+ holdMs: number,
1217
+ driftScale: number,
1218
+ outW: number,
1219
+ outH: number,
1220
+ tmp: string,
1221
+ sceneIdx: number
1222
+ ): Promise<string> {
1223
+ const durMs = await probeDurationMs(scenePath);
1224
+ const framePng = resolve(tmp, `scene-${sceneIdx}-hold.png`);
1225
+ await runFfmpeg([
1226
+ "-ss",
1227
+ (Math.max(0, durMs - 60) / 1000).toFixed(3),
1228
+ "-i",
1229
+ scenePath,
1230
+ "-frames:v",
1231
+ "1",
1232
+ "-update",
1233
+ "1",
1234
+ framePng,
1235
+ ]);
1236
+ const holdSec = holdMs / 1000;
1237
+ const frames = Math.max(1, Math.round(holdSec * FPS));
1238
+ const holdMp4 = resolve(tmp, `scene-${sceneIdx}-hold.mp4`);
1239
+ // Ease the creep (ease-out quad) so it starts moving right away and settles.
1240
+ const z = `'1+(${driftScale.toFixed(4)}-1)*(1-pow(1-min(on/${frames}\,1)\,2))'`;
1241
+ await runFfmpeg([
1242
+ "-loop",
1243
+ "1",
1244
+ "-framerate",
1245
+ String(FPS),
1246
+ "-i",
1247
+ framePng,
1248
+ "-t",
1249
+ holdSec.toFixed(3),
1250
+ "-vf",
1251
+ `scale=iw*2:ih*2:flags=lanczos,` +
1252
+ `zoompan=z=${z}:x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=1:s=${outW}x${outH}:fps=${FPS},` +
1253
+ `setsar=1`,
1254
+ "-r",
1255
+ String(FPS),
1256
+ "-an",
1257
+ "-c:v",
1258
+ "libx264",
1259
+ "-preset",
1260
+ "veryfast",
1261
+ "-crf",
1262
+ "20",
1263
+ "-pix_fmt",
1264
+ "yuv420p",
1265
+ holdMp4,
1266
+ ]);
1267
+ // The PNG-sourced hold carries an edit list; re-encode the join so the pts
1268
+ // start clean at 0 (a stream-copy concat would leak negative timestamps
1269
+ // into the overlay passes — see AGENTS.md).
1270
+ return concatSegments(
1271
+ [scenePath, holdMp4],
1272
+ resolve(tmp, `scene-${sceneIdx}-held.mp4`),
1273
+ tmp,
1274
+ `scene-${sceneIdx}-held`,
1275
+ true
1276
+ );
378
1277
  }
379
1278
 
380
1279
  /**
@@ -384,7 +1283,7 @@ export async function compose(
384
1283
  * needs an API key so compose can't regenerate them itself; warn loudly and
385
1284
  * name the fix instead.
386
1285
  */
387
- async function warnStaleCaptions(project: Project): Promise<void> {
1286
+ async function warnStaleCaptions(project: Project): Promise<boolean> {
388
1287
  try {
389
1288
  const [cap, narr] = await Promise.all([
390
1289
  fs.stat(project.captionsCuesPath),
@@ -396,10 +1295,12 @@ async function warnStaleCaptions(project: Project): Promise<void> {
396
1295
  `the burned captions will not match the audio. ` +
397
1296
  `Fix: aidemo captions ${project.dir}` + ` (then re-run compose).`
398
1297
  );
1298
+ return true;
399
1299
  }
400
1300
  } catch {
401
1301
  /* captions or narration absent — burnCaptions handles missing captions */
402
1302
  }
1303
+ return false;
403
1304
  }
404
1305
 
405
1306
  /**
@@ -450,12 +1351,96 @@ async function concatSegments(
450
1351
  * padded). Net effect: identical A/V duration + alignment as hard cuts, but the
451
1352
  * boundaries cross-dissolve. Re-encodes (xfade needs filter_complex).
452
1353
  */
1354
+ /**
1355
+ * Does the boundary between these two scenes cross a page navigation? Either
1356
+ * the next scene opens with a load (its first idle span is the `goto` wait) or
1357
+ * the previous scene's last action was a navigation. Both are already in the
1358
+ * timeline, so no extra probing is needed.
1359
+ */
1360
+ /**
1361
+ * Per-scene address-pill text for `frame.url: "auto"`: the host of the URL the
1362
+ * scene is on, carried forward from the last `goto` (falling back to
1363
+ * `baseUrl`). Consecutive scenes on the same host share one span.
1364
+ */
1365
+ function frameUrlSpans(
1366
+ scenes: Timeline["scenes"],
1367
+ spans: Array<{ a: number; b: number }>
1368
+ ): Array<{ a: number; b: number; label: string }> {
1369
+ let current = "";
1370
+ const out: Array<{ a: number; b: number; label: string }> = [];
1371
+ for (let i = 0; i < scenes.length; i++) {
1372
+ for (const a of scenes[i].actions ?? []) {
1373
+ if (a.op === "goto" && a.target) current = prettyHost(a.target) || current;
1374
+ }
1375
+ const span = spans[i];
1376
+ if (!span) continue;
1377
+ const last = out.at(-1);
1378
+ if (last && last.label === current) last.b = span.b;
1379
+ else out.push({ a: i === 0 ? 0 : span.a, b: span.b, label: current });
1380
+ }
1381
+ if (out.length) out[out.length - 1].b += 3600_000; // cover holds/cards past the last scene
1382
+ return out;
1383
+ }
1384
+
1385
+ function prettyHost(url?: string): string {
1386
+ if (!url) return "";
1387
+ try {
1388
+ const u = new URL(url, "http://localhost");
1389
+ if (u.protocol !== "http:" && u.protocol !== "https:") return "";
1390
+ return u.host.replace(/^www\./, "") + (u.pathname !== "/" ? u.pathname : "");
1391
+ } catch {
1392
+ return "";
1393
+ }
1394
+ }
1395
+
1396
+ function crossesNavigation(
1397
+ prev: Timeline["scenes"][number],
1398
+ next: Timeline["scenes"][number]
1399
+ ): boolean {
1400
+ const opensOnLoad = (next.idleSpans ?? []).some(
1401
+ (sp) => sp.label === "load" && sp.startMs - next.startMs < 1500
1402
+ );
1403
+ if (opensOnLoad) return true;
1404
+ const lastOp = (prev.actions ?? []).filter((a) => !a.skipped).at(-1)?.op;
1405
+ const firstOp = (next.actions ?? [])[0]?.op;
1406
+ return lastOp === "goto" || lastOp === "back" || firstOp === "goto" || firstOp === "back";
1407
+ }
1408
+
453
1409
  async function crossfadeScenes(
454
1410
  sceneVideos: string[],
455
- durationMs: number,
456
- outPath: string
1411
+ durationsMs: number[],
1412
+ outPath: string,
1413
+ tmp: string
457
1414
  ): Promise<string> {
458
- const D = Math.max(0.05, durationMs / 1000);
1415
+ // Hard-cut boundaries split the run into groups: each group is crossfaded on
1416
+ // its own (duration is preserved inside a group) and the groups are then
1417
+ // concatenated, which IS the hard cut.
1418
+ if (durationsMs.some((d) => d <= 0)) {
1419
+ const groups: string[][] = [[sceneVideos[0]]];
1420
+ const groupDurs: number[][] = [[]];
1421
+ for (let i = 1; i < sceneVideos.length; i++) {
1422
+ if (durationsMs[i - 1] <= 0) {
1423
+ groups.push([sceneVideos[i]]);
1424
+ groupDurs.push([]);
1425
+ } else {
1426
+ groups[groups.length - 1].push(sceneVideos[i]);
1427
+ groupDurs[groupDurs.length - 1].push(durationsMs[i - 1]);
1428
+ }
1429
+ }
1430
+ const parts: string[] = [];
1431
+ for (let g = 0; g < groups.length; g++) {
1432
+ if (groups[g].length === 1) {
1433
+ parts.push(groups[g][0]);
1434
+ continue;
1435
+ }
1436
+ parts.push(
1437
+ await crossfadeScenes(groups[g], groupDurs[g], resolve(tmp, `xfade-${g}.mp4`), tmp)
1438
+ );
1439
+ }
1440
+ if (parts.length === 1) return parts[0];
1441
+ return concatSegments(parts, outPath, tmp, "xfade-groups", true);
1442
+ }
1443
+ const D = Math.max(0.05, Math.max(...durationsMs) / 1000);
459
1444
  // Measure each segment so the offsets track real (not assumed) durations.
460
1445
  const durs = await Promise.all(sceneVideos.map((p) => probeDurationMs(p)));
461
1446
  // Freeze-tail margin: a touch longer than D so the fade never starves at the
@@ -484,8 +1469,9 @@ async function crossfadeScenes(
484
1469
  cumMs += durs[i - 1];
485
1470
  const offset = (cumMs / 1000).toFixed(3);
486
1471
  const out = i === sceneVideos.length - 1 ? "xout" : `x${i}`;
1472
+ const d = Math.max(0.05, durationsMs[i - 1] / 1000);
487
1473
  filters.push(
488
- `[${prev}][${labels[i]}]xfade=transition=fade:duration=${D.toFixed(
1474
+ `[${prev}][${labels[i]}]xfade=transition=fade:duration=${d.toFixed(
489
1475
  3
490
1476
  )}:offset=${offset}[${out}]`
491
1477
  );
@@ -564,18 +1550,23 @@ async function cardSegment(
564
1550
  name: string,
565
1551
  tmp: string,
566
1552
  storyboard: Storyboard,
567
- outW: number,
568
- outH: number,
569
- pxScale: number
1553
+ logicalW: number,
1554
+ logicalH: number,
1555
+ pxScale: number,
1556
+ logoDataUri?: string
570
1557
  ): Promise<string> {
571
1558
  const png = resolve(tmp, `${name}.png`);
572
1559
  await renderCardPng(
573
1560
  card,
574
1561
  png,
575
- storyboard.video.width,
576
- storyboard.video.height,
577
- pxScale
1562
+ Math.round(logicalW),
1563
+ Math.round(logicalH),
1564
+ pxScale,
1565
+ storyboard.brand,
1566
+ logoDataUri
578
1567
  );
1568
+ const outW = Math.round(logicalW * pxScale) & ~1;
1569
+ const outH = Math.round(logicalH * pxScale) & ~1;
579
1570
  const durSec = card.durationMs / 1000;
580
1571
  const fadeSec = Math.min(card.fadeMs / 1000, durSec / 3);
581
1572
  const out = resolve(tmp, `${name}.mp4`);
@@ -609,6 +1600,60 @@ async function cardSegment(
609
1600
  return out;
610
1601
  }
611
1602
 
1603
+ /** Union of overlapping/adjacent idle spans (auto + annotated), sorted. */
1604
+ function mergeIdle(spans: IdleSpan[]): IdleSpan[] {
1605
+ const sorted = [...spans].sort((a, b) => a.startMs - b.startMs);
1606
+ const out: IdleSpan[] = [];
1607
+ for (const s of sorted) {
1608
+ const last = out[out.length - 1];
1609
+ if (last && s.startMs <= last.endMs) last.endMs = Math.max(last.endMs, s.endMs);
1610
+ else out.push({ ...s });
1611
+ }
1612
+ return out;
1613
+ }
1614
+
1615
+ /**
1616
+ * Motionless spans of the raw file (video time) → idle spans for one scene
1617
+ * (timeline time, the same base the player writes). Spans are clipped to the
1618
+ * scene, and the first/last 300 ms are left alone: the frame right after a
1619
+ * click and the one the scene ends on are the beats a viewer reads, and
1620
+ * trimming into them makes cuts feel clipped.
1621
+ */
1622
+ function sceneAutoIdle(
1623
+ freezes: FreezeSpan[],
1624
+ tl: Timeline["scenes"][number],
1625
+ leadInMs: number
1626
+ ): IdleSpan[] {
1627
+ const EDGE_MS = 300;
1628
+ const lo = tl.startMs + leadInMs + EDGE_MS;
1629
+ const hi = tl.endMs + leadInMs - EDGE_MS;
1630
+ // Windows the author explicitly asked to be ON SCREEN: an attention overlay
1631
+ // and its hold, a deliberate `focus`, a named still. A static confirmation
1632
+ // card is motionless by nature, so freezedetect happily proposes trimming
1633
+ // exactly the beat the scene exists for — and compose then holds the NEXT
1634
+ // page's frame under that scene's narration (issue #55).
1635
+ const protectedWindows: Array<[number, number]> = [
1636
+ ...(tl.attentionEvents ?? []).map(
1637
+ (e): [number, number] => [e.tMs + leadInMs, e.tMs + e.holdMs + leadInMs]
1638
+ ),
1639
+ ...(tl.focusEvents ?? [])
1640
+ .filter((e) => e.kind === "focus")
1641
+ .map((e): [number, number] => [e.tMs + leadInMs, e.tMs + (e.holdMs ?? 1200) + leadInMs]),
1642
+ ...(tl.stillEvents ?? []).map(
1643
+ (e): [number, number] => [e.tMs + leadInMs - 200, e.tMs + leadInMs + 200]
1644
+ ),
1645
+ ];
1646
+ const out: IdleSpan[] = [];
1647
+ for (const f of freezes) {
1648
+ const a = Math.max(f.startMs, lo);
1649
+ const b = Math.min(f.endMs, hi);
1650
+ if (b - a <= 200) continue;
1651
+ if (protectedWindows.some(([pa, pb]) => pa < b && pb > a)) continue;
1652
+ out.push({ startMs: a - leadInMs, endMs: b - leadInMs, label: "auto-idle" });
1653
+ }
1654
+ return out;
1655
+ }
1656
+
612
1657
  /**
613
1658
  * Build the list of [startMs,endMs] video-time spans to keep for a scene:
614
1659
  * non-idle content in full, each idle span capped to IDLE_CAP_MS.
@@ -641,6 +1686,14 @@ function keepIntervals(
641
1686
  return keeps.filter(([a, b]) => b - a > 20);
642
1687
  }
643
1688
 
1689
+ /** One retime piece: kept-span offsets [a,b) play at factor f, starting at stretched ms `out`. */
1690
+ interface RetimePiece {
1691
+ a: number;
1692
+ b: number;
1693
+ f: number;
1694
+ out: number;
1695
+ }
1696
+
644
1697
  /**
645
1698
  * Where a raw-video timestamp lands inside the concatenation of the kept
646
1699
  * spans. Timestamps inside a trimmed gap clamp to the gap's start.
@@ -655,6 +1708,24 @@ function offsetInKeeps(keeps: Array<[number, number]>, t: number): number {
655
1708
  return off;
656
1709
  }
657
1710
 
1711
+ /** Inverse of `offsetInKeeps`: kept-span offset → raw video ms. */
1712
+ function rawAtKeepOffset(keeps: Array<[number, number]>, local: number): number {
1713
+ let off = 0;
1714
+ for (const [a, b] of keeps) {
1715
+ const len = b - a;
1716
+ if (local <= off + len) return a + (local - off);
1717
+ off += len;
1718
+ }
1719
+ return keeps.length ? keeps[keeps.length - 1][1] : 0;
1720
+ }
1721
+
1722
+ /** mm:ss.t for a ms offset, for log lines that point back into the raw take. */
1723
+ function stampMs(ms: number): string {
1724
+ const s = Math.max(0, ms) / 1000;
1725
+ const m = Math.floor(s / 60);
1726
+ return `${m}:${(s - m * 60).toFixed(1).padStart(4, "0")}`;
1727
+ }
1728
+
658
1729
  /** Extract each keep span from raw and concat into one mp4 (uniform h264). */
659
1730
  async function extractAndConcat(
660
1731
  rawVideo: string,
@@ -723,67 +1794,159 @@ async function extractAndConcat(
723
1794
  */
724
1795
  const CAPTION_BOTTOM_GAP = 96;
725
1796
 
1797
+ /**
1798
+ * Max caption cues overlaid in a single ffmpeg pass. Each cue costs one input
1799
+ * and one filter link; ffmpeg fails to configure the graph somewhere past ~90
1800
+ * (issue #36), so long narrations are burned in several passes.
1801
+ */
1802
+ const CAPTION_BATCH = 32;
1803
+
1804
+ /**
1805
+ * Overlay many time-gated PNGs onto `src` in passes of ≤ CAPTION_BATCH. Each
1806
+ * item is one extra input plus one overlay link; ffmpeg fails to configure a
1807
+ * graph somewhere past ~90 of them (issue #36), so long lists are burned in
1808
+ * several re-encoding passes. Shared by captions, attention beats, key chips.
1809
+ */
1810
+ async function overlayPngBatches(
1811
+ src: string,
1812
+ items: OverlayItem[],
1813
+ tmp: string,
1814
+ label: string
1815
+ ): Promise<{ video: string; passes: number }> {
1816
+ if (items.length === 0) return { video: src, passes: 0 };
1817
+ const batches: OverlayItem[][] = [];
1818
+ for (let i = 0; i < items.length; i += CAPTION_BATCH) {
1819
+ batches.push(items.slice(i, i + CAPTION_BATCH));
1820
+ }
1821
+ if (batches.length > 1) {
1822
+ log(`${label} overlay: ${batches.length} pass(es) of <= ${CAPTION_BATCH} item(s)`);
1823
+ }
1824
+ let cur = src;
1825
+ let out = src;
1826
+ for (let b = 0; b < batches.length; b++) {
1827
+ const batch = batches[b];
1828
+ const args: string[] = ["-i", cur];
1829
+ for (const it of batch) args.push("-i", it.png);
1830
+ const chain: string[] = [];
1831
+ let prev = "0:v";
1832
+ batch.forEach((it, i) => {
1833
+ const inp = `${i + 1}:v`;
1834
+ const lbl = i === batch.length - 1 ? "vout" : `o${i}`;
1835
+ // Escape the enable-expression commas so ffmpeg doesn't read them as
1836
+ // filter separators.
1837
+ chain.push(
1838
+ `[${prev}][${inp}]overlay=${it.x}:${it.y}:enable=between(t\\,${it.a.toFixed(3)}\\,${it.b.toFixed(3)})[${lbl}]`
1839
+ );
1840
+ prev = lbl;
1841
+ });
1842
+ out = resolve(tmp, `${label}-${b}.mp4`);
1843
+ await runFfmpeg([
1844
+ ...args,
1845
+ "-filter_complex",
1846
+ chain.join(";"),
1847
+ "-map",
1848
+ "[vout]",
1849
+ "-an",
1850
+ "-c:v",
1851
+ "libx264",
1852
+ "-preset",
1853
+ "veryfast",
1854
+ "-crf",
1855
+ "20",
1856
+ "-pix_fmt",
1857
+ "yuv420p",
1858
+ out,
1859
+ ]);
1860
+ cur = out;
1861
+ }
1862
+ return { video: out, passes: batches.length };
1863
+ }
1864
+
1865
+ /** Height of an overlay PNG for the caption-collision check (spotlight → its hole). */
1866
+ function pngHeightHint(it: OverlayItem, placed: PlacedAttention[], pxScale: number): number {
1867
+ if (it.x === 0 && it.y === 0) {
1868
+ const ev = placed.find((p) => p.kind === "spotlight" && p.a === it.a && p.b === it.b);
1869
+ if (ev) return Math.round((ev.y + ev.h + (ev.padding ?? 10)) * pxScale);
1870
+ return 0;
1871
+ }
1872
+ return it.png.includes("/key-") ? 60 * pxScale : 48 * pxScale;
1873
+ }
1874
+
1875
+ interface CaptionPlacement {
1876
+ defaultPosition: "top" | "bottom";
1877
+ /** Caption look (style/font/size/color/background), brand font already merged. */
1878
+ look?: CaptionsConfig & { font?: string };
1879
+ /** Logical (CSS px) width of the frame the strip must span. */
1880
+ logicalWidth?: number;
1881
+ /** Content-time windows (ms, before the intro shift) with a fixed position. */
1882
+ windows: Array<{ a: number; b: number; position: "top" | "bottom" }>;
1883
+ /** Content-time windows (ms) during which the bottom band is occupied. */
1884
+ collisions: Array<{ a: number; b: number }>;
1885
+ }
1886
+
726
1887
  async function burnCaptions(
727
1888
  project: Project,
728
1889
  storyboard: Storyboard,
729
1890
  silentVideo: string,
730
1891
  tmp: string,
731
1892
  introMs: number,
732
- pxScale: number
733
- ): Promise<string> {
734
- if (!(await exists(project.captionsCuesPath))) return silentVideo;
1893
+ pxScale: number,
1894
+ placement: CaptionPlacement = { defaultPosition: "bottom", windows: [], collisions: [] }
1895
+ ): Promise<{ video: string; cues: number; passes: number; flipped: number }> {
1896
+ const none = { video: silentVideo, cues: 0, passes: 0, flipped: 0 };
1897
+ if (placement.look?.style === "none") {
1898
+ log("captions: style none — not burned (SRT/VTT files still written)");
1899
+ return none;
1900
+ }
1901
+ if (!(await exists(project.captionsCuesPath))) return none;
735
1902
  const cues = (await readJson<Cue[]>(project.captionsCuesPath)) ?? [];
736
- if (cues.length === 0) return silentVideo;
1903
+ if (cues.length === 0) return none;
737
1904
 
738
1905
  log(`rendering ${cues.length} caption image(s)`);
739
1906
  const rendered = await renderCaptionPngs(
740
1907
  cues,
741
1908
  resolve(tmp, "captions"),
742
- storyboard.video.width,
743
- pxScale
1909
+ Math.round(placement.logicalWidth ?? storyboard.video.width),
1910
+ pxScale,
1911
+ placement.look ?? {}
744
1912
  );
745
1913
 
746
- const args: string[] = ["-i", silentVideo];
747
- for (const r of rendered) args.push("-i", r.png);
748
-
749
- // Chain one overlay per cue, each enabled only during its time window.
750
- // Cue times are narration-relative; the intro card shifts them right.
751
- const gap = Math.round(CAPTION_BOTTOM_GAP * pxScale);
752
- const chain: string[] = [];
753
- let prev = "0:v";
754
- rendered.forEach((r, i) => {
755
- const inp = `${i + 1}:v`;
756
- const out = i === rendered.length - 1 ? "vout" : `o${i}`;
757
- const a = ((r.startMs + introMs) / 1000).toFixed(3);
758
- const b = ((r.endMs + introMs) / 1000).toFixed(3);
759
- // Escape the enable-expression commas so ffmpeg doesn't read them as filter
760
- // separators. Lift the strip clear of the app's bottom input bar (the
761
- // ChatGPT composer) so captions never sit on top of the prompt being typed.
762
- chain.push(
763
- `[${prev}][${inp}]overlay=0:H-h-${gap}:enable=between(t\\,${a}\\,${b})[${out}]`
764
- );
765
- prev = out;
1914
+ // Cue times are narration-relative; the intro card shifts them right. The
1915
+ // strip sits `gap` above the bottom edge (clear of an app's bottom input
1916
+ // bar) — or, per scene / on a collision with an attention overlay, at the
1917
+ // top: the PNG is a bottom-anchored strip, so "top" flips it vertically.
1918
+ // A "bar" strip is flush with the frame edge (no gap, no pill offset).
1919
+ const bar = placement.look?.style === "bar";
1920
+ const gap = bar ? 0 : Math.round(CAPTION_BOTTOM_GAP * pxScale);
1921
+ const { height: outH } = await probeVideoDims(silentVideo);
1922
+ let flipped = 0;
1923
+ const overlap = (a: number, b: number, w: { a: number; b: number }) => a < w.b && b > w.a;
1924
+ const items: OverlayItem[] = rendered.map((r) => {
1925
+ let pos = placement.defaultPosition;
1926
+ const win = placement.windows.find((w) => r.startMs >= w.a && r.startMs < w.b);
1927
+ if (win) pos = win.position;
1928
+ else if (pos === "bottom" && placement.collisions.some((c) => overlap(r.startMs, r.endMs, c))) {
1929
+ pos = "top";
1930
+ flipped++;
1931
+ }
1932
+ const stripH = Math.round(r.height * pxScale);
1933
+ // The strip PNG is bottom-anchored (pill bottom ≈ 22 px above the strip's
1934
+ // edge). For "top", slide the strip up so the pill's bottom lands ~gap+70
1935
+ // px from the top edge — a one- or two-line pill stays fully in frame.
1936
+ const topY = bar
1937
+ ? -(stripH - Math.round(62 * pxScale))
1938
+ : gap + Math.round(70 * pxScale) - (stripH - Math.round(22 * pxScale));
1939
+ return {
1940
+ png: r.png,
1941
+ x: 0,
1942
+ y: pos === "top" ? topY : outH - stripH - gap,
1943
+ a: (r.startMs + introMs) / 1000,
1944
+ b: (r.endMs + introMs) / 1000,
1945
+ };
766
1946
  });
767
-
768
- const out = resolve(tmp, "captioned.mp4");
769
- await runFfmpeg([
770
- ...args,
771
- "-filter_complex",
772
- chain.join(";"),
773
- "-map",
774
- "[vout]",
775
- "-an",
776
- "-c:v",
777
- "libx264",
778
- "-preset",
779
- "veryfast",
780
- "-crf",
781
- "20",
782
- "-pix_fmt",
783
- "yuv420p",
784
- out,
785
- ]);
786
- return out;
1947
+ if (flipped) log(`captions: ${flipped} cue(s) moved to the top (overlay in the bottom band)`);
1948
+ const r = await overlayPngBatches(silentVideo, items, tmp, "captioned");
1949
+ return { video: r.video, cues: cues.length, passes: r.passes, flipped };
787
1950
  }
788
1951
 
789
1952
  /**
@@ -795,11 +1958,54 @@ async function burnCaptions(
795
1958
  * fixed -22dB bed). The bed is trimmed to the video, faded out at the end,
796
1959
  * and the narration is delayed past the intro card.
797
1960
  */
1961
+ /** Output sizing with a preset's defaults filled in (explicit keys win). */
1962
+ function resolveOutputSizing(output: Output | undefined): Output {
1963
+ if (!output) return {};
1964
+ const p = output.preset ? OUTPUT_PRESETS[output.preset] : null;
1965
+ return {
1966
+ ...output,
1967
+ width: output.width ?? p?.width,
1968
+ height: output.height ?? p?.height,
1969
+ // An explicit fit wins over the preset's (issue #58): `preset: "youtube"`
1970
+ // + `fit: "cover"` is how you get an edge-to-edge 16:9 cut from a 16:10
1971
+ // capture, and before this the preset silently forced "contain".
1972
+ fit: output.fit ?? p?.fit ?? "contain",
1973
+ };
1974
+ }
1975
+
1976
+ /** Write an ffmetadata file with one chapter per scene (content time + intro). */
1977
+ async function writeChapters(
1978
+ tmp: string,
1979
+ scenes: Array<{ id: string; title: string; durationMs: number }>
1980
+ ): Promise<string> {
1981
+ const lines = [";FFMETADATA1"];
1982
+ let t = 0;
1983
+ for (const s of scenes) {
1984
+ const a = Math.round(t);
1985
+ const b = Math.round(t + s.durationMs);
1986
+ lines.push("[CHAPTER]", "TIMEBASE=1/1000", `START=${a}`, `END=${b}`, `title=${s.title.replace(/[\\=;#\n]/g, " ")}`);
1987
+ t += s.durationMs;
1988
+ }
1989
+ const path = resolve(tmp, "chapters.ffmeta");
1990
+ await fs.writeFile(path, lines.join("\n") + "\n");
1991
+ log(`chapters: ${scenes.length} marker(s)`);
1992
+ return path;
1993
+ }
1994
+
1995
+ async function fileToDataUri(path: string): Promise<string> {
1996
+ const ext = path.toLowerCase().split(".").pop() ?? "png";
1997
+ const mime =
1998
+ ext === "svg" ? "image/svg+xml" : ext === "jpg" || ext === "jpeg" ? "image/jpeg" : "image/png";
1999
+ const buf = await fs.readFile(path);
2000
+ return `data:${mime};base64,${buf.toString("base64")}`;
2001
+ }
2002
+
798
2003
  async function muxAudio(
799
2004
  project: Project,
800
2005
  storyboard: Storyboard,
801
2006
  video: string,
802
- introMs: number
2007
+ introMs: number,
2008
+ chaptersPath: string | null = null
803
2009
  ): Promise<void> {
804
2010
  let musicPath = await resolveMusic(project, storyboard);
805
2011
  const videoMs = await probeDurationMs(video);
@@ -819,9 +2025,15 @@ async function muxAudio(
819
2025
  }
820
2026
  }
821
2027
 
822
- // Inputs, fixed order: 0=video, 1=narration, 2=music (if any).
2028
+ // Inputs, fixed order: 0=video, 1=narration, 2=music (if any), then the
2029
+ // chapters metadata file (mapped by index, never a stream).
823
2030
  const args: string[] = ["-i", video, "-i", project.narrationPath];
824
2031
  if (musicPath) args.push("-stream_loop", String(musicLoops), "-i", musicPath);
2032
+ let chaptersIdx = -1;
2033
+ if (chaptersPath) {
2034
+ chaptersIdx = musicPath ? 3 : 2;
2035
+ args.push("-i", chaptersPath);
2036
+ }
825
2037
 
826
2038
  const delay = introMs > 0 ? `,adelay=${Math.round(introMs)}:all=1` : "";
827
2039
  const filters: string[] = [];
@@ -892,6 +2104,7 @@ async function muxAudio(
892
2104
 
893
2105
  args.push("-filter_complex", filters.join(";"));
894
2106
  args.push("-map", "0:v", "-map", "[aout]");
2107
+ if (chaptersIdx >= 0) args.push("-map_metadata", String(chaptersIdx));
895
2108
  args.push(
896
2109
  "-c:v",
897
2110
  "copy",
@@ -899,6 +2112,10 @@ async function muxAudio(
899
2112
  "aac",
900
2113
  "-b:a",
901
2114
  "192k",
2115
+ // moov atom up front so the MP4 streams/scrubs in a browser tab or a PR
2116
+ // preview before it has fully downloaded (frames identical; container only).
2117
+ "-movflags",
2118
+ "+faststart",
902
2119
  "-t",
903
2120
  durSec,
904
2121
  project.outputPath